diff --git a/.gitignore b/.gitignore
index eb1fc96..2a82ba7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,10 @@
target
build
+# Filesystem contract test options and credentials
+auth-keys.xml
+azure-auth-keys.xml
+
# External tool builders
*/.externalToolBuilders
*/maven-eclipse.xml
@@ -22,12 +26,6 @@ build
hadoop-common-project/hadoop-kms/downloads/
hadoop-hdfs-project/hadoop-hdfs/downloads
hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads
-hadoop-common-project/hadoop-common/src/test/resources/contract-test-options.xml
-hadoop-tools/hadoop-openstack/src/test/resources/contract-test-options.xml
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/tla/yarnregistry.toolbox
yarnregistry.pdf
-hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml
-hadoop-tools/hadoop-aws/src/test/resources/contract-test-options.xml
-hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml
-hadoop-tools/hadoop-openstack/src/test/resources/auth-keys.xml
patchprocess/
diff --git a/LICENSE.txt b/LICENSE.txt
index 0e4b492..61ebbd6 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -345,6 +345,38 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+For hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h:
+
+Copyright (c) 2012 The FreeBSD Foundation
+All rights reserved.
+
+This software was developed by Pawel Jakub Dawidek under sponsorship from
+the FreeBSD Foundation.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGE.
+
+=============
+
The binary distribution of this product bundles binaries of leveldb
(http://code.google.com/p/leveldb/), which is available under the following
license:
diff --git a/hadoop-assemblies/pom.xml b/hadoop-assemblies/pom.xml
index 0ec1dc6..91501f6 100644
--- a/hadoop-assemblies/pom.xml
+++ b/hadoop-assemblies/pom.xml
@@ -23,12 +23,12 @@
org.apache.hadoophadoop-project
- 2.8.0-SNAPSHOT
+ 2.8.0../hadoop-projectorg.apache.hadoophadoop-assemblies
- 2.8.0-SNAPSHOT
+ 2.8.0Apache Hadoop AssembliesApache Hadoop Assemblies
diff --git a/hadoop-build-tools/pom.xml b/hadoop-build-tools/pom.xml
index 02b7862..71a80dc 100644
--- a/hadoop-build-tools/pom.xml
+++ b/hadoop-build-tools/pom.xml
@@ -18,7 +18,7 @@
hadoop-mainorg.apache.hadoop
- 2.8.0-SNAPSHOT
+ 2.8.04.0.0hadoop-build-tools
diff --git a/hadoop-client/pom.xml b/hadoop-client/pom.xml
index c8d7fa9..7e28e6a 100644
--- a/hadoop-client/pom.xml
+++ b/hadoop-client/pom.xml
@@ -18,12 +18,12 @@
org.apache.hadoophadoop-project-dist
- 2.8.0-SNAPSHOT
+ 2.8.0../hadoop-project-distorg.apache.hadoophadoop-client
- 2.8.0-SNAPSHOT
+ 2.8.0jarApache Hadoop Client
@@ -100,7 +100,7 @@
org.apache.hadoop
- hadoop-hdfs-client
+ hadoop-hdfscompile
@@ -123,6 +123,30 @@
javax.servletservlet-api
+
+ io.netty
+ netty
+
+
+ io.netty
+ netty-all
+
+
+ xerces
+ xercesImpl
+
+
+ commons-daemon
+ commons-daemon
+
+
+ org.mortbay.jetty
+ jetty-util
+
+
+ org.fusesource.leveldbjni
+ leveldbjni-all
+
diff --git a/hadoop-common-project/hadoop-annotations/pom.xml b/hadoop-common-project/hadoop-annotations/pom.xml
index c2bbc5f..dac090a 100644
--- a/hadoop-common-project/hadoop-annotations/pom.xml
+++ b/hadoop-common-project/hadoop-annotations/pom.xml
@@ -20,12 +20,12 @@
org.apache.hadoophadoop-project
- 2.8.0-SNAPSHOT
+ 2.8.0../../hadoop-projectorg.apache.hadoophadoop-annotations
- 2.8.0-SNAPSHOT
+ 2.8.0Apache Hadoop AnnotationsApache Hadoop Annotationsjar
diff --git a/hadoop-common-project/hadoop-auth-examples/pom.xml b/hadoop-common-project/hadoop-auth-examples/pom.xml
index 20f1012..d9a0f99 100644
--- a/hadoop-common-project/hadoop-auth-examples/pom.xml
+++ b/hadoop-common-project/hadoop-auth-examples/pom.xml
@@ -20,12 +20,12 @@
org.apache.hadoophadoop-project
- 2.8.0-SNAPSHOT
+ 2.8.0../../hadoop-projectorg.apache.hadoophadoop-auth-examples
- 2.8.0-SNAPSHOT
+ 2.8.0warApache Hadoop Auth Examples
diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml
index 9d99a05..3ebbc06 100644
--- a/hadoop-common-project/hadoop-auth/pom.xml
+++ b/hadoop-common-project/hadoop-auth/pom.xml
@@ -20,12 +20,12 @@
org.apache.hadoophadoop-project
- 2.8.0-SNAPSHOT
+ 2.8.0../../hadoop-projectorg.apache.hadoophadoop-auth
- 2.8.0-SNAPSHOT
+ 2.8.0jarApache Hadoop Auth
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 42a2f21..4da088a 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -20,12 +20,12 @@
org.apache.hadoophadoop-project-dist
- 2.8.0-SNAPSHOT
+ 2.8.0../../hadoop-project-distorg.apache.hadoophadoop-common
- 2.8.0-SNAPSHOT
+ 2.8.0Apache Hadoop CommonApache Hadoop Commonjar
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index c144711..04946ad 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -71,7 +71,6 @@
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
-import static org.apache.hadoop.ipc.RpcConstants.CONNECTION_CONTEXT_CALL_ID;
import static org.apache.hadoop.ipc.RpcConstants.PING_CALL_ID;
/** A client for an IPC service. IPC calls take a single {@link Writable} as a
@@ -1767,7 +1766,9 @@ public void close() throws Exception {
}
void setSaslClient(SaslRpcClient client) throws IOException {
- setInputStream(client.getInputStream(in));
+ // Wrap the input stream in a BufferedInputStream to fill the buffer
+ // before reading its length (HADOOP-14062).
+ setInputStream(new BufferedInputStream(client.getInputStream(in)));
setOutputStream(client.getOutputStream(out));
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
index 21d3dd6..4b14059 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
@@ -21,7 +21,7 @@
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.IOException;
-import java.util.Arrays;
+import java.security.MessageDigest;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -467,7 +467,7 @@ public synchronized String getTokenTrackingId(TokenIdent identifier) {
public synchronized void verifyToken(TokenIdent identifier, byte[] password)
throws InvalidToken {
byte[] storedPassword = retrievePassword(identifier);
- if (!Arrays.equals(password, storedPassword)) {
+ if (!MessageDigest.isEqual(password, storedPassword)) {
throw new InvalidToken("token " + formatTokenId(identifier)
+ " is invalid, password doesn't match");
}
@@ -516,7 +516,7 @@ public synchronized long renewToken(Token token,
+ id.getSequenceNumber());
}
byte[] password = createPassword(token.getIdentifier(), key.getKey());
- if (!Arrays.equals(password, token.getPassword())) {
+ if (!MessageDigest.isEqual(password, token.getPassword())) {
throw new AccessControlException(renewer
+ " is trying to renew a token "
+ formatTokenId(id) + " with wrong password");
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 900cca0..b119bc7 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -827,6 +827,15 @@
+ fs.s3a.security.credential.provider.path
+
+
+ Optional comma separated list of credential providers, a list
+ which is prepended to that set in hadoop.security.credential.provider.path
+
+
+
+fs.s3a.connection.maximum15Controls the maximum number of simultaneous connections to S3.
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md b/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
index 66c25e5..a1c2307 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
@@ -208,7 +208,7 @@ The following parameters can be used to control the node health monitoring scrip
|:---- |:---- |:---- |
| `yarn.nodemanager.health-checker.script.path` | Node health script | Script to check for node's health status. |
| `yarn.nodemanager.health-checker.script.opts` | Node health script options | Options for script to check for node's health status. |
-| `yarn.nodemanager.health-checker.script.interval-ms` | Node health script interval | Time interval for running health script. |
+| `yarn.nodemanager.health-checker.interval-ms` | Node health script interval | Time interval for running health script. |
| `yarn.nodemanager.health-checker.script.timeout-ms` | Node health script timeout interval | Timeout for health script execution. |
The health checker script is not supposed to give ERROR if only some of the local disks become bad. NodeManager has the ability to periodically check the health of the local disks (specifically checks nodemanager-local-dirs and nodemanager-log-dirs) and after reaching the threshold of number of bad directories based on the value set for the config property yarn.nodemanager.disk-health-checker.min-healthy-disks, the whole node is marked unhealthy and this info is sent to resource manager also. The boot disk is either raided or a failure in the boot disk is identified by the health checker script.
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md b/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
index 209b48d..de871b1 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
@@ -102,6 +102,7 @@ In summary, first, provision the credentials into a provider then configure the
|YARN |WebAppUtils uptakes the use of the credential provider API through the new method on Configuration called getPassword. This provides an alternative to storing the passwords in clear text within the ssl-server.xml file while maintaining backward compatibility.|TODO|
|AWS S3/S3A |Uses Configuration.getPassword to get the S3 credentials. They may be resolved through the credential provider API or from the config for backward compatibility.|[AWS S3/S3A Usage](../../hadoop-aws/tools/hadoop-aws/index.html)|
|Azure WASB |Uses Configuration.getPassword to get the WASB credentials. They may be resolved through the credential provider API or from the config for backward compatibility.|[Azure WASB Usage](../../hadoop-azure/index.html)|
+|Azure ADLS |Uses Configuration.getPassword to get the ADLS credentials. They may be resolved through the credential provider API or from the config for backward compatibility.|[Azure ADLS Usage](../../hadoop-azure-datalake/index.html)|
|Apache Accumulo|The trace.password property is used by the Tracer to authenticate with Accumulo and persist the traces in the trace table. The credential provider API is used to acquire the trace.password from a provider or from configuration for backward compatibility.|TODO|
|Apache Slider |A capability has been added to Slider to prompt the user for needed passwords and store them using CredentialProvider so they can be retrieved by an app later.|TODO|
|Apache Hive |Protection of the metastore password, SSL related passwords and JDO string password has been added through the use of the Credential Provider API|TODO|
diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml
index 1181dfb..dac856f 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -22,12 +22,12 @@
org.apache.hadoophadoop-project
- 2.8.0-SNAPSHOT
+ 2.8.0../../hadoop-projectorg.apache.hadoophadoop-kms
- 2.8.0-SNAPSHOT
+ 2.8.0warApache Hadoop KMS
diff --git a/hadoop-common-project/hadoop-minikdc/pom.xml b/hadoop-common-project/hadoop-minikdc/pom.xml
index 5470a52..1920b3f 100644
--- a/hadoop-common-project/hadoop-minikdc/pom.xml
+++ b/hadoop-common-project/hadoop-minikdc/pom.xml
@@ -18,13 +18,13 @@
org.apache.hadoophadoop-project
- 2.8.0-SNAPSHOT
+ 2.8.0../../hadoop-project4.0.0org.apache.hadoophadoop-minikdc
- 2.8.0-SNAPSHOT
+ 2.8.0Apache Hadoop MiniKDCApache Hadoop MiniKDCjar
diff --git a/hadoop-common-project/hadoop-nfs/pom.xml b/hadoop-common-project/hadoop-nfs/pom.xml
index 932c23e..de787e2 100644
--- a/hadoop-common-project/hadoop-nfs/pom.xml
+++ b/hadoop-common-project/hadoop-nfs/pom.xml
@@ -20,12 +20,12 @@
org.apache.hadoophadoop-project
- 2.8.0-SNAPSHOT
+ 2.8.0../../hadoop-projectorg.apache.hadoophadoop-nfs
- 2.8.0-SNAPSHOT
+ 2.8.0jarApache Hadoop NFS
diff --git a/hadoop-common-project/pom.xml b/hadoop-common-project/pom.xml
index dc6f150..fbcc215 100644
--- a/hadoop-common-project/pom.xml
+++ b/hadoop-common-project/pom.xml
@@ -20,12 +20,12 @@
org.apache.hadoophadoop-project
- 2.8.0-SNAPSHOT
+ 2.8.0../hadoop-projectorg.apache.hadoophadoop-common-project
- 2.8.0-SNAPSHOT
+ 2.8.0Apache Hadoop Common ProjectApache Hadoop Common Projectpom
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index 2708ce0..7c164c5 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -20,12 +20,12 @@
org.apache.hadoophadoop-project
- 2.8.0-SNAPSHOT
+ 2.8.0../hadoop-projectorg.apache.hadoophadoop-dist
- 2.8.0-SNAPSHOT
+ 2.8.0Apache Hadoop DistributionApache Hadoop Distributionjar
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
index 0cf0d84..0f17629 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
@@ -20,12 +20,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
org.apache.hadoophadoop-project-dist
- 2.8.0-SNAPSHOT
+ 2.8.0../../hadoop-project-distorg.apache.hadoophadoop-hdfs-client
- 2.8.0-SNAPSHOT
+ 2.8.0Apache Hadoop HDFS ClientApache Hadoop HDFS Clientjar
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 463ce23..873fb03 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -505,33 +505,36 @@ else if (offset >= locatedBlocks.getFileLength()) {
}
else {
// search cached blocks first
- int targetBlockIdx = locatedBlocks.findBlock(offset);
- if (targetBlockIdx < 0) { // block is not cached
- targetBlockIdx = LocatedBlocks.getInsertIndex(targetBlockIdx);
- // fetch more blocks
- final LocatedBlocks newBlocks = dfsClient.getLocatedBlocks(src, offset);
- assert (newBlocks != null) : "Could not find target position " + offset;
- locatedBlocks.insertRange(targetBlockIdx, newBlocks.getLocatedBlocks());
- }
- blk = locatedBlocks.get(targetBlockIdx);
+ blk = fetchBlockAt(offset, 0, true);
}
return blk;
}
}
/** Fetch a block from namenode and cache it */
- protected void fetchBlockAt(long offset) throws IOException {
+ protected LocatedBlock fetchBlockAt(long offset) throws IOException {
+ return fetchBlockAt(offset, 0, false); // don't use cache
+ }
+
+ /** Fetch a block from namenode and cache it */
+ private LocatedBlock fetchBlockAt(long offset, long length, boolean useCache)
+ throws IOException {
synchronized(infoLock) {
int targetBlockIdx = locatedBlocks.findBlock(offset);
if (targetBlockIdx < 0) { // block is not cached
targetBlockIdx = LocatedBlocks.getInsertIndex(targetBlockIdx);
+ useCache = false;
}
- // fetch blocks
- final LocatedBlocks newBlocks = dfsClient.getLocatedBlocks(src, offset);
- if (newBlocks == null) {
- throw new IOException("Could not find target position " + offset);
+ if (!useCache) { // fetch blocks
+ final LocatedBlocks newBlocks = (length == 0)
+ ? dfsClient.getLocatedBlocks(src, offset)
+ : dfsClient.getLocatedBlocks(src, offset, length);
+ if (newBlocks == null || newBlocks.locatedBlockCount() == 0) {
+ throw new EOFException("Could not find target position " + offset);
+ }
+ locatedBlocks.insertRange(targetBlockIdx, newBlocks.getLocatedBlocks());
}
- locatedBlocks.insertRange(targetBlockIdx, newBlocks.getLocatedBlocks());
+ return locatedBlocks.get(targetBlockIdx);
}
}
@@ -586,28 +589,15 @@ protected void fetchBlockAt(long offset) throws IOException {
assert (locatedBlocks != null) : "locatedBlocks is null";
List blockRange = new ArrayList<>();
// search cached blocks first
- int blockIdx = locatedBlocks.findBlock(offset);
- if (blockIdx < 0) { // block is not cached
- blockIdx = LocatedBlocks.getInsertIndex(blockIdx);
- }
long remaining = length;
long curOff = offset;
while(remaining > 0) {
- LocatedBlock blk = null;
- if(blockIdx < locatedBlocks.locatedBlockCount())
- blk = locatedBlocks.get(blockIdx);
- if (blk == null || curOff < blk.getStartOffset()) {
- LocatedBlocks newBlocks;
- newBlocks = dfsClient.getLocatedBlocks(src, curOff, remaining);
- locatedBlocks.insertRange(blockIdx, newBlocks.getLocatedBlocks());
- continue;
- }
+ LocatedBlock blk = fetchBlockAt(curOff, remaining, true);
assert curOff >= blk.getStartOffset() : "Block not found";
blockRange.add(blk);
long bytesRead = blk.getStartOffset() + blk.getBlockSize() - curOff;
remaining -= bytesRead;
curOff += bytesRead;
- blockIdx++;
}
return blockRange;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index 451e2ab..7e2eb9e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -88,6 +88,7 @@ message DatanodeInfoProto {
optional uint64 lastUpdate = 6 [default = 0];
optional uint32 xceiverCount = 7 [default = 0];
optional string location = 8;
+ optional uint64 nonDfsUsed = 9;
enum AdminState {
NORMAL = 0;
DECOMMISSION_INPROGRESS = 1;
@@ -99,7 +100,6 @@ message DatanodeInfoProto {
optional uint64 cacheUsed = 12 [default = 0];
optional uint64 lastUpdateMonotonic = 13 [default = 0];
optional string upgradeDomain = 14;
- optional uint64 nonDfsUsed = 15;
}
/**
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index f48b439..74371f2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -22,12 +22,12 @@
org.apache.hadoophadoop-project
- 2.8.0-SNAPSHOT
+ 2.8.0../../hadoop-projectorg.apache.hadoophadoop-hdfs-httpfs
- 2.8.0-SNAPSHOT
+ 2.8.0warApache Hadoop HttpFS
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
index f4f9033..f7b9e3e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
@@ -20,12 +20,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
org.apache.hadoophadoop-project-dist
- 2.8.0-SNAPSHOT
+ 2.8.0../../hadoop-project-distorg.apache.hadoophadoop-hdfs-native-client
- 2.8.0-SNAPSHOT
+ 2.8.0Apache Hadoop HDFS Native ClientApache Hadoop HDFS Native Clientjar
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
index b09905b..b20e6e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
@@ -20,12 +20,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
org.apache.hadoophadoop-project
- 2.8.0-SNAPSHOT
+ 2.8.0../../hadoop-projectorg.apache.hadoophadoop-hdfs-nfs
- 2.8.0-SNAPSHOT
+ 2.8.0Apache Hadoop HDFS-NFSApache Hadoop HDFS-NFSjar
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 824b0ca..ad8fa59 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -20,12 +20,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
org.apache.hadoophadoop-project-dist
- 2.8.0-SNAPSHOT
+ 2.8.0../../hadoop-project-distorg.apache.hadoophadoop-hdfs
- 2.8.0-SNAPSHOT
+ 2.8.0Apache Hadoop HDFSApache Hadoop HDFSjar
@@ -184,11 +184,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
compile
- com.twitter
- hpack
- compile
-
- xercesxercesImplcompile
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
index 45e8e68..ab1329e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
@@ -20,13 +20,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
org.apache.hadoophadoop-project
- 2.8.0-SNAPSHOT
+ 2.8.0../../../../../hadoop-projectorg.apache.hadoop.contribhadoop-hdfs-bkjournal
- 2.8.0-SNAPSHOT
+ 2.8.0Apache Hadoop HDFS BookKeeper JournalApache Hadoop HDFS BookKeeper Journaljar
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
index caee6cc..ad830f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
@@ -26,8 +26,8 @@
import javax.servlet.ServletContext;
import javax.servlet.ServletException;
+import io.netty.bootstrap.ChannelFactory;
import io.netty.bootstrap.ServerBootstrap;
-import io.netty.channel.ChannelFactory;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
@@ -144,8 +144,16 @@ public DatanodeHttpServer(final Configuration conf,
.childHandler(new ChannelInitializer() {
@Override
protected void initChannel(SocketChannel ch) throws Exception {
- ch.pipeline().addLast(new PortUnificationServerHandler(jettyAddr,
- conf, confForCreate, restCsrfPreventionFilter));
+ ChannelPipeline p = ch.pipeline();
+ p.addLast(new HttpRequestDecoder(),
+ new HttpResponseEncoder());
+ if (restCsrfPreventionFilter != null) {
+ p.addLast(new RestCsrfPreventionFilterHandler(
+ restCsrfPreventionFilter));
+ }
+ p.addLast(
+ new ChunkedWriteHandler(),
+ new URLDispatcher(jettyAddr, conf, confForCreate));
}
});
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/PortUnificationServerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/PortUnificationServerHandler.java
deleted file mode 100644
index ff10c6d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/PortUnificationServerHandler.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.datanode.web;
-
-import java.net.InetSocketAddress;
-import java.util.List;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.datanode.web.dtp.DtpHttp2Handler;
-import org.apache.hadoop.security.http.RestCsrfPreventionFilter;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufUtil;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.handler.codec.ByteToMessageDecoder;
-import io.netty.handler.codec.http.HttpServerCodec;
-import io.netty.handler.codec.http2.Http2CodecUtil;
-import io.netty.handler.stream.ChunkedWriteHandler;
-
-/**
- * A port unification handler to support HTTP/1.1 and HTTP/2 on the same port.
- */
-@InterfaceAudience.Private
-public class PortUnificationServerHandler extends ByteToMessageDecoder {
-
- private static final ByteBuf HTTP2_CLIENT_CONNECTION_PREFACE = Http2CodecUtil
- .connectionPrefaceBuf();
-
- // we only want to support HTTP/1.1 and HTTP/2, so the first 3 bytes is
- // enough. No HTTP/1.1 request could start with "PRI"
- private static final int MAGIC_HEADER_LENGTH = 3;
-
- private final InetSocketAddress proxyHost;
-
- private final Configuration conf;
-
- private final Configuration confForCreate;
-
- private final RestCsrfPreventionFilter restCsrfPreventionFilter;
-
- public PortUnificationServerHandler(InetSocketAddress proxyHost,
- Configuration conf, Configuration confForCreate,
- RestCsrfPreventionFilter restCsrfPreventionFilter) {
- this.proxyHost = proxyHost;
- this.conf = conf;
- this.confForCreate = confForCreate;
- this.restCsrfPreventionFilter = restCsrfPreventionFilter;
- }
-
- private void configureHttp1(ChannelHandlerContext ctx) {
- ctx.pipeline().addLast(new HttpServerCodec());
- if (this.restCsrfPreventionFilter != null) {
- ctx.pipeline().addLast(new RestCsrfPreventionFilterHandler(
- this.restCsrfPreventionFilter));
- }
- ctx.pipeline().addLast(new ChunkedWriteHandler(),
- new URLDispatcher(proxyHost, conf, confForCreate));
- }
-
- private void configureHttp2(ChannelHandlerContext ctx) {
- if (this.restCsrfPreventionFilter != null) {
- ctx.pipeline().addLast(new RestCsrfPreventionFilterHandler(
- this.restCsrfPreventionFilter));
- }
- ctx.pipeline().addLast(new DtpHttp2Handler());
- }
-
- @Override
- protected void decode(ChannelHandlerContext ctx, ByteBuf in,
- List
+### Configurations different S3 buckets
+
+Different S3 buckets can be accessed with different S3A client configurations.
+This allows for different endpoints, data read and write strategies, as well
+as login details.
+
+1. All `fs.s3a` options other than a small set of unmodifiable values
+ (currently `fs.s3a.impl`) can be set on a per bucket basis.
+1. The bucket specific option is set by replacing the `fs.s3a.` prefix on an option
+with `fs.s3a.bucket.BUCKETNAME.`, where `BUCKETNAME` is the name of the bucket.
+1. When connecting to a bucket, all options explicitly set will override
+the base `fs.s3a.` values.
+
+As an example, a configuration could have a base configuration to use the IAM
+role information available when deployed in Amazon EC2.
+
+```xml
+
+ fs.s3a.aws.credentials.provider
+ org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider
+
+```
+
+This will be the default authentication mechanism for S3A buckets.
+
+A bucket `s3a://nightly/` used for nightly data uses a session key:
+
+```xml
+
+ fs.s3a.bucket.nightly.access.key
+ AKAACCESSKEY-2
+
+
+
+ fs.s3a.bucket.nightly.secret.key
+ SESSIONSECRETKEY
+
+
+
+ fs.s3a.bucket.nightly.session.token
+ Short-lived-session-token
+
+
+
+ fs.s3a.bucket.nightly.aws.credentials.provider
+ org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider
+
+```
+
+Finally, the public `s3a://landsat-pds/` bucket is accessed anonymously:
+
+```xml
+
+ fs.s3a.bucket.landsat-pds.aws.credentials.provider
+ org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider
+
+```
+
+**Customizing S3A secrets held in credential files**
+
+Although most properties are automatically propagated from their
+`fs.s3a.bucket.`-prefixed custom entry to that of the base `fs.s3a.` option
+supporting secrets kept in Hadoop credential files is slightly more complex.
+This is because the property values are kept in these files, and cannot be
+dynamically patched.
+
+Instead, callers need to create different configuration files for each
+bucket, setting the base secrets (`fs.s3a.bucket.nightly.access.key`, etc),
+then declare the path to the appropriate credential file in
+a bucket-specific version of the property `fs.s3a.security.credential.provider.path`.
+
+
### Working with buckets in different regions
S3 Buckets are hosted in different regions, the default being US-East.
@@ -953,6 +1049,16 @@ If the wrong endpoint is used, the request may fail. This may be reported as a 3
or as a 400 Bad Request.
+If you are trying to mix endpoints for different buckets, use a per-bucket endpoint
+declaration. For example:
+
+```xml
+
+ fs.s3a.bucket.landsat-pds.endpoint
+ s3.amazonaws.com
+ The endpoint for s3a://landsat-pds URLs
+
+```
### Stabilizing: S3A Fast Upload
@@ -1632,15 +1738,15 @@ org.apache.hadoop.fs.s3a.AWSS3IOException: Received permanent redirect response
1. If not using "V4" authentication (see above), the original S3 endpoint
can be used:
-```
-
- fs.s3a.endpoint
- s3.amazonaws.com
-
+```xml
+
+ fs.s3a.endpoint
+ s3.amazonaws.com
+
```
-Using the explicit endpoint for the region is recommended for speed and the
-ability to use the V4 signing API.
+Using the explicit endpoint for the region is recommended for speed and
+to use the V4 signing API.
### "Timeout waiting for connection from pool" when writing to S3A
@@ -2226,32 +2332,33 @@ is hosted in Amazon's US-east datacenter.
1. If the property is set to a different path, then that data must be readable
and "sufficiently" large.
-To test on different S3 endpoints, or alternate infrastructures supporting
-the same APIs, the option `fs.s3a.scale.test.csvfile` must either be
-set to " ", or an object of at least 10MB is uploaded to the object store, and
-the `fs.s3a.scale.test.csvfile` option set to its path.
+(the reason the space or newline is needed is to add "an empty entry"; an empty
+`` would be considered undefined and pick up the default)
+
+Of using a test file in an S3 region requiring a different endpoint value
+set in `fs.s3a.endpoint`, a bucket-specific endpoint must be defined.
+For the default test dataset, hosted in the `landsat-pds` bucket, this is:
```xml
- fs.s3a.scale.test.csvfile
-
+ fs.s3a.bucket.landsat-pds.endpoint
+ s3.amazonaws.com
+ The endpoint for s3a://landsat-pds URLs
```
-(the reason the space or newline is needed is to add "an empty entry"; an empty
-`` would be considered undefined and pick up the default)
-
-*Note:* if using a test file in an S3 region requiring a different endpoint value
-set in `fs.s3a.endpoint`, define it in `fs.s3a.scale.test.csvfile.endpoint`.
-If the default CSV file is used, the tests will automatically use the us-east
-endpoint:
+To test on alternate infrastructures supporting
+the same APIs, the option `fs.s3a.scale.test.csvfile` must either be
+set to " ", or an object of at least 10MB is uploaded to the object store, and
+the `fs.s3a.scale.test.csvfile` option set to its path.
```xml
- fs.s3a.scale.test.csvfile.endpoint
- s3.amazonaws.com
+ fs.s3a.scale.test.csvfile
+
```
+
### Viewing Integration Test Reports
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
index 819d9d8..1c32996 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
@@ -128,7 +128,6 @@ public void testAnonymousProvider() throws Exception {
AnonymousAWSCredentialsProvider.class.getName());
Path testFile = new Path(
conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
- S3ATestUtils.useCSVDataEndpoint(conf);
FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf);
assertNotNull(fs);
assertTrue(fs instanceof S3AFileSystem);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java
index 74cad00..87f676c 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputArray.java
@@ -24,9 +24,12 @@
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.io.IOUtils;
+import org.junit.BeforeClass;
import org.junit.Test;
import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
import static org.apache.hadoop.fs.s3a.Constants.*;
@@ -38,6 +41,14 @@
* multipart tests are kept in scale tests.
*/
public class ITestS3ABlockOutputArray extends AbstractS3ATestBase {
+ private static final int BLOCK_SIZE = 256 * 1024;
+
+ private static byte[] dataset;
+
+ @BeforeClass
+ public static void setupDataset() {
+ dataset = ContractTestUtils.dataset(BLOCK_SIZE, 0, 256);
+ }
@Override
protected Configuration createConfiguration() {
@@ -65,9 +76,9 @@ public void testRegularUpload() throws IOException {
}
@Test(expected = IOException.class)
- public void testDoubleStreamClose() throws Throwable {
- Path dest = path("testDoubleStreamClose");
- describe(" testDoubleStreamClose");
+ public void testWriteAfterStreamClose() throws Throwable {
+ Path dest = path("testWriteAfterStreamClose");
+ describe(" testWriteAfterStreamClose");
FSDataOutputStream stream = getFileSystem().create(dest, true);
byte[] data = ContractTestUtils.dataset(16, 'a', 26);
try {
@@ -79,7 +90,25 @@ public void testDoubleStreamClose() throws Throwable {
}
}
- public void verifyUpload(String name, int fileSize) throws IOException {
+ @Test
+ public void testBlocksClosed() throws Throwable {
+ Path dest = path("testBlocksClosed");
+ describe(" testBlocksClosed");
+ FSDataOutputStream stream = getFileSystem().create(dest, true);
+ S3AInstrumentation.OutputStreamStatistics statistics
+ = S3ATestUtils.getOutputStreamStatistics(stream);
+ byte[] data = ContractTestUtils.dataset(16, 'a', 26);
+ stream.write(data);
+ LOG.info("closing output stream");
+ stream.close();
+ assertEquals("total allocated blocks in " + statistics,
+ 1, statistics.blocksAllocated());
+ assertEquals("actively allocated blocks in " + statistics,
+ 0, statistics.blocksActivelyAllocated());
+ LOG.info("end of test case");
+ }
+
+ private void verifyUpload(String name, int fileSize) throws IOException {
Path dest = path(name);
describe(name + " upload to " + dest);
ContractTestUtils.createAndVerifyFile(
@@ -87,4 +116,43 @@ public void verifyUpload(String name, int fileSize) throws IOException {
dest,
fileSize);
}
+
+ /**
+ * Create a factory for used in mark/reset tests.
+ * @param fileSystem source FS
+ * @return the factory
+ */
+ protected S3ADataBlocks.BlockFactory createFactory(S3AFileSystem fileSystem) {
+ return new S3ADataBlocks.ArrayBlockFactory(fileSystem);
+ }
+
+ private void markAndResetDatablock(S3ADataBlocks.BlockFactory factory)
+ throws Exception {
+ S3AInstrumentation instrumentation =
+ new S3AInstrumentation(new URI("s3a://example"));
+ S3AInstrumentation.OutputStreamStatistics outstats
+ = instrumentation.newOutputStreamStatistics(null);
+ S3ADataBlocks.DataBlock block = factory.create(1, BLOCK_SIZE, outstats);
+ block.write(dataset, 0, dataset.length);
+ S3ADataBlocks.BlockUploadData uploadData = block.startUpload();
+ InputStream stream = uploadData.getUploadStream();
+ assertNotNull(stream);
+ assertTrue("Mark not supported in " + stream, stream.markSupported());
+ assertEquals(0, stream.read());
+ stream.mark(BLOCK_SIZE);
+ // read a lot
+ long l = 0;
+ while (stream.read() != -1) {
+ // do nothing
+ l++;
+ }
+ stream.reset();
+ assertEquals(1, stream.read());
+ }
+
+ @Test
+ public void testMarkReset() throws Throwable {
+ markAndResetDatablock(createFactory(getFileSystem()));
+ }
+
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputByteBuffer.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputByteBuffer.java
index 504426b..02f3de0 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputByteBuffer.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputByteBuffer.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.fs.s3a;
-
/**
* Use {@link Constants#FAST_UPLOAD_BYTEBUFFER} for buffering.
*/
@@ -27,4 +26,8 @@ protected String getBlockOutputBufferName() {
return Constants.FAST_UPLOAD_BYTEBUFFER;
}
+ protected S3ADataBlocks.BlockFactory createFactory(S3AFileSystem fileSystem) {
+ return new S3ADataBlocks.ByteBufferBlockFactory(fileSystem);
+ }
+
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputDisk.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputDisk.java
index 550706d..abe8656 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputDisk.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputDisk.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.fs.s3a;
+import org.junit.Assume;
+
/**
* Use {@link Constants#FAST_UPLOAD_BUFFER_DISK} for buffering.
*/
@@ -27,4 +29,14 @@ protected String getBlockOutputBufferName() {
return Constants.FAST_UPLOAD_BUFFER_DISK;
}
+ /**
+ * The disk stream doesn't support mark/reset; calls
+ * {@code Assume} to skip the test.
+ * @param fileSystem source FS
+ * @return null
+ */
+ protected S3ADataBlocks.BlockFactory createFactory(S3AFileSystem fileSystem) {
+ Assume.assumeTrue("mark/reset nopt supoprted", false);
+ return null;
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
index baf0f79..6625dc2 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
@@ -35,6 +35,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.fs.s3a.S3ATestConstants.TEST_FS_S3A_NAME;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
@@ -45,6 +46,7 @@
import java.io.File;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
+import java.util.Collection;
import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.UserGroupInformation;
@@ -54,6 +56,10 @@
import org.apache.http.HttpStatus;
import org.junit.rules.TemporaryFolder;
+import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.S3AUtils.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+
/**
* S3A tests for configuration.
*/
@@ -535,4 +541,128 @@ public S3AFileSystem run() throws Exception{
fieldType.isAssignableFrom(obj.getClass()));
return fieldType.cast(obj);
}
+
+ @Test
+ public void testBucketConfigurationPropagation() throws Throwable {
+ Configuration config = new Configuration(false);
+ setBucketOption(config, "b", "base", "1024");
+ String basekey = "fs.s3a.base";
+ assertOptionEquals(config, basekey, null);
+ String bucketKey = "fs.s3a.bucket.b.base";
+ assertOptionEquals(config, bucketKey, "1024");
+ Configuration updated = propagateBucketOptions(config, "b");
+ assertOptionEquals(updated, basekey, "1024");
+ // original conf is not updated
+ assertOptionEquals(config, basekey, null);
+
+ String[] sources = updated.getPropertySources(basekey);
+ assertEquals(1, sources.length);
+ String sourceInfo = sources[0];
+ assertTrue("Wrong source " + sourceInfo, sourceInfo.contains(bucketKey));
+ }
+
+ @Test
+ public void testBucketConfigurationPropagationResolution() throws Throwable {
+ Configuration config = new Configuration(false);
+ String basekey = "fs.s3a.base";
+ String baseref = "fs.s3a.baseref";
+ String baseref2 = "fs.s3a.baseref2";
+ config.set(basekey, "orig");
+ config.set(baseref2, "${fs.s3a.base}");
+ setBucketOption(config, "b", basekey, "1024");
+ setBucketOption(config, "b", baseref, "${fs.s3a.base}");
+ Configuration updated = propagateBucketOptions(config, "b");
+ assertOptionEquals(updated, basekey, "1024");
+ assertOptionEquals(updated, baseref, "1024");
+ assertOptionEquals(updated, baseref2, "1024");
+ }
+
+ @Test
+ public void testMultipleBucketConfigurations() throws Throwable {
+ Configuration config = new Configuration(false);
+ setBucketOption(config, "b", USER_AGENT_PREFIX, "UA-b");
+ setBucketOption(config, "c", USER_AGENT_PREFIX, "UA-c");
+ config.set(USER_AGENT_PREFIX, "UA-orig");
+ Configuration updated = propagateBucketOptions(config, "c");
+ assertOptionEquals(updated, USER_AGENT_PREFIX, "UA-c");
+ }
+
+ @Test
+ public void testBucketConfigurationSkipsUnmodifiable() throws Throwable {
+ Configuration config = new Configuration(false);
+ String impl = "fs.s3a.impl";
+ config.set(impl, "orig");
+ setBucketOption(config, "b", impl, "b");
+ String metastoreImpl = "fs.s3a.metadatastore.impl";
+ String ddb = "org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore";
+ setBucketOption(config, "b", metastoreImpl, ddb);
+ setBucketOption(config, "b", "impl2", "b2");
+ setBucketOption(config, "b", "bucket.b.loop", "b3");
+ assertOptionEquals(config, "fs.s3a.bucket.b.impl", "b");
+
+ Configuration updated = propagateBucketOptions(config, "b");
+ assertOptionEquals(updated, impl, "orig");
+ assertOptionEquals(updated, "fs.s3a.impl2", "b2");
+ assertOptionEquals(updated, metastoreImpl, ddb);
+ assertOptionEquals(updated, "fs.s3a.bucket.b.loop", null);
+ }
+
+ @Test
+ public void testConfOptionPropagationToFS() throws Exception {
+ Configuration config = new Configuration();
+ String testFSName = config.getTrimmed(TEST_FS_S3A_NAME, "");
+ String bucket = new URI(testFSName).getHost();
+ setBucketOption(config, bucket, "propagation", "propagated");
+ fs = S3ATestUtils.createTestFileSystem(config);
+ Configuration updated = fs.getConf();
+ assertOptionEquals(updated, "fs.s3a.propagation", "propagated");
+ }
+
+ @Test
+ public void testSecurityCredentialPropagationNoOverride() throws Exception {
+ Configuration config = new Configuration();
+ config.set(CREDENTIAL_PROVIDER_PATH, "base");
+ patchSecurityCredentialProviders(config);
+ assertOptionEquals(config, CREDENTIAL_PROVIDER_PATH,
+ "base");
+ }
+
+ @Test
+ public void testSecurityCredentialPropagationOverrideNoBase()
+ throws Exception {
+ Configuration config = new Configuration();
+ config.unset(CREDENTIAL_PROVIDER_PATH);
+ config.set(S3A_SECURITY_CREDENTIAL_PROVIDER_PATH, "override");
+ patchSecurityCredentialProviders(config);
+ assertOptionEquals(config, CREDENTIAL_PROVIDER_PATH,
+ "override");
+ }
+
+ @Test
+ public void testSecurityCredentialPropagationOverride() throws Exception {
+ Configuration config = new Configuration();
+ config.set(CREDENTIAL_PROVIDER_PATH, "base");
+ config.set(S3A_SECURITY_CREDENTIAL_PROVIDER_PATH, "override");
+ patchSecurityCredentialProviders(config);
+ assertOptionEquals(config, CREDENTIAL_PROVIDER_PATH,
+ "override,base");
+ Collection all = config.getStringCollection(
+ CREDENTIAL_PROVIDER_PATH);
+ assertTrue(all.contains("override"));
+ assertTrue(all.contains("base"));
+ }
+
+ @Test
+ public void testSecurityCredentialPropagationEndToEnd() throws Exception {
+ Configuration config = new Configuration();
+ config.set(CREDENTIAL_PROVIDER_PATH, "base");
+ setBucketOption(config, "b", S3A_SECURITY_CREDENTIAL_PROVIDER_PATH,
+ "override");
+ Configuration updated = propagateBucketOptions(config, "b");
+
+ patchSecurityCredentialProviders(updated);
+ assertOptionEquals(updated, CREDENTIAL_PROVIDER_PATH,
+ "override,base");
+ }
+
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
index 8c22f47..acbe610 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
@@ -86,18 +86,6 @@
String DEFAULT_CSVTEST_FILE = "s3a://landsat-pds/scene_list.gz";
/**
- * Endpoint for the S3 CSV/scale tests. This defaults to
- * being us-east.
- */
- String KEY_CSVTEST_ENDPOINT = S3A_SCALE_TEST + "csvfile.endpoint";
-
- /**
- * Endpoint for the S3 CSV/scale tests. This defaults to
- * being us-east.
- */
- String DEFAULT_CSVTEST_ENDPOINT = "s3.amazonaws.com";
-
- /**
* Name of the property to define the timeout for scale tests: {@value}.
* Measured in seconds.
*/
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
index 462914c..9528967 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@ -20,12 +20,14 @@
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.scale.S3AScaleTestBase;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.internal.AssumptionViolatedException;
import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URI;
@@ -34,11 +36,14 @@
import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
import static org.apache.hadoop.fs.s3a.S3ATestConstants.*;
import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.junit.Assert.*;
/**
* Utilities for the S3A tests.
*/
public final class S3ATestUtils {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ S3ATestUtils.class);
/**
* Value to set a system property to (in maven) to declare that
@@ -137,20 +142,6 @@ public static FileContext createTestFileContext(Configuration conf)
}
/**
- * patch the endpoint option so that irrespective of where other tests
- * are working, the IO performance tests can work with the landsat
- * images.
- * @param conf configuration to patch
- */
- public static void useCSVDataEndpoint(Configuration conf) {
- String endpoint = conf.getTrimmed(S3AScaleTestBase.KEY_CSVTEST_ENDPOINT,
- S3AScaleTestBase.DEFAULT_CSVTEST_ENDPOINT);
- if (!endpoint.isEmpty()) {
- conf.set(ENDPOINT, endpoint);
- }
- }
-
- /**
* Get a long test property.
*
*
Look up configuration value (which can pick up core-default.xml),
@@ -511,4 +502,59 @@ public static void assertInstanceOf(Class> expectedClass, Object obj) {
*/
private S3ATestUtils() {
}
+
+ /**
+ * Set a bucket specific property to a particular value.
+ * If the generic key passed in has an {@code fs.s3a. prefix},
+ * that's stripped off, so that when the the bucket properties are propagated
+ * down to the generic values, that value gets copied down.
+ * @param conf configuration to set
+ * @param bucket bucket name
+ * @param genericKey key; can start with "fs.s3a."
+ * @param value value to set
+ */
+ public static void setBucketOption(Configuration conf, String bucket,
+ String genericKey, String value) {
+ final String baseKey = genericKey.startsWith(FS_S3A_PREFIX) ?
+ genericKey.substring(FS_S3A_PREFIX.length())
+ : genericKey;
+ conf.set(FS_S3A_BUCKET_PREFIX + bucket + '.' + baseKey, value);
+ }
+
+ /**
+ * Assert that a configuration option matches the expected value.
+ * @param conf configuration
+ * @param key option key
+ * @param expected expected value
+ */
+ public static void assertOptionEquals(Configuration conf,
+ String key,
+ String expected) {
+ assertEquals("Value of " + key, expected, conf.get(key));
+ }
+
+ /**
+ * Assume that a condition is met. If not: log at WARN and
+ * then throw an {@link AssumptionViolatedException}.
+ * @param message
+ * @param condition
+ */
+ public static void assume(String message, boolean condition) {
+ if (!condition) {
+ LOG.warn(message);
+ }
+ Assume.assumeTrue(message, condition);
+ }
+
+ /**
+ * Get the statistics from a wrapped block output stream.
+ * @param out output stream
+ * @return the (active) stats of the write
+ */
+ public static S3AInstrumentation.OutputStreamStatistics
+ getOutputStreamStatistics(FSDataOutputStream out) {
+ S3ABlockOutputStream blockOutputStream
+ = (S3ABlockOutputStream) out.getWrappedStream();
+ return blockOutputStream.getStatistics();
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestDataBlocks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestDataBlocks.java
index 9fa95fd..700ef5c 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestDataBlocks.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestDataBlocks.java
@@ -51,9 +51,8 @@ public void testByteBufferIO() throws Throwable {
new S3ADataBlocks.ByteBufferBlockFactory(null)) {
int limit = 128;
S3ADataBlocks.ByteBufferBlockFactory.ByteBufferBlock block
- = factory.create(limit);
- assertEquals("outstanding buffers in " + factory,
- 1, factory.getOutstandingBufferCount());
+ = factory.create(1, limit, null);
+ assertOutstandingBuffers(factory, 1);
byte[] buffer = ContractTestUtils.toAsciiByteArray("test data");
int bufferLen = buffer.length;
@@ -66,24 +65,23 @@ public void testByteBufferIO() throws Throwable {
block.hasCapacity(limit - bufferLen));
// now start the write
- S3ADataBlocks.ByteBufferBlockFactory.ByteBufferInputStream
- stream = block.startUpload();
+ S3ADataBlocks.BlockUploadData blockUploadData = block.startUpload();
+ S3ADataBlocks.ByteBufferBlockFactory.ByteBufferBlock.ByteBufferInputStream
+ stream =
+ (S3ADataBlocks.ByteBufferBlockFactory.ByteBufferBlock.ByteBufferInputStream)
+ blockUploadData.getUploadStream();
+ assertTrue("Mark not supported in " + stream, stream.markSupported());
assertTrue("!hasRemaining() in " + stream, stream.hasRemaining());
int expected = bufferLen;
assertEquals("wrong available() in " + stream,
expected, stream.available());
assertEquals('t', stream.read());
+ stream.mark(limit);
expected--;
assertEquals("wrong available() in " + stream,
expected, stream.available());
- // close the block. The buffer must remain outstanding here;
- // the stream manages the lifecycle of it now
- block.close();
- assertEquals("outstanding buffers in " + factory,
- 1, factory.getOutstandingBufferCount());
- block.close();
// read into a byte array with an offset
int offset = 5;
@@ -109,16 +107,31 @@ public void testByteBufferIO() throws Throwable {
0, stream.available());
assertTrue("hasRemaining() in " + stream, !stream.hasRemaining());
+ // go the mark point
+ stream.reset();
+ assertEquals('e', stream.read());
+
// when the stream is closed, the data should be returned
stream.close();
- assertEquals("outstanding buffers in " + factory,
- 0, factory.getOutstandingBufferCount());
+ assertOutstandingBuffers(factory, 1);
+ block.close();
+ assertOutstandingBuffers(factory, 0);
stream.close();
- assertEquals("outstanding buffers in " + factory,
- 0, factory.getOutstandingBufferCount());
-
+ assertOutstandingBuffers(factory, 0);
}
}
+ /**
+ * Assert the number of buffers active for a block factory.
+ * @param factory factory
+ * @param expectedCount expected count.
+ */
+ private static void assertOutstandingBuffers(
+ S3ADataBlocks.ByteBufferBlockFactory factory,
+ int expectedCount) {
+ assertEquals("outstanding buffers in " + factory,
+ expectedCount, factory.getOutstandingBufferCount());
+ }
+
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java
index fcb6444..89fae82 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java
@@ -34,11 +34,13 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.StorageStatistics;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.s3a.S3AFileStatus;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.S3AInstrumentation;
import org.apache.hadoop.fs.s3a.Statistic;
import org.apache.hadoop.util.Progressable;
@@ -159,13 +161,20 @@ public void test_010_CreateHugeFile() throws IOException {
Statistic putBytesPending = Statistic.OBJECT_PUT_BYTES_PENDING;
ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
-
+ S3AInstrumentation.OutputStreamStatistics streamStatistics;
long blocksPer10MB = blocksPerMB * 10;
ProgressCallback progress = new ProgressCallback(timer);
try (FSDataOutputStream out = fs.create(hugefile,
true,
uploadBlockSize,
progress)) {
+ try {
+ streamStatistics = getOutputStreamStatistics(out);
+ } catch (ClassCastException e) {
+ LOG.info("Wrapped output stream is not block stream: {}",
+ out.getWrappedStream());
+ streamStatistics = null;
+ }
for (long block = 1; block <= blocks; block++) {
out.write(data);
@@ -190,7 +199,8 @@ public void test_010_CreateHugeFile() throws IOException {
}
}
// now close the file
- LOG.info("Closing file and completing write operation");
+ LOG.info("Closing stream {}", out);
+ LOG.info("Statistics : {}", streamStatistics);
ContractTestUtils.NanoTimer closeTimer
= new ContractTestUtils.NanoTimer();
out.close();
@@ -201,6 +211,7 @@ public void test_010_CreateHugeFile() throws IOException {
filesizeMB, uploadBlockSize);
logFSState();
bandwidth(timer, filesize);
+ LOG.info("Statistics after stream closed: {}", streamStatistics);
long putRequestCount = storageStatistics.getLong(putRequests);
Long putByteCount = storageStatistics.getLong(putBytes);
LOG.info("PUT {} bytes in {} operations; {} MB/operation",
@@ -214,7 +225,14 @@ public void test_010_CreateHugeFile() throws IOException {
S3AFileStatus status = fs.getFileStatus(hugefile);
ContractTestUtils.assertIsFile(hugefile, status);
assertEquals("File size in " + status, filesize, status.getLen());
- progress.verifyNoFailures("Put file " + hugefile + " of size " + filesize);
+ if (progress != null) {
+ progress.verifyNoFailures("Put file " + hugefile
+ + " of size " + filesize);
+ }
+ if (streamStatistics != null) {
+ assertEquals("actively allocated blocks in " + streamStatistics,
+ 0, streamStatistics.blocksActivelyAllocated());
+ }
}
/**
@@ -285,7 +303,9 @@ private void verifyNoFailures(String operation) {
void assumeHugeFileExists() throws IOException {
S3AFileSystem fs = getFileSystem();
ContractTestUtils.assertPathExists(fs, "huge file not created", hugefile);
- ContractTestUtils.assertIsFile(fs, hugefile);
+ FileStatus status = fs.getFileStatus(hugefile);
+ ContractTestUtils.assertIsFile(hugefile, status);
+ assertTrue("File " + hugefile + " is empty", status.getLen() > 0);
}
private void logFSState() {
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AConcurrentOps.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AConcurrentOps.java
new file mode 100644
index 0000000..b4d3862
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AConcurrentOps.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.scale;
+
+import java.io.IOException;
+
+import java.net.URI;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.contract.ContractTestUtils.NanoTimer;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+
+import org.junit.After;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.fs.s3a.Constants.*;
+
+/**
+ * Tests concurrent operations on a single S3AFileSystem instance.
+ */
+public class ITestS3AConcurrentOps extends S3AScaleTestBase {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ ITestS3AConcurrentOps.class);
+ private final int concurrentRenames = 10;
+ private Path testRoot;
+ private Path[] source = new Path[concurrentRenames];
+ private Path[] target = new Path[concurrentRenames];
+ private S3AFileSystem fs;
+ private S3AFileSystem auxFs;
+
+ @Override
+ protected int getTestTimeoutSeconds() {
+ return 16 * 60;
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ fs = getRestrictedFileSystem();
+ auxFs = getNormalFileSystem();
+
+ testRoot = path("/ITestS3AConcurrentOps");
+ testRoot = S3ATestUtils.createTestPath(testRoot);
+
+ for (int i = 0; i < concurrentRenames; i++){
+ source[i] = new Path(testRoot, "source" + i);
+ target[i] = new Path(testRoot, "target" + i);
+ }
+
+ LOG.info("Generating data...");
+ auxFs.mkdirs(testRoot);
+ byte[] zeroes = ContractTestUtils.dataset(1024*1024, 0, Integer.MAX_VALUE);
+ for (Path aSource : source) {
+ try(FSDataOutputStream out = auxFs.create(aSource)) {
+ for (int mb = 0; mb < 20; mb++) {
+ LOG.debug("{}: Block {}...", aSource, mb);
+ out.write(zeroes);
+ }
+ }
+ }
+ LOG.info("Data generated...");
+ }
+
+ private S3AFileSystem getRestrictedFileSystem() throws Exception {
+ Configuration conf = getConfiguration();
+ conf.setInt(MAX_THREADS, 2);
+ conf.setInt(MAX_TOTAL_TASKS, 1);
+
+ conf.set(MIN_MULTIPART_THRESHOLD, "10M");
+ conf.set(MULTIPART_SIZE, "5M");
+
+ S3AFileSystem s3a = getFileSystem();
+ URI rootURI = new URI(conf.get(TEST_FS_S3A_NAME));
+ s3a.initialize(rootURI, conf);
+ return s3a;
+ }
+
+ private S3AFileSystem getNormalFileSystem() throws Exception {
+ S3AFileSystem s3a = new S3AFileSystem();
+ Configuration conf = new Configuration();
+ URI rootURI = new URI(conf.get(TEST_FS_S3A_NAME));
+ s3a.initialize(rootURI, conf);
+ return s3a;
+ }
+
+ @After
+ public void teardown() throws Exception {
+ if (auxFs != null) {
+ auxFs.delete(testRoot, true);
+ }
+ }
+
+ /**
+ * Attempts to trigger a deadlock that would happen if any bounded resource
+ * pool became saturated with control tasks that depended on other tasks
+ * that now can't enter the resource pool to get completed.
+ */
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testParallelRename() throws InterruptedException,
+ ExecutionException, IOException {
+ ExecutorService executor = Executors.newFixedThreadPool(
+ concurrentRenames, new ThreadFactory() {
+ private AtomicInteger count = new AtomicInteger(0);
+
+ public Thread newThread(Runnable r) {
+ return new Thread(r,
+ "testParallelRename" + count.getAndIncrement());
+ }
+ });
+ ((ThreadPoolExecutor)executor).prestartAllCoreThreads();
+ Future[] futures = new Future[concurrentRenames];
+ for (int i = 0; i < concurrentRenames; i++) {
+ final int index = i;
+ futures[i] = executor.submit(new Callable() {
+ @Override
+ public Boolean call() throws Exception {
+ NanoTimer timer = new NanoTimer();
+ boolean result = fs.rename(source[index], target[index]);
+ timer.end("parallel rename %d", index);
+ LOG.info("Rename {} ran from {} to {}", index,
+ timer.getStartTime(), timer.getEndTime());
+ return result;
+ }
+ });
+ }
+ LOG.info("Waiting for tasks to complete...");
+ LOG.info("Deadlock may have occurred if nothing else is logged" +
+ " or the test times out");
+ for (int i = 0; i < concurrentRenames; i++) {
+ assertTrue("No future " + i, futures[i].get());
+ assertPathExists("target path", target[i]);
+ assertPathDoesNotExist("source path", source[i]);
+ }
+ LOG.info("All tasks have completed successfully");
+ }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
index e36d086..236ffcd 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
@@ -28,7 +28,6 @@
import org.apache.hadoop.fs.s3a.S3AInputPolicy;
import org.apache.hadoop.fs.s3a.S3AInputStream;
import org.apache.hadoop.fs.s3a.S3AInstrumentation;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.CompressionCodec;
@@ -36,7 +35,6 @@
import org.apache.hadoop.util.LineReader;
import org.junit.After;
import org.junit.Assert;
-import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
@@ -47,6 +45,7 @@
import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume;
/**
* Look at the performance of S3a operations.
@@ -79,10 +78,11 @@ public void openFS() throws IOException {
String testFile = conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE);
if (testFile.isEmpty()) {
assumptionMessage = "Empty test property: " + KEY_CSVTEST_FILE;
+ LOG.warn(assumptionMessage);
testDataAvailable = false;
} else {
- S3ATestUtils.useCSVDataEndpoint(conf);
testData = new Path(testFile);
+ LOG.info("Using {} as input stream source", testData);
Path path = this.testData;
bindS3aFS(path);
try {
@@ -113,7 +113,7 @@ public void cleanup() {
* Declare that the test requires the CSV test dataset.
*/
private void requireCSVTestData() {
- Assume.assumeTrue(assumptionMessage, testDataAvailable);
+ assume(assumptionMessage, testDataAvailable);
}
/**
@@ -146,7 +146,7 @@ FSDataInputStream openTestFile(S3AInputPolicy inputPolicy, long readahead)
/**
* Open a test file with the read buffer specified in the setting
- * {@link #KEY_READ_BUFFER_SIZE}.
+ * {@link org.apache.hadoop.fs.s3a.S3ATestConstants#KEY_READ_BUFFER_SIZE}.
*
* @param path path to open
* @param inputPolicy input policy to use
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java
index 9da621f..0f844b1 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java
@@ -28,7 +28,6 @@
import org.apache.hadoop.fs.s3a.Statistic;
import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
-import org.junit.Assume;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -91,14 +90,13 @@ public void setup() throws Exception {
super.setup();
testPath = path("/tests3ascale");
LOG.debug("Scale test operation count = {}", getOperationCount());
- // multipart purges are disabled on the scale tests
- // check for the test being enabled
enabled = getTestPropertyBool(
getConf(),
KEY_SCALE_TESTS_ENABLED,
DEFAULT_SCALE_TESTS_ENABLED);
- Assume.assumeTrue("Scale test disabled: to enable set property " +
- KEY_SCALE_TESTS_ENABLED, isEnabled());
+ assume("Scale test disabled: to enable set property " +
+ KEY_SCALE_TESTS_ENABLED,
+ isEnabled());
}
/**
diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml
new file mode 100644
index 0000000..a8c3b16
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/pom.xml
@@ -0,0 +1,160 @@
+
+
+
+ 4.0.0
+
+ org.apache.hadoop
+ hadoop-project
+ 2.8.0
+ ../../hadoop-project
+
+ org.apache.hadoop
+ hadoop-azure-datalake
+ Apache Hadoop Azure Data Lake support
+
+ This module contains code to support integration with Azure Data Lake.
+
+ jar
+
+ 2.4.0
+ 0.9.1
+ UTF-8
+ true
+
+
+
+
+ org.apache.maven.plugins
+ maven-project-info-reports-plugin
+
+
+ false
+ false
+
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+
+ test-jar
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+
+ deplist
+ compile
+
+ list
+
+
+
+ ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt
+
+
+
+
+
+
+
+
+
+
+
+ org.eclipse.m2e
+ lifecycle-mapping
+ 1.0.0
+
+
+
+
+
+ org.apache.maven.plugins
+
+ maven-enforcer-plugin
+
+ [1.0.0,)
+
+ enforce
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ com.microsoft.azure
+ azure-data-lake-store-sdk
+ 2.1.4
+
+
+
+ org.apache.hadoop
+ hadoop-common
+
+
+ com.squareup.okhttp
+ okhttp
+ 2.4.0
+
+
+ junit
+ junit
+ test
+
+
+ com.eclipsesource.minimal-json
+ minimal-json
+ 0.9.1
+ test
+
+
+ org.apache.hadoop
+ hadoop-common
+ test
+ test-jar
+
+
+ com.squareup.okhttp
+ mockwebserver
+ 2.4.0
+ test
+
+
+
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/Adl.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/Adl.java
new file mode 100644
index 0000000..7ec04cf
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/Adl.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DelegateToFileSystem;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * Expose adl:// scheme to access ADL file system.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class Adl extends DelegateToFileSystem {
+
+ Adl(URI theUri, Configuration conf) throws IOException, URISyntaxException {
+ super(theUri, createDataLakeFileSystem(conf), conf, AdlFileSystem.SCHEME,
+ false);
+ }
+
+ private static AdlFileSystem createDataLakeFileSystem(Configuration conf) {
+ AdlFileSystem fs = new AdlFileSystem();
+ fs.setConf(conf);
+ return fs;
+ }
+
+ /**
+ * @return Default port for ADL File system to communicate
+ */
+ @Override
+ public final int getUriDefaultPort() {
+ return AdlFileSystem.DEFAULT_PORT;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
new file mode 100644
index 0000000..8fc8e00
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.adl;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Constants.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class AdlConfKeys {
+ // OAuth2 Common Configuration
+ public static final String AZURE_AD_REFRESH_URL_KEY = "dfs.adls.oauth2"
+ + ".refresh.url";
+
+ // optional when provider type is refresh or client id.
+ public static final String AZURE_AD_TOKEN_PROVIDER_CLASS_KEY =
+ "dfs.adls.oauth2.access.token.provider";
+ public static final String AZURE_AD_CLIENT_ID_KEY =
+ "dfs.adls.oauth2.client.id";
+ public static final String AZURE_AD_TOKEN_PROVIDER_TYPE_KEY =
+ "dfs.adls.oauth2.access.token.provider.type";
+
+ // OAuth Refresh Token Configuration
+ public static final String AZURE_AD_REFRESH_TOKEN_KEY =
+ "dfs.adls.oauth2.refresh.token";
+
+ public static final String TOKEN_PROVIDER_TYPE_REFRESH_TOKEN = "RefreshToken";
+ // OAuth Client Cred Token Configuration
+ public static final String AZURE_AD_CLIENT_SECRET_KEY =
+ "dfs.adls.oauth2.credential";
+ public static final String TOKEN_PROVIDER_TYPE_CLIENT_CRED =
+ "ClientCredential";
+
+ public static final String READ_AHEAD_BUFFER_SIZE_KEY =
+ "adl.feature.client.cache.readahead";
+
+ public static final String WRITE_BUFFER_SIZE_KEY =
+ "adl.feature.client.cache.drop.behind.writes";
+ static final String SECURE_TRANSPORT_SCHEME = "https";
+ static final String INSECURE_TRANSPORT_SCHEME = "http";
+ static final String ADL_DEBUG_OVERRIDE_LOCAL_USER_AS_OWNER =
+ "adl.debug.override.localuserasfileowner";
+
+ static final boolean ADL_DEBUG_SET_LOCAL_USER_AS_OWNER_DEFAULT = false;
+ static final long ADL_BLOCK_SIZE = 256 * 1024 * 1024;
+ static final int ADL_REPLICATION_FACTOR = 1;
+ static final String ADL_HADOOP_CLIENT_NAME = "hadoop-azure-datalake-";
+ static final String ADL_HADOOP_CLIENT_VERSION =
+ "2.0.0-SNAPSHOT";
+ static final String ADL_EVENTS_TRACKING_CLUSTERNAME =
+ "adl.events.tracking.clustername";
+
+ static final String ADL_EVENTS_TRACKING_CLUSTERTYPE =
+ "adl.events.tracking.clustertype";
+ static final int DEFAULT_READ_AHEAD_BUFFER_SIZE = 4 * 1024 * 1024;
+ static final int DEFAULT_WRITE_AHEAD_BUFFER_SIZE = 4 * 1024 * 1024;
+
+ static final String LATENCY_TRACKER_KEY =
+ "adl.dfs.enable.client.latency.tracker";
+ static final boolean LATENCY_TRACKER_DEFAULT = true;
+
+ static final String ADL_EXPERIMENT_POSITIONAL_READ_KEY =
+ "adl.feature.experiment.positional.read.enable";
+ static final boolean ADL_EXPERIMENT_POSITIONAL_READ_DEFAULT = true;
+
+ static final String ADL_SUPPORT_ACL_BIT_IN_FSPERMISSION =
+ "adl.feature.support.acl.bit";
+ static final boolean ADL_SUPPORT_ACL_BIT_IN_FSPERMISSION_DEFAULT = true;
+
+ static final String ADL_ENABLEUPN_FOR_OWNERGROUP_KEY =
+ "adl.feature.ownerandgroup.enableupn";
+ static final boolean ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT = false;
+
+ private AdlConfKeys() {
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
new file mode 100644
index 0000000..e0e273e
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
@@ -0,0 +1,981 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.microsoft.azure.datalake.store.ADLStoreClient;
+import com.microsoft.azure.datalake.store.ADLStoreOptions;
+import com.microsoft.azure.datalake.store.DirectoryEntry;
+import com.microsoft.azure.datalake.store.DirectoryEntryType;
+import com.microsoft.azure.datalake.store.IfExists;
+import com.microsoft.azure.datalake.store.LatencyTracker;
+import com.microsoft.azure.datalake.store.UserGroupRepresentation;
+import com.microsoft.azure.datalake.store.oauth2.AccessTokenProvider;
+import com.microsoft.azure.datalake.store.oauth2.ClientCredsTokenProvider;
+import com.microsoft.azure.datalake.store.oauth2.RefreshTokenBasedTokenProvider;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.ContentSummary.Builder;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.InvalidPathException;
+import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.Options.Rename;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.adl.oauth2.AzureADTokenProvider;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.ProviderUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.VersionInfo;
+
+import static org.apache.hadoop.fs.adl.AdlConfKeys.*;
+
+/**
+ * A FileSystem to access Azure Data Lake Store.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class AdlFileSystem extends FileSystem {
+ public static final String SCHEME = "adl";
+ static final int DEFAULT_PORT = 443;
+ private URI uri;
+ private String userName;
+ private boolean overrideOwner;
+ private ADLStoreClient adlClient;
+ private Path workingDirectory;
+ private boolean aclBitStatus;
+ private UserGroupRepresentation oidOrUpn;
+
+
+ // retained for tests
+ private AccessTokenProvider tokenProvider;
+ private AzureADTokenProvider azureTokenProvider;
+
+ @Override
+ public String getScheme() {
+ return SCHEME;
+ }
+
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public int getDefaultPort() {
+ return DEFAULT_PORT;
+ }
+
+ @Override
+ public boolean supportsSymlinks() {
+ return false;
+ }
+
+ /**
+ * Called after a new FileSystem instance is constructed.
+ *
+ * @param storeUri a uri whose authority section names the host, port, etc.
+ * for this FileSystem
+ * @param conf the configuration
+ */
+ @Override
+ public void initialize(URI storeUri, Configuration conf) throws IOException {
+ super.initialize(storeUri, conf);
+ this.setConf(conf);
+ this.uri = URI
+ .create(storeUri.getScheme() + "://" + storeUri.getAuthority());
+
+ try {
+ userName = UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException e) {
+ userName = "hadoop";
+ }
+
+ this.setWorkingDirectory(getHomeDirectory());
+
+ overrideOwner = getConf().getBoolean(ADL_DEBUG_OVERRIDE_LOCAL_USER_AS_OWNER,
+ ADL_DEBUG_SET_LOCAL_USER_AS_OWNER_DEFAULT);
+
+ aclBitStatus = conf.getBoolean(ADL_SUPPORT_ACL_BIT_IN_FSPERMISSION,
+ ADL_SUPPORT_ACL_BIT_IN_FSPERMISSION_DEFAULT);
+
+ String accountFQDN = null;
+ String mountPoint = null;
+ String hostname = storeUri.getHost();
+ if (!hostname.contains(".") && !hostname.equalsIgnoreCase(
+ "localhost")) { // this is a symbolic name. Resolve it.
+ String hostNameProperty = "dfs.adls." + hostname + ".hostname";
+ String mountPointProperty = "dfs.adls." + hostname + ".mountpoint";
+ accountFQDN = getNonEmptyVal(conf, hostNameProperty);
+ mountPoint = getNonEmptyVal(conf, mountPointProperty);
+ } else {
+ accountFQDN = hostname;
+ }
+
+ if (storeUri.getPort() > 0) {
+ accountFQDN = accountFQDN + ":" + storeUri.getPort();
+ }
+
+ adlClient = ADLStoreClient
+ .createClient(accountFQDN, getAccessTokenProvider(conf));
+
+ ADLStoreOptions options = new ADLStoreOptions();
+ options.enableThrowingRemoteExceptions();
+
+ if (getTransportScheme().equalsIgnoreCase(INSECURE_TRANSPORT_SCHEME)) {
+ options.setInsecureTransport();
+ }
+
+ if (mountPoint != null) {
+ options.setFilePathPrefix(mountPoint);
+ }
+
+ String clusterName = conf.get(ADL_EVENTS_TRACKING_CLUSTERNAME, "UNKNOWN");
+ String clusterType = conf.get(ADL_EVENTS_TRACKING_CLUSTERTYPE, "UNKNOWN");
+
+ String clientVersion = ADL_HADOOP_CLIENT_NAME + (StringUtils
+ .isEmpty(VersionInfo.getVersion().trim()) ?
+ ADL_HADOOP_CLIENT_VERSION.trim() :
+ VersionInfo.getVersion().trim());
+ options.setUserAgentSuffix(clientVersion + "/" +
+ VersionInfo.getVersion().trim() + "/" + clusterName + "/"
+ + clusterType);
+
+ adlClient.setOptions(options);
+
+ boolean trackLatency = conf
+ .getBoolean(LATENCY_TRACKER_KEY, LATENCY_TRACKER_DEFAULT);
+ if (!trackLatency) {
+ LatencyTracker.disable();
+ }
+
+ boolean enableUPN = conf.getBoolean(ADL_ENABLEUPN_FOR_OWNERGROUP_KEY,
+ ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT);
+ oidOrUpn = enableUPN ? UserGroupRepresentation.UPN :
+ UserGroupRepresentation.OID;
+ }
+
+ /**
+ * This method is provided for convenience for derived classes to define
+ * custom {@link AzureADTokenProvider} instance.
+ *
+ * In order to ensure secure hadoop infrastructure and user context for which
+ * respective {@link AdlFileSystem} instance is initialized,
+ * Loading {@link AzureADTokenProvider} is not sufficient.
+ *
+ * The order of loading {@link AzureADTokenProvider} is to first invoke
+ * {@link #getCustomAccessTokenProvider(Configuration)}, If method return null
+ * which means no implementation provided by derived classes, then
+ * configuration object is loaded to retrieve token configuration as specified
+ * is documentation.
+ *
+ * Custom token management takes the higher precedence during initialization.
+ *
+ * @param conf Configuration object
+ * @return null if the no custom {@link AzureADTokenProvider} token management
+ * is specified.
+ * @throws IOException if failed to initialize token provider.
+ */
+ protected synchronized AzureADTokenProvider getCustomAccessTokenProvider(
+ Configuration conf) throws IOException {
+ String className = getNonEmptyVal(conf, AZURE_AD_TOKEN_PROVIDER_CLASS_KEY);
+
+ Class extends AzureADTokenProvider> azureADTokenProviderClass =
+ conf.getClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, null,
+ AzureADTokenProvider.class);
+ if (azureADTokenProviderClass == null) {
+ throw new IllegalArgumentException(
+ "Configuration " + className + " " + "not defined/accessible.");
+ }
+
+ azureTokenProvider = ReflectionUtils
+ .newInstance(azureADTokenProviderClass, conf);
+ if (azureTokenProvider == null) {
+ throw new IllegalArgumentException("Failed to initialize " + className);
+ }
+
+ azureTokenProvider.initialize(conf);
+ return azureTokenProvider;
+ }
+
+ private AccessTokenProvider getAccessTokenProvider(Configuration config)
+ throws IOException {
+ Configuration conf = ProviderUtils.excludeIncompatibleCredentialProviders(
+ config, AdlFileSystem.class);
+ TokenProviderType type = conf.getEnum(
+ AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
+
+ switch (type) {
+ case RefreshToken:
+ tokenProvider = getConfRefreshTokenBasedTokenProvider(conf);
+ break;
+ case ClientCredential:
+ tokenProvider = getConfCredentialBasedTokenProvider(conf);
+ break;
+ case Custom:
+ default:
+ AzureADTokenProvider azureADTokenProvider = getCustomAccessTokenProvider(
+ conf);
+ tokenProvider = new SdkTokenProviderAdapter(azureADTokenProvider);
+ break;
+ }
+
+ return tokenProvider;
+ }
+
+ private AccessTokenProvider getConfCredentialBasedTokenProvider(
+ Configuration conf) throws IOException {
+ String clientId = getPasswordString(conf, AZURE_AD_CLIENT_ID_KEY);
+ String refreshUrl = getPasswordString(conf, AZURE_AD_REFRESH_URL_KEY);
+ String clientSecret = getPasswordString(conf, AZURE_AD_CLIENT_SECRET_KEY);
+ return new ClientCredsTokenProvider(refreshUrl, clientId, clientSecret);
+ }
+
+ private AccessTokenProvider getConfRefreshTokenBasedTokenProvider(
+ Configuration conf) throws IOException {
+ String clientId = getPasswordString(conf, AZURE_AD_CLIENT_ID_KEY);
+ String refreshToken = getPasswordString(conf, AZURE_AD_REFRESH_TOKEN_KEY);
+ return new RefreshTokenBasedTokenProvider(clientId, refreshToken);
+ }
+
+ @VisibleForTesting
+ AccessTokenProvider getTokenProvider() {
+ return tokenProvider;
+ }
+
+ @VisibleForTesting
+ AzureADTokenProvider getAzureTokenProvider() {
+ return azureTokenProvider;
+ }
+
+ /**
+ * Constructing home directory locally is fine as long as Hadoop
+ * local user name and ADL user name relationship story is not fully baked
+ * yet.
+ *
+ * @return Hadoop local user home directory.
+ */
+ @Override
+ public Path getHomeDirectory() {
+ return makeQualified(new Path("/user/" + userName));
+ }
+
+ /**
+ * Create call semantic is handled differently in case of ADL. Create
+ * semantics is translated to Create/Append
+ * semantics.
+ * 1. No dedicated connection to server.
+ * 2. Buffering is locally done, Once buffer is full or flush is invoked on
+ * the by the caller. All the pending
+ * data is pushed to ADL as APPEND operation code.
+ * 3. On close - Additional call is send to server to close the stream, and
+ * release lock from the stream.
+ *
+ * Necessity of Create/Append semantics is
+ * 1. ADL backend server does not allow idle connection for longer duration
+ * . In case of slow writer scenario,
+ * observed connection timeout/Connection reset causing occasional job
+ * failures.
+ * 2. Performance boost to jobs which are slow writer, avoided network latency
+ * 3. ADL equally better performing with multiple of 4MB chunk as append
+ * calls.
+ *
+ * @param f File path
+ * @param permission Access permission for the newly created file
+ * @param overwrite Remove existing file and recreate new one if true
+ * otherwise throw error if file exist
+ * @param bufferSize Buffer size, ADL backend does not honour
+ * @param replication Replication count, ADL backend does not honour
+ * @param blockSize Block size, ADL backend does not honour
+ * @param progress Progress indicator
+ * @return FSDataOutputStream OutputStream on which application can push
+ * stream of bytes
+ * @throws IOException when system error, internal server error or user error
+ */
+ @Override
+ public FSDataOutputStream create(Path f, FsPermission permission,
+ boolean overwrite, int bufferSize, short replication, long blockSize,
+ Progressable progress) throws IOException {
+ statistics.incrementWriteOps(1);
+ IfExists overwriteRule = overwrite ? IfExists.OVERWRITE : IfExists.FAIL;
+ return new FSDataOutputStream(new AdlFsOutputStream(adlClient
+ .createFile(toRelativeFilePath(f), overwriteRule,
+ Integer.toOctalString(applyUMask(permission).toShort()), true),
+ getConf()), this.statistics);
+ }
+
+ /**
+ * Opens an FSDataOutputStream at the indicated Path with write-progress
+ * reporting. Same as create(), except fails if parent directory doesn't
+ * already exist.
+ *
+ * @param f the file name to open
+ * @param permission Access permission for the newly created file
+ * @param flags {@link CreateFlag}s to use for this stream.
+ * @param bufferSize the size of the buffer to be used. ADL backend does
+ * not honour
+ * @param replication required block replication for the file. ADL backend
+ * does not honour
+ * @param blockSize Block size, ADL backend does not honour
+ * @param progress Progress indicator
+ * @throws IOException when system error, internal server error or user error
+ * @see #setPermission(Path, FsPermission)
+ * @deprecated API only for 0.20-append
+ */
+ @Override
+ public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
+ EnumSet flags, int bufferSize, short replication,
+ long blockSize, Progressable progress) throws IOException {
+ statistics.incrementWriteOps(1);
+ IfExists overwriteRule = IfExists.FAIL;
+ for (CreateFlag flag : flags) {
+ if (flag == CreateFlag.OVERWRITE) {
+ overwriteRule = IfExists.OVERWRITE;
+ break;
+ }
+ }
+
+ return new FSDataOutputStream(new AdlFsOutputStream(adlClient
+ .createFile(toRelativeFilePath(f), overwriteRule,
+ Integer.toOctalString(applyUMask(permission).toShort()), false),
+ getConf()), this.statistics);
+ }
+
+ /**
+ * Append to an existing file (optional operation).
+ *
+ * @param f the existing file to be appended.
+ * @param bufferSize the size of the buffer to be used. ADL backend does
+ * not honour
+ * @param progress Progress indicator
+ * @throws IOException when system error, internal server error or user error
+ */
+ @Override
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ statistics.incrementWriteOps(1);
+ return new FSDataOutputStream(
+ new AdlFsOutputStream(adlClient.getAppendStream(toRelativeFilePath(f)),
+ getConf()), this.statistics);
+ }
+
+ /**
+ * Azure data lake does not support user configuration for data replication
+ * hence not leaving system to query on
+ * azure data lake.
+ *
+ * Stub implementation
+ *
+ * @param p Not honoured
+ * @param replication Not honoured
+ * @return True hard coded since ADL file system does not support
+ * replication configuration
+ * @throws IOException No exception would not thrown in this case however
+ * aligning with parent api definition.
+ */
+ @Override
+ public boolean setReplication(final Path p, final short replication)
+ throws IOException {
+ statistics.incrementWriteOps(1);
+ return true;
+ }
+
+ /**
+ * Open call semantic is handled differently in case of ADL. Instead of
+ * network stream is returned to the user,
+ * Overridden FsInputStream is returned.
+ *
+ * @param f File path
+ * @param buffersize Buffer size, Not honoured
+ * @return FSDataInputStream InputStream on which application can read
+ * stream of bytes
+ * @throws IOException when system error, internal server error or user error
+ */
+ @Override
+ public FSDataInputStream open(final Path f, final int buffersize)
+ throws IOException {
+ statistics.incrementReadOps(1);
+ return new FSDataInputStream(
+ new AdlFsInputStream(adlClient.getReadStream(toRelativeFilePath(f)),
+ statistics, getConf()));
+ }
+
+ /**
+ * Return a file status object that represents the path.
+ *
+ * @param f The path we want information from
+ * @return a FileStatus object
+ * @throws IOException when the path does not exist or any other error;
+ * IOException see specific implementation
+ */
+ @Override
+ public FileStatus getFileStatus(final Path f) throws IOException {
+ statistics.incrementReadOps(1);
+ DirectoryEntry entry =
+ adlClient.getDirectoryEntry(toRelativeFilePath(f), oidOrUpn);
+ return toFileStatus(entry, f);
+ }
+
+ /**
+ * List the statuses of the files/directories in the given path if the path is
+ * a directory.
+ *
+ * @param f given path
+ * @return the statuses of the files/directories in the given patch
+ * @throws IOException when the path does not exist or any other error;
+ * IOException see specific implementation
+ */
+ @Override
+ public FileStatus[] listStatus(final Path f) throws IOException {
+ statistics.incrementReadOps(1);
+ List entries =
+ adlClient.enumerateDirectory(toRelativeFilePath(f), oidOrUpn);
+ return toFileStatuses(entries, f);
+ }
+
+ /**
+ * Renames Path src to Path dst. Can take place on local fs
+ * or remote DFS.
+ *
+ * ADLS support POSIX standard for rename operation.
+ *
+ * @param src path to be renamed
+ * @param dst new path after rename
+ * @return true if rename is successful
+ * @throws IOException on failure
+ */
+ @Override
+ public boolean rename(final Path src, final Path dst) throws IOException {
+ statistics.incrementWriteOps(1);
+ if (toRelativeFilePath(src).equals("/")) {
+ return false;
+ }
+
+ return adlClient.rename(toRelativeFilePath(src), toRelativeFilePath(dst));
+ }
+
+ @Override
+ @Deprecated
+ public void rename(final Path src, final Path dst,
+ final Options.Rename... options) throws IOException {
+ statistics.incrementWriteOps(1);
+ boolean overwrite = false;
+ for (Rename renameOption : options) {
+ if (renameOption == Rename.OVERWRITE) {
+ overwrite = true;
+ break;
+ }
+ }
+ adlClient
+ .rename(toRelativeFilePath(src), toRelativeFilePath(dst), overwrite);
+ }
+
+ /**
+ * Concat existing files together.
+ *
+ * @param trg the path to the target destination.
+ * @param srcs the paths to the sources to use for the concatenation.
+ * @throws IOException when system error, internal server error or user error
+ */
+ @Override
+ public void concat(final Path trg, final Path[] srcs) throws IOException {
+ statistics.incrementWriteOps(1);
+ List sourcesList = new ArrayList();
+ for (Path entry : srcs) {
+ sourcesList.add(toRelativeFilePath(entry));
+ }
+ adlClient.concatenateFiles(toRelativeFilePath(trg), sourcesList);
+ }
+
+ /**
+ * Delete a file.
+ *
+ * @param path the path to delete.
+ * @param recursive if path is a directory and set to
+ * true, the directory is deleted else throws an exception.
+ * In case of a file the recursive can be set to either
+ * true or false.
+ * @return true if delete is successful else false.
+ * @throws IOException when system error, internal server error or user error
+ */
+ @Override
+ public boolean delete(final Path path, final boolean recursive)
+ throws IOException {
+ statistics.incrementWriteOps(1);
+ String relativePath = toRelativeFilePath(path);
+ // Delete on root directory not supported.
+ if (relativePath.equals("/")) {
+ // This is important check after recent commit
+ // HADOOP-12977 and HADOOP-13716 validates on root for
+ // 1. if root is empty and non recursive delete then return false.
+ // 2. if root is non empty and non recursive delete then throw exception.
+ if (!recursive
+ && adlClient.enumerateDirectory(toRelativeFilePath(path), 1).size()
+ > 0) {
+ throw new IOException("Delete on root is not supported.");
+ }
+ return false;
+ }
+
+ return recursive ?
+ adlClient.deleteRecursive(relativePath) :
+ adlClient.delete(relativePath);
+ }
+
+ /**
+ * Make the given file and all non-existent parents into
+ * directories. Has the semantics of Unix 'mkdir -p'.
+ * Existence of the directory hierarchy is not an error.
+ *
+ * @param path path to create
+ * @param permission to apply to path
+ */
+ @Override
+ public boolean mkdirs(final Path path, final FsPermission permission)
+ throws IOException {
+ statistics.incrementWriteOps(1);
+ return adlClient.createDirectory(toRelativeFilePath(path),
+ Integer.toOctalString(applyUMask(permission).toShort()));
+ }
+
+ private FileStatus[] toFileStatuses(final List entries,
+ final Path parent) {
+ FileStatus[] fileStatuses = new FileStatus[entries.size()];
+ int index = 0;
+ for (DirectoryEntry entry : entries) {
+ FileStatus status = toFileStatus(entry, parent);
+ if (!(entry.name == null || entry.name == "")) {
+ status.setPath(
+ new Path(parent.makeQualified(uri, workingDirectory), entry.name));
+ }
+
+ fileStatuses[index++] = status;
+ }
+
+ return fileStatuses;
+ }
+
+ private FsPermission applyUMask(FsPermission permission) {
+ if (permission == null) {
+ permission = FsPermission.getDefault();
+ }
+ return permission.applyUMask(FsPermission.getUMask(getConf()));
+ }
+
+ private FileStatus toFileStatus(final DirectoryEntry entry, final Path f) {
+ boolean isDirectory = entry.type == DirectoryEntryType.DIRECTORY;
+ long lastModificationData = entry.lastModifiedTime.getTime();
+ long lastAccessTime = entry.lastAccessTime.getTime();
+ // set aclBit from ADLS backend response if
+ // ADL_SUPPORT_ACL_BIT_IN_FSPERMISSION is true.
+ final boolean aclBit = aclBitStatus ? entry.aclBit : false;
+
+ FsPermission permission = new AdlPermission(aclBit,
+ Short.valueOf(entry.permission, 8));
+ String user = entry.user;
+ String group = entry.group;
+
+ FileStatus status;
+ if (overrideOwner) {
+ status = new FileStatus(entry.length, isDirectory, ADL_REPLICATION_FACTOR,
+ ADL_BLOCK_SIZE, lastModificationData, lastAccessTime, permission,
+ userName, "hdfs", this.makeQualified(f));
+ } else {
+ status = new FileStatus(entry.length, isDirectory, ADL_REPLICATION_FACTOR,
+ ADL_BLOCK_SIZE, lastModificationData, lastAccessTime, permission,
+ user, group, this.makeQualified(f));
+ }
+
+ return status;
+ }
+
+ /**
+ * Set owner of a path (i.e. a file or a directory).
+ * The parameters owner and group cannot both be null.
+ *
+ * @param path The path
+ * @param owner If it is null, the original username remains unchanged.
+ * @param group If it is null, the original groupname remains unchanged.
+ */
+ @Override
+ public void setOwner(final Path path, final String owner, final String group)
+ throws IOException {
+ statistics.incrementWriteOps(1);
+ adlClient.setOwner(toRelativeFilePath(path), owner, group);
+ }
+
+ /**
+ * Set permission of a path.
+ *
+ * @param path The path
+ * @param permission Access permission
+ */
+ @Override
+ public void setPermission(final Path path, final FsPermission permission)
+ throws IOException {
+ statistics.incrementWriteOps(1);
+ adlClient.setPermission(toRelativeFilePath(path),
+ Integer.toOctalString(permission.toShort()));
+ }
+
+ /**
+ * Modifies ACL entries of files and directories. This method can add new ACL
+ * entries or modify the permissions on existing ACL entries. All existing
+ * ACL entries that are not specified in this call are retained without
+ * changes. (Modifications are merged into the current ACL.)
+ *
+ * @param path Path to modify
+ * @param aclSpec List of AclEntry describing modifications
+ * @throws IOException if an ACL could not be modified
+ */
+ @Override
+ public void modifyAclEntries(final Path path, final List aclSpec)
+ throws IOException {
+ statistics.incrementWriteOps(1);
+ List msAclEntries = new
+ ArrayList();
+ for (AclEntry aclEntry : aclSpec) {
+ msAclEntries.add(com.microsoft.azure.datalake.store.acl.AclEntry
+ .parseAclEntry(aclEntry.toString()));
+ }
+ adlClient.modifyAclEntries(toRelativeFilePath(path), msAclEntries);
+ }
+
+ /**
+ * Removes ACL entries from files and directories. Other ACL entries are
+ * retained.
+ *
+ * @param path Path to modify
+ * @param aclSpec List of AclEntry describing entries to remove
+ * @throws IOException if an ACL could not be modified
+ */
+ @Override
+ public void removeAclEntries(final Path path, final List aclSpec)
+ throws IOException {
+ statistics.incrementWriteOps(1);
+ List msAclEntries = new
+ ArrayList();
+ for (AclEntry aclEntry : aclSpec) {
+ msAclEntries.add(com.microsoft.azure.datalake.store.acl.AclEntry
+ .parseAclEntry(aclEntry.toString(), true));
+ }
+ adlClient.removeAclEntries(toRelativeFilePath(path), msAclEntries);
+ }
+
+ /**
+ * Removes all default ACL entries from files and directories.
+ *
+ * @param path Path to modify
+ * @throws IOException if an ACL could not be modified
+ */
+ @Override
+ public void removeDefaultAcl(final Path path) throws IOException {
+ statistics.incrementWriteOps(1);
+ adlClient.removeDefaultAcls(toRelativeFilePath(path));
+ }
+
+ /**
+ * Removes all but the base ACL entries of files and directories. The entries
+ * for user, group, and others are retained for compatibility with permission
+ * bits.
+ *
+ * @param path Path to modify
+ * @throws IOException if an ACL could not be removed
+ */
+ @Override
+ public void removeAcl(final Path path) throws IOException {
+ statistics.incrementWriteOps(1);
+ adlClient.removeAllAcls(toRelativeFilePath(path));
+ }
+
+ /**
+ * Fully replaces ACL of files and directories, discarding all existing
+ * entries.
+ *
+ * @param path Path to modify
+ * @param aclSpec List of AclEntry describing modifications, must include
+ * entries for user, group, and others for compatibility with
+ * permission bits.
+ * @throws IOException if an ACL could not be modified
+ */
+ @Override
+ public void setAcl(final Path path, final List aclSpec)
+ throws IOException {
+ statistics.incrementWriteOps(1);
+ List msAclEntries = new
+ ArrayList();
+ for (AclEntry aclEntry : aclSpec) {
+ msAclEntries.add(com.microsoft.azure.datalake.store.acl.AclEntry
+ .parseAclEntry(aclEntry.toString()));
+ }
+
+ adlClient.setAcl(toRelativeFilePath(path), msAclEntries);
+ }
+
+ /**
+ * Gets the ACL of a file or directory.
+ *
+ * @param path Path to get
+ * @return AclStatus describing the ACL of the file or directory
+ * @throws IOException if an ACL could not be read
+ */
+ @Override
+ public AclStatus getAclStatus(final Path path) throws IOException {
+ statistics.incrementReadOps(1);
+ com.microsoft.azure.datalake.store.acl.AclStatus adlStatus =
+ adlClient.getAclStatus(toRelativeFilePath(path), oidOrUpn);
+ AclStatus.Builder aclStatusBuilder = new AclStatus.Builder();
+ aclStatusBuilder.owner(adlStatus.owner);
+ aclStatusBuilder.group(adlStatus.group);
+ aclStatusBuilder.setPermission(
+ new FsPermission(Short.valueOf(adlStatus.octalPermissions, 8)));
+ aclStatusBuilder.stickyBit(adlStatus.stickyBit);
+ String aclListString = com.microsoft.azure.datalake.store.acl.AclEntry
+ .aclListToString(adlStatus.aclSpec);
+ List aclEntries = AclEntry.parseAclSpec(aclListString, true);
+ aclStatusBuilder.addEntries(aclEntries);
+ return aclStatusBuilder.build();
+ }
+
+ /**
+ * Checks if the user can access a path. The mode specifies which access
+ * checks to perform. If the requested permissions are granted, then the
+ * method returns normally. If access is denied, then the method throws an
+ * {@link AccessControlException}.
+ *
+ * @param path Path to check
+ * @param mode type of access to check
+ * @throws AccessControlException if access is denied
+ * @throws java.io.FileNotFoundException if the path does not exist
+ * @throws IOException see specific implementation
+ */
+ @Override
+ public void access(final Path path, FsAction mode) throws IOException {
+ statistics.incrementReadOps(1);
+ if (!adlClient.checkAccess(toRelativeFilePath(path), mode.SYMBOL)) {
+ throw new AccessControlException("Access Denied : " + path.toString());
+ }
+ }
+
+ /**
+ * Return the {@link ContentSummary} of a given {@link Path}.
+ *
+ * @param f path to use
+ */
+ @Override
+ public ContentSummary getContentSummary(Path f) throws IOException {
+ statistics.incrementReadOps(1);
+ com.microsoft.azure.datalake.store.ContentSummary msSummary = adlClient
+ .getContentSummary(toRelativeFilePath(f));
+ return new Builder().length(msSummary.length)
+ .directoryCount(msSummary.directoryCount).fileCount(msSummary.fileCount)
+ .spaceConsumed(msSummary.spaceConsumed).build();
+ }
+
+ @VisibleForTesting
+ protected String getTransportScheme() {
+ return SECURE_TRANSPORT_SCHEME;
+ }
+
+ @VisibleForTesting
+ String toRelativeFilePath(Path path) {
+ return path.makeQualified(uri, workingDirectory).toUri().getPath();
+ }
+
+ /**
+ * Get the current working directory for the given file system.
+ *
+ * @return the directory pathname
+ */
+ @Override
+ public Path getWorkingDirectory() {
+ return workingDirectory;
+ }
+
+ /**
+ * Set the current working directory for the given file system. All relative
+ * paths will be resolved relative to it.
+ *
+ * @param dir Working directory path.
+ */
+ @Override
+ public void setWorkingDirectory(final Path dir) {
+ if (dir == null) {
+ throw new InvalidPathException("Working directory cannot be set to NULL");
+ }
+
+ /**
+ * Do not validate the scheme and URI of the passsed parameter. When Adls
+ * runs as additional file system, working directory set has the default
+ * file system scheme and uri.
+ *
+ * Found a problem during PIG execution in
+ * https://github.com/apache/pig/blob/branch-0
+ * .15/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer
+ * /PigInputFormat.java#L235
+ * However similar problem would be present in other application so
+ * defaulting to build working directory using relative path only.
+ */
+ this.workingDirectory = this.makeAbsolute(dir);
+ }
+
+ /**
+ * Return the number of bytes that large input files should be optimally
+ * be split into to minimize i/o time.
+ *
+ * @deprecated use {@link #getDefaultBlockSize(Path)} instead
+ */
+ @Deprecated
+ public long getDefaultBlockSize() {
+ return ADL_BLOCK_SIZE;
+ }
+
+ /**
+ * Return the number of bytes that large input files should be optimally
+ * be split into to minimize i/o time. The given path will be used to
+ * locate the actual filesystem. The full path does not have to exist.
+ *
+ * @param f path of file
+ * @return the default block size for the path's filesystem
+ */
+ public long getDefaultBlockSize(Path f) {
+ return getDefaultBlockSize();
+ }
+
+ /**
+ * Get the block size.
+ * @param f the filename
+ * @return the number of bytes in a block
+ */
+ /**
+ * @deprecated Use getFileStatus() instead
+ */
+ @Deprecated
+ public long getBlockSize(Path f) throws IOException {
+ return ADL_BLOCK_SIZE;
+ }
+
+ @Override
+ public BlockLocation[] getFileBlockLocations(final FileStatus status,
+ final long offset, final long length) throws IOException {
+ if (status == null) {
+ return null;
+ }
+
+ if ((offset < 0) || (length < 0)) {
+ throw new IllegalArgumentException("Invalid start or len parameter");
+ }
+
+ if (status.getLen() < offset) {
+ return new BlockLocation[0];
+ }
+
+ final String[] name = {"localhost"};
+ final String[] host = {"localhost"};
+ long blockSize = ADL_BLOCK_SIZE;
+ int numberOfLocations =
+ (int) (length / blockSize) + ((length % blockSize == 0) ? 0 : 1);
+ BlockLocation[] locations = new BlockLocation[numberOfLocations];
+ for (int i = 0; i < locations.length; i++) {
+ long currentOffset = offset + (i * blockSize);
+ long currentLength = Math.min(blockSize, offset + length - currentOffset);
+ locations[i] = new BlockLocation(name, host, currentOffset,
+ currentLength);
+ }
+
+ return locations;
+ }
+
+ @Override
+ public BlockLocation[] getFileBlockLocations(final Path p, final long offset,
+ final long length) throws IOException {
+ // read ops incremented in getFileStatus
+ FileStatus fileStatus = getFileStatus(p);
+ return getFileBlockLocations(fileStatus, offset, length);
+ }
+
+ /**
+ * Get replication.
+ *
+ * @param src file name
+ * @return file replication
+ * @deprecated Use getFileStatus() instead
+ */
+ @Deprecated
+ public short getReplication(Path src) {
+ return ADL_REPLICATION_FACTOR;
+ }
+
+ private Path makeAbsolute(Path path) {
+ return path.isAbsolute() ? path : new Path(this.workingDirectory, path);
+ }
+
+ private static String getNonEmptyVal(Configuration conf, String key) {
+ String value = conf.get(key);
+ if (StringUtils.isEmpty(value)) {
+ throw new IllegalArgumentException(
+ "No value for " + key + " found in conf file.");
+ }
+ return value;
+ }
+
+ /**
+ * A wrapper of {@link Configuration#getPassword(String)}. It returns
+ * String instead of char[].
+ *
+ * @param conf the configuration
+ * @param key the property key
+ * @return the password string
+ * @throws IOException if the password was not found
+ */
+ private static String getPasswordString(Configuration conf, String key)
+ throws IOException {
+ char[] passchars = conf.getPassword(key);
+ if (passchars == null) {
+ throw new IOException("Password " + key + " not found");
+ }
+ return new String(passchars);
+ }
+
+ @VisibleForTesting
+ public void setUserGroupRepresentationAsUPN(boolean enableUPN) {
+ oidOrUpn = enableUPN ? UserGroupRepresentation.UPN :
+ UserGroupRepresentation.OID;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFsInputStream.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFsInputStream.java
new file mode 100644
index 0000000..5248cbf
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFsInputStream.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.adl;
+
+import com.microsoft.azure.datalake.store.ADLFileInputStream;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.FileSystem.Statistics;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .ADL_EXPERIMENT_POSITIONAL_READ_DEFAULT;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .ADL_EXPERIMENT_POSITIONAL_READ_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .DEFAULT_READ_AHEAD_BUFFER_SIZE;
+import static org.apache.hadoop.fs.adl.AdlConfKeys.READ_AHEAD_BUFFER_SIZE_KEY;
+
+/**
+ * Wraps {@link ADLFileInputStream} implementation.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class AdlFsInputStream extends FSInputStream {
+
+ private final ADLFileInputStream in;
+ private final Statistics stat;
+ private final boolean enablePositionalReadExperiment;
+
+ public AdlFsInputStream(ADLFileInputStream inputStream, Statistics statistics,
+ Configuration conf) throws IOException {
+ this.in = inputStream;
+ this.in.setBufferSize(conf.getInt(READ_AHEAD_BUFFER_SIZE_KEY,
+ DEFAULT_READ_AHEAD_BUFFER_SIZE));
+ enablePositionalReadExperiment = conf
+ .getBoolean(ADL_EXPERIMENT_POSITIONAL_READ_KEY,
+ ADL_EXPERIMENT_POSITIONAL_READ_DEFAULT);
+ stat = statistics;
+ }
+
+ @Override
+ public synchronized void seek(long pos) throws IOException {
+ in.seek(pos);
+ }
+
+ /**
+ * Return the current offset from the start of the file.
+ */
+ @Override
+ public synchronized long getPos() throws IOException {
+ return in.getPos();
+ }
+
+ @Override
+ public boolean seekToNewSource(long l) throws IOException {
+ return false;
+ }
+
+ @Override
+ public synchronized int read() throws IOException {
+ int ch = in.read();
+ if (stat != null && ch != -1) {
+ stat.incrementBytesRead(1);
+ }
+ return ch;
+ }
+
+ @Override
+ public int read(long position, byte[] buffer, int offset, int length)
+ throws IOException {
+ int numberOfByteRead = 0;
+ if (enablePositionalReadExperiment) {
+ numberOfByteRead = in.read(position, buffer, offset, length);
+ } else {
+ numberOfByteRead = super.read(position, buffer, offset, length);
+ }
+
+ if (stat != null && numberOfByteRead > 0) {
+ stat.incrementBytesRead(numberOfByteRead);
+ }
+ return numberOfByteRead;
+ }
+
+ @Override
+ public synchronized int read(byte[] buffer, int offset, int length)
+ throws IOException {
+ int numberOfByteRead = in.read(buffer, offset, length);
+ if (stat != null && numberOfByteRead > 0) {
+ stat.incrementBytesRead(numberOfByteRead);
+ }
+ return numberOfByteRead;
+ }
+
+ /**
+ * This method returns the remaining bytes in the stream, rather than the
+ * expected Java
+ * interpretation of {@link java.io.InputStream#available()}, which expects
+ * the
+ * number of remaining
+ * bytes in the local buffer. Moreover, it caps the value returned to a
+ * maximum of Integer.MAX_VALUE.
+ * These changed behaviors are to ensure compatibility with the
+ * expectations of HBase WAL reader,
+ * which depends on available() returning the number of bytes in stream.
+ *
+ * Given all other FileSystems in the hadoop ecosystem (especially HDFS) do
+ * this, it is possible other
+ * apps other than HBase would also pick up expectation of this behavior
+ * based on HDFS implementation.
+ * Therefore keeping this quirky behavior here, to ensure compatibility.
+ *
+ * @return remaining bytes in the stream, with maximum of Integer.MAX_VALUE.
+ * @throws IOException If fails to get the position or file length from SDK.
+ */
+ @Override
+ public synchronized int available() throws IOException {
+ return (int) Math.min(in.length() - in.getPos(), Integer.MAX_VALUE);
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ in.close();
+ }
+
+ @Override
+ public synchronized long skip(long pos) throws IOException {
+ return in.skip(pos);
+ }
+
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFsOutputStream.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFsOutputStream.java
new file mode 100644
index 0000000..2b89fb0
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFsOutputStream.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.adl;
+
+import com.microsoft.azure.datalake.store.ADLFileOutputStream;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Syncable;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .DEFAULT_WRITE_AHEAD_BUFFER_SIZE;
+import static org.apache.hadoop.fs.adl.AdlConfKeys.WRITE_BUFFER_SIZE_KEY;
+
+/**
+ * Wraps {@link com.microsoft.azure.datalake.store.ADLFileOutputStream}
+ * implementation.
+ *
+ * Flush semantics.
+ * no-op, since some parts of hadoop ecosystem call flush(), expecting it to
+ * have no perf impact. In hadoop filesystems, flush() itself guarantees no
+ * durability: that is achieved by calling hflush() or hsync()
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class AdlFsOutputStream extends OutputStream implements Syncable {
+ private final ADLFileOutputStream out;
+
+ public AdlFsOutputStream(ADLFileOutputStream out, Configuration configuration)
+ throws IOException {
+ this.out = out;
+ out.setBufferSize(configuration
+ .getInt(WRITE_BUFFER_SIZE_KEY, DEFAULT_WRITE_AHEAD_BUFFER_SIZE));
+ }
+
+ @Override
+ public synchronized void write(int b) throws IOException {
+ out.write(b);
+ }
+
+ @Override
+ public synchronized void write(byte[] b, int off, int len)
+ throws IOException {
+ out.write(b, off, len);
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ out.close();
+ }
+
+ public synchronized void sync() throws IOException {
+ out.flush();
+ }
+
+ public synchronized void hflush() throws IOException {
+ out.flush();
+ }
+
+ public synchronized void hsync() throws IOException {
+ out.flush();
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlPermission.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlPermission.java
new file mode 100644
index 0000000..af3342a
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlPermission.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl;
+
+import org.apache.hadoop.fs.permission.FsPermission;
+
+/**
+ * Hadoop shell command -getfacl does not invoke getAclStatus if FsPermission
+ * from getFileStatus has not set ACL bit to true. By default getAclBit returns
+ * false.
+ *
+ * Provision to make additional call to invoke getAclStatus would be redundant
+ * when adls is running as additional FS. To avoid this redundancy, provided
+ * configuration to return true/false on getAclBit.
+ */
+class AdlPermission extends FsPermission {
+ private final boolean aclBit;
+
+ AdlPermission(boolean aclBitStatus, Short aShort) {
+ super(aShort);
+ this.aclBit = aclBitStatus;
+ }
+
+ /**
+ * Returns true if "adl.feature.support.acl.bit" configuration is set to
+ * true.
+ *
+ * If configuration is not set then default value is true.
+ *
+ * @return If configuration is not set then default value is true.
+ */
+ public boolean getAclBit() {
+ return aclBit;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof FsPermission) {
+ FsPermission that = (FsPermission) obj;
+ return this.getUserAction() == that.getUserAction()
+ && this.getGroupAction() == that.getGroupAction()
+ && this.getOtherAction() == that.getOtherAction()
+ && this.getStickyBit() == that.getStickyBit();
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return toShort();
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/SdkTokenProviderAdapter.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/SdkTokenProviderAdapter.java
new file mode 100644
index 0000000..7b107ae
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/SdkTokenProviderAdapter.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.adl;
+
+import com.microsoft.azure.datalake.store.oauth2.AccessTokenProvider;
+import com.microsoft.azure.datalake.store.oauth2.AzureADToken;
+import org.apache.hadoop.fs.adl.oauth2.AzureADTokenProvider;
+
+import java.io.IOException;
+
+final class SdkTokenProviderAdapter extends AccessTokenProvider {
+
+ private AzureADTokenProvider tokenProvider;
+
+ SdkTokenProviderAdapter(AzureADTokenProvider tp) {
+ this.tokenProvider = tp;
+ }
+
+ protected AzureADToken refreshToken() throws IOException {
+ AzureADToken azureADToken = new AzureADToken();
+ azureADToken.accessToken = tokenProvider.getAccessToken();
+ azureADToken.expiry = tokenProvider.getExpiryTime();
+ return azureADToken;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/TokenProviderType.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/TokenProviderType.java
new file mode 100644
index 0000000..9fd4f4f
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/TokenProviderType.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.adl;
+
+enum TokenProviderType {
+ RefreshToken,
+ ClientCredential,
+ Custom
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/oauth2/AzureADTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/oauth2/AzureADTokenProvider.java
new file mode 100644
index 0000000..a0b3922
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/oauth2/AzureADTokenProvider.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.adl.oauth2;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+
+import java.io.IOException;
+import java.util.Date;
+
+/**
+ * Provide an Azure Active Directory supported
+ * OAuth2 access token to be used to authenticate REST calls against Azure data
+ * lake file system {@link org.apache.hadoop.fs.adl.AdlFileSystem}.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class AzureADTokenProvider {
+
+ /**
+ * Initialize with supported configuration. This method is invoked when the
+ * {@link org.apache.hadoop.fs.adl.AdlFileSystem#initialize
+ * (URI, Configuration)} method is invoked.
+ *
+ * @param configuration Configuration object
+ * @throws IOException if instance can not be configured.
+ */
+ public abstract void initialize(Configuration configuration)
+ throws IOException;
+
+ /**
+ * Obtain the access token that should be added to https connection's header.
+ * Will be called depending upon {@link #getExpiryTime()} expiry time is set,
+ * so implementations should be performant. Implementations are responsible
+ * for any refreshing of the token.
+ *
+ * @return String containing the access token
+ * @throws IOException if there is an error fetching the token
+ */
+ public abstract String getAccessToken() throws IOException;
+
+ /**
+ * Obtain expiry time of the token. If implementation is performant enough to
+ * maintain expiry and expect {@link #getAccessToken()} call for every
+ * connection then safe to return current or past time.
+ *
+ * However recommended to use the token expiry time received from Azure Active
+ * Directory.
+ *
+ * @return Date to expire access token retrieved from AAD.
+ */
+ public abstract Date getExpiryTime();
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/oauth2/package-info.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/oauth2/package-info.java
new file mode 100644
index 0000000..1613941
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/oauth2/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/**
+ * public interface to expose OAuth2 authentication related features.
+ */
+package org.apache.hadoop.fs.adl.oauth2;
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/package-info.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/package-info.java
new file mode 100644
index 0000000..456eebc
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/**
+ * Supporting classes for metrics instrumentation.
+ */
+package org.apache.hadoop.fs.adl;
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
new file mode 100644
index 0000000..6d9e173
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
@@ -0,0 +1,265 @@
+
+
+# Hadoop Azure Data Lake Support
+
+* [Introduction](#Introduction)
+* [Features](#Features)
+* [Limitations](#Limitations)
+* [Usage](#Usage)
+ * [Concepts](#Concepts)
+ * [OAuth2 Support](#OAuth2_Support)
+ * [Configuring Credentials & FileSystem](#Configuring_Credentials)
+ * [Using Refresh Token](#Refresh_Token)
+ * [Using Client Keys](#Client_Credential_Token)
+ * [Protecting the Credentials with Credential Providers](#Credential_Provider)
+ * [Enabling ADL Filesystem](#Enabling_ADL)
+ * [Accessing adl URLs](#Accessing_adl_URLs)
+ * [User/Group Representation](#OIDtoUPNConfiguration)
+* [Testing the hadoop-azure Module](#Testing_the_hadoop-azure_Module)
+
+## Introduction
+
+The hadoop-azure-datalake module provides support for integration with
+[Azure Data Lake Store]( https://azure.microsoft.com/en-in/documentation/services/data-lake-store/).
+The jar file is named azure-datalake-store.jar.
+
+## Features
+
+* Read and write data stored in an Azure Data Lake Storage account.
+* Reference file system paths using URLs using the `adl` scheme for Secure Webhdfs i.e. SSL
+ encrypted access.
+* Can act as a source of data in a MapReduce job, or a sink.
+* Tested on both Linux and Windows.
+* Tested for scale.
+* API setOwner/setAcl/removeAclEntries/modifyAclEntries accepts UPN or OID
+ (Object ID) as user and group name.
+
+## Limitations
+Partial or no support for the following operations :
+
+* Operation on Symbolic Link
+* Proxy Users
+* File Truncate
+* File Checksum
+* File replication factor
+* Home directory the active user on Hadoop cluster.
+* Extended Attributes(XAttrs) Operations
+* Snapshot Operations
+* Delegation Token Operations
+* User and group information returned as ListStatus and GetFileStatus is in form of GUID associated in Azure Active Directory.
+
+## Usage
+
+### Concepts
+Azure Data Lake Storage access path syntax is
+
+ adl://.azuredatalakestore.net/
+
+Get started with azure data lake account with [https://azure.microsoft.com/en-in/documentation/articles/data-lake-store-get-started-portal/](https://azure.microsoft.com/en-in/documentation/articles/data-lake-store-get-started-portal/)
+
+#### OAuth2 Support
+Usage of Azure Data Lake Storage requires OAuth2 bearer token to be present as part of the HTTPS header as per OAuth2 specification. Valid OAuth2 bearer token should be obtained from Azure Active Directory for valid users who have access to Azure Data Lake Storage Account.
+
+Azure Active Directory (Azure AD) is Microsoft's multi-tenant cloud based directory and identity management service. See [https://azure.microsoft.com/en-in/documentation/articles/active-directory-whatis/](https://azure.microsoft.com/en-in/documentation/articles/active-directory-whatis/)
+
+Following sections describes on OAuth2 configuration in core-site.xml.
+
+## Configuring Credentials & FileSystem
+Credentials can be configured using either a refresh token (associated with a user) or a client credential (analogous to a service principal).
+
+### Using Refresh Token
+
+Add the following properties to your core-site.xml
+
+
+ dfs.adls.oauth2.access.token.provider.type
+ RefreshToken
+
+
+Application require to set Client id and OAuth2 refresh token from Azure Active Directory associated with client id. See [https://github.com/AzureAD/azure-activedirectory-library-for-java](https://github.com/AzureAD/azure-activedirectory-library-for-java).
+
+**Do not share client id and refresh token, it must be kept secret.**
+
+
+ dfs.adls.oauth2.client.id
+
+
+
+
+ dfs.adls.oauth2.refresh.token
+
+
+
+
+### Using Client Keys
+
+#### Generating the Service Principal
+1. Go to the portal (https://portal.azure.com)
+2. Under "Browse", look for Active Directory and click on it.
+3. Create "Web Application". Remember the name you create here - that is what you will add to your ADL account as authorized user.
+4. Go through the wizard
+5. Once app is created, Go to app configuration, and find the section on "keys"
+6. Select a key duration and hit save. Save the generated keys.
+7. Note down the properties you will need to auth:
+ - The client ID
+ - The key you just generated above
+ - The token endpoint (select "View endpoints" at the bottom of the page and copy/paste the OAuth2 .0 Token Endpoint value)
+ - Resource: Always https://management.core.windows.net/ , for all customers
+
+#### Adding the service principal to your ADL Account
+1. Go to the portal again, and open your ADL account
+2. Select Users under Settings
+3. Add your user name you created in Step 6 above (note that it does not show up in the list, but will be found if you searched for the name)
+4. Add "Owner" role
+
+#### Configure core-site.xml
+Add the following properties to your core-site.xml
+
+
+ dfs.adls.oauth2.refresh.url
+ TOKEN ENDPOINT FROM STEP 7 ABOVE
+
+
+
+ dfs.adls.oauth2.client.id
+ CLIENT ID FROM STEP 7 ABOVE
+
+
+
+ dfs.adls.oauth2.credential
+ PASSWORD FROM STEP 7 ABOVE
+
+
+
+### Protecting the Credentials with Credential Providers
+
+In many Hadoop clusters, the core-site.xml file is world-readable. To protect
+these credentials from prying eyes, it is recommended that you use the
+credential provider framework to securely store them and access them through
+configuration.
+
+All ADLS credential properties can be protected by credential providers.
+For additional reading on the credential provider API, see
+[Credential Provider API](../hadoop-project-dist/hadoop-common/CredentialProviderAPI.html).
+
+#### Provisioning
+
+```
+% hadoop credential create dfs.adls.oauth2.refresh.token -value 123
+ -provider localjceks://file/home/foo/adls.jceks
+% hadoop credential create dfs.adls.oauth2.credential -value 123
+ -provider localjceks://file/home/foo/adls.jceks
+```
+
+#### Configuring core-site.xml or command line property
+
+```
+
+ hadoop.security.credential.provider.path
+ localjceks://file/home/foo/adls.jceks
+ Path to interrogate for protected credentials.
+
+```
+
+#### Running DistCp
+
+```
+% hadoop distcp
+ [-D hadoop.security.credential.provider.path=localjceks://file/home/user/adls.jceks]
+ hdfs://:9001/user/foo/007020615
+ adl://.azuredatalakestore.net/testDir/
+```
+
+NOTE: You may optionally add the provider path property to the distcp command
+line instead of added job specific configuration to a generic core-site.xml.
+The square brackets above illustrate this capability.
+
+
+## Enabling ADL Filesystem
+
+For ADL FileSystem to take effect. Update core-site.xml with
+
+
+ fs.adl.impl
+ org.apache.hadoop.fs.adl.AdlFileSystem
+
+
+
+ fs.AbstractFileSystem.adl.impl
+ org.apache.hadoop.fs.adl.Adl
+
+
+
+### Accessing adl URLs
+
+After credentials are configured in core-site.xml, any Hadoop component may
+reference files in that Azure Data Lake Storage account by using URLs of the following
+format:
+
+ adl://.azuredatalakestore.net/
+
+The schemes `adl` identify a URL on a file system backed by Azure
+Data Lake Storage. `adl` utilizes encrypted HTTPS access for all interaction with
+the Azure Data Lake Storage API.
+
+For example, the following
+[FileSystem Shell](../hadoop-project-dist/hadoop-common/FileSystemShell.html)
+commands demonstrate access to a storage account named `youraccount`.
+
+ > hadoop fs -mkdir adl://yourcontainer.azuredatalakestore.net/testDir
+
+ > hadoop fs -put testFile adl://yourcontainer.azuredatalakestore.net/testDir/testFile
+
+ > hadoop fs -cat adl://yourcontainer.azuredatalakestore.net/testDir/testFile
+ test file content
+
+### User/Group Representation
+The hadoop-azure-datalake module provides support for configuring how
+User/Group information is represented during
+getFileStatus/listStatus/getAclStatus.
+
+Add the following properties to your core-site.xml
+
+
+ adl.feature.ownerandgroup.enableupn
+ true
+
+ When true : User and Group in FileStatus/AclStatus response is
+ represented as user friendly name as per Azure AD profile.
+
+ When false (default) : User and Group in FileStatus/AclStatus
+ response is represented by the unique identifier from Azure AD
+ profile (Object ID as GUID).
+
+ For performance optimization, Recommended default value.
+
+
+
+## Testing the azure-datalake-store Module
+The hadoop-azure module includes a full suite of unit tests. Most of the tests will run without additional configuration by running mvn test. This includes tests against mocked storage, which is an in-memory emulation of Azure Data Lake Storage.
+
+A selection of tests can run against the Azure Data Lake Storage. To run these
+tests, please create `src/test/resources/auth-keys.xml` with Adl account
+information mentioned in the above sections and the following properties.
+
+
+ dfs.adl.test.contract.enable
+ true
+
+
+
+ test.fs.adl.name
+ adl://yourcontainer.azuredatalakestore.net
+
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java
new file mode 100644
index 0000000..55c8f81
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.adl;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.adl.common.CustomMockTokenProvider;
+import org.apache.hadoop.fs.adl.oauth2.AzureADTokenProvider;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY;
+
+import com.squareup.okhttp.mockwebserver.MockWebServer;
+
+import org.junit.After;
+import org.junit.Before;
+
+/**
+ * Mock server to simulate Adls backend calls. This infrastructure is expandable
+ * to override expected server response based on the derived test functionality.
+ * Common functionality to generate token information before request is send to
+ * adls backend is also managed within AdlMockWebServer implementation using
+ * {@link org.apache.hadoop.fs.adl.common.CustomMockTokenProvider}.
+ */
+public class AdlMockWebServer {
+ // Create a MockWebServer. These are lean enough that you can create a new
+ // instance for every unit test.
+ private MockWebServer server = null;
+ private TestableAdlFileSystem fs = null;
+ private int port = 0;
+ private Configuration conf = new Configuration();
+
+ public MockWebServer getMockServer() {
+ return server;
+ }
+
+ public TestableAdlFileSystem getMockAdlFileSystem() {
+ return fs;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ public Configuration getConf() {
+ return conf;
+ }
+
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ @Before
+ public void preTestSetup() throws IOException, URISyntaxException {
+ server = new MockWebServer();
+
+ // Start the server.
+ server.start();
+
+ // Ask the server for its URL. You'll need this to make HTTP requests.
+ URL baseUrl = server.getUrl("");
+ port = baseUrl.getPort();
+
+ // Exercise your application code, which should make those HTTP requests.
+ // Responses are returned in the same order that they are enqueued.
+ fs = new TestableAdlFileSystem();
+
+ conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
+ CustomMockTokenProvider.class, AzureADTokenProvider.class);
+
+ URI uri = new URI("adl://localhost:" + port);
+ fs.initialize(uri, conf);
+ }
+
+ @After
+ public void postTestSetup() throws IOException {
+ fs.close();
+ server.shutdown();
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestACLFeatures.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestACLFeatures.java
new file mode 100644
index 0000000..b420daa
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestACLFeatures.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclEntryScope;
+import org.apache.hadoop.fs.permission.AclEntryType;
+import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.AccessControlException;
+
+import com.squareup.okhttp.mockwebserver.MockResponse;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Stub adl server and test acl data conversion within SDK and Hadoop adl
+ * client.
+ */
+public class TestACLFeatures extends AdlMockWebServer {
+
+ @Test(expected=AccessControlException.class)
+ public void testModifyAclEntries() throws URISyntaxException, IOException {
+ getMockServer().enqueue(new MockResponse().setResponseCode(200));
+ List entries = new ArrayList();
+ AclEntry.Builder aclEntryBuilder = new AclEntry.Builder();
+ aclEntryBuilder.setName("hadoop");
+ aclEntryBuilder.setType(AclEntryType.USER);
+ aclEntryBuilder.setPermission(FsAction.ALL);
+ aclEntryBuilder.setScope(AclEntryScope.ACCESS);
+ entries.add(aclEntryBuilder.build());
+
+ aclEntryBuilder.setName("hdfs");
+ aclEntryBuilder.setType(AclEntryType.GROUP);
+ aclEntryBuilder.setPermission(FsAction.READ_WRITE);
+ aclEntryBuilder.setScope(AclEntryScope.DEFAULT);
+ entries.add(aclEntryBuilder.build());
+
+ getMockAdlFileSystem().modifyAclEntries(new Path("/test1/test2"), entries);
+
+ getMockServer().enqueue(new MockResponse().setResponseCode(403)
+ .setBody(TestADLResponseData.getAccessControlException()));
+
+ getMockAdlFileSystem()
+ .modifyAclEntries(new Path("/test1/test2"), entries);
+ }
+
+ @Test(expected=AccessControlException.class)
+ public void testRemoveAclEntriesWithOnlyUsers()
+ throws URISyntaxException, IOException {
+ getMockServer().enqueue(new MockResponse().setResponseCode(200));
+ List entries = new ArrayList();
+ AclEntry.Builder aclEntryBuilder = new AclEntry.Builder();
+ aclEntryBuilder.setName("hadoop");
+ aclEntryBuilder.setType(AclEntryType.USER);
+ entries.add(aclEntryBuilder.build());
+
+ getMockAdlFileSystem().removeAclEntries(new Path("/test1/test2"), entries);
+
+ getMockServer().enqueue(new MockResponse().setResponseCode(403)
+ .setBody(TestADLResponseData.getAccessControlException()));
+
+ getMockAdlFileSystem()
+ .removeAclEntries(new Path("/test1/test2"), entries);
+ }
+
+ @Test(expected=AccessControlException.class)
+ public void testRemoveAclEntries() throws URISyntaxException, IOException {
+ getMockServer().enqueue(new MockResponse().setResponseCode(200));
+ List entries = new ArrayList();
+ AclEntry.Builder aclEntryBuilder = new AclEntry.Builder();
+ aclEntryBuilder.setName("hadoop");
+ aclEntryBuilder.setType(AclEntryType.USER);
+ aclEntryBuilder.setPermission(FsAction.ALL);
+ aclEntryBuilder.setScope(AclEntryScope.ACCESS);
+ entries.add(aclEntryBuilder.build());
+
+ aclEntryBuilder.setName("hdfs");
+ aclEntryBuilder.setType(AclEntryType.GROUP);
+ aclEntryBuilder.setPermission(FsAction.READ_WRITE);
+ aclEntryBuilder.setScope(AclEntryScope.DEFAULT);
+ entries.add(aclEntryBuilder.build());
+
+ getMockAdlFileSystem().removeAclEntries(new Path("/test1/test2"), entries);
+
+ getMockServer().enqueue(new MockResponse().setResponseCode(403)
+ .setBody(TestADLResponseData.getAccessControlException()));
+
+ getMockAdlFileSystem()
+ .removeAclEntries(new Path("/test1/test2"), entries);
+ }
+
+ @Test(expected=AccessControlException.class)
+ public void testRemoveDefaultAclEntries()
+ throws URISyntaxException, IOException {
+ getMockServer().enqueue(new MockResponse().setResponseCode(200));
+ getMockAdlFileSystem().removeDefaultAcl(new Path("/test1/test2"));
+
+ getMockServer().enqueue(new MockResponse().setResponseCode(403)
+ .setBody(TestADLResponseData.getAccessControlException()));
+
+ getMockAdlFileSystem().removeDefaultAcl(new Path("/test1/test2"));
+ }
+
+ @Test(expected=AccessControlException.class)
+ public void testRemoveAcl() throws URISyntaxException, IOException {
+ getMockServer().enqueue(new MockResponse().setResponseCode(200));
+ getMockAdlFileSystem().removeAcl(new Path("/test1/test2"));
+
+ getMockServer().enqueue(new MockResponse().setResponseCode(403)
+ .setBody(TestADLResponseData.getAccessControlException()));
+
+ getMockAdlFileSystem().removeAcl(new Path("/test1/test2"));
+ }
+
+ @Test(expected=AccessControlException.class)
+ public void testSetAcl() throws URISyntaxException, IOException {
+ getMockServer().enqueue(new MockResponse().setResponseCode(200));
+ List entries = new ArrayList();
+ AclEntry.Builder aclEntryBuilder = new AclEntry.Builder();
+ aclEntryBuilder.setName("hadoop");
+ aclEntryBuilder.setType(AclEntryType.USER);
+ aclEntryBuilder.setPermission(FsAction.ALL);
+ aclEntryBuilder.setScope(AclEntryScope.ACCESS);
+ entries.add(aclEntryBuilder.build());
+
+ aclEntryBuilder.setName("hdfs");
+ aclEntryBuilder.setType(AclEntryType.GROUP);
+ aclEntryBuilder.setPermission(FsAction.READ_WRITE);
+ aclEntryBuilder.setScope(AclEntryScope.DEFAULT);
+ entries.add(aclEntryBuilder.build());
+
+ getMockAdlFileSystem().setAcl(new Path("/test1/test2"), entries);
+
+ getMockServer().enqueue(new MockResponse().setResponseCode(403)
+ .setBody(TestADLResponseData.getAccessControlException()));
+
+ getMockAdlFileSystem().setAcl(new Path("/test1/test2"), entries);
+ }
+
+ @Test(expected=AccessControlException.class)
+ public void testCheckAccess() throws URISyntaxException, IOException {
+ getMockServer().enqueue(new MockResponse().setResponseCode(200));
+ getMockAdlFileSystem().access(new Path("/test1/test2"), FsAction.ALL);
+
+ getMockServer().enqueue(new MockResponse().setResponseCode(200));
+ getMockAdlFileSystem().access(new Path("/test1/test2"), FsAction.EXECUTE);
+
+ getMockServer().enqueue(new MockResponse().setResponseCode(200));
+ getMockAdlFileSystem().access(new Path("/test1/test2"), FsAction.READ);
+
+ getMockServer().enqueue(new MockResponse().setResponseCode(200));
+ getMockAdlFileSystem()
+ .access(new Path("/test1/test2"), FsAction.READ_EXECUTE);
+
+ getMockServer().enqueue(new MockResponse().setResponseCode(200));
+ getMockAdlFileSystem()
+ .access(new Path("/test1/test2"), FsAction.READ_WRITE);
+
+ getMockServer().enqueue(new MockResponse().setResponseCode(200));
+ getMockAdlFileSystem().access(new Path("/test1/test2"), FsAction.NONE);
+
+ getMockServer().enqueue(new MockResponse().setResponseCode(200));
+ getMockAdlFileSystem().access(new Path("/test1/test2"), FsAction.WRITE);
+
+ getMockServer().enqueue(new MockResponse().setResponseCode(200));
+ getMockAdlFileSystem()
+ .access(new Path("/test1/test2"), FsAction.WRITE_EXECUTE);
+
+ getMockServer().enqueue(new MockResponse().setResponseCode(403)
+ .setBody(TestADLResponseData.getAccessControlException()));
+
+ getMockAdlFileSystem()
+ .access(new Path("/test1/test2"), FsAction.WRITE_EXECUTE);
+ }
+
+ @Test(expected=AccessControlException.class)
+ public void testSetPermission() throws URISyntaxException, IOException {
+ getMockServer().enqueue(new MockResponse().setResponseCode(200));
+ getMockAdlFileSystem()
+ .setPermission(new Path("/test1/test2"), FsPermission.getDefault());
+
+ getMockServer().enqueue(new MockResponse().setResponseCode(403)
+ .setBody(TestADLResponseData.getAccessControlException()));
+
+ getMockAdlFileSystem()
+ .setPermission(new Path("/test1/test2"), FsPermission.getDefault());
+ }
+
+ @Test(expected=AccessControlException.class)
+ public void testSetOwner() throws URISyntaxException, IOException {
+ getMockServer().enqueue(new MockResponse().setResponseCode(200));
+ getMockAdlFileSystem().setOwner(new Path("/test1/test2"), "hadoop", "hdfs");
+
+ getMockServer().enqueue(new MockResponse().setResponseCode(403)
+ .setBody(TestADLResponseData.getAccessControlException()));
+
+ getMockAdlFileSystem()
+ .setOwner(new Path("/test1/test2"), "hadoop", "hdfs");
+ }
+
+ @Test
+ public void getAclStatusAsExpected() throws URISyntaxException, IOException {
+ getMockServer().enqueue(new MockResponse().setResponseCode(200)
+ .setBody(TestADLResponseData.getGetAclStatusJSONResponse()));
+ AclStatus aclStatus = getMockAdlFileSystem()
+ .getAclStatus(new Path("/test1/test2"));
+ Assert.assertEquals(aclStatus.getGroup(), "supergroup");
+ Assert.assertEquals(aclStatus.getOwner(), "hadoop");
+ Assert.assertEquals((Short) aclStatus.getPermission().toShort(),
+ Short.valueOf("775", 8));
+
+ for (AclEntry entry : aclStatus.getEntries()) {
+ if (!(entry.toString().equalsIgnoreCase("user:carla:rw-") || entry
+ .toString().equalsIgnoreCase("group::r-x"))) {
+ Assert.fail("Unexpected entry : " + entry.toString());
+ }
+ }
+ }
+
+ @Test(expected=FileNotFoundException.class)
+ public void getAclStatusNotExists() throws URISyntaxException, IOException {
+ getMockServer().enqueue(new MockResponse().setResponseCode(404)
+ .setBody(TestADLResponseData.getFileNotFoundException()));
+
+ getMockAdlFileSystem().getAclStatus(new Path("/test1/test2"));
+ }
+
+ @Test(expected=AccessControlException.class)
+ public void testAclStatusDenied() throws URISyntaxException, IOException {
+ getMockServer().enqueue(new MockResponse().setResponseCode(403)
+ .setBody(TestADLResponseData.getAccessControlException()));
+
+ getMockAdlFileSystem().getAclStatus(new Path("/test1/test2"));
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestADLResponseData.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestADLResponseData.java
new file mode 100644
index 0000000..788242e
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestADLResponseData.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl;
+
+import org.apache.hadoop.fs.FileStatus;
+
+import java.util.Random;
+
+/**
+ * Mock up response data returned from Adl storage account.
+ */
+public final class TestADLResponseData {
+
+ private TestADLResponseData() {
+
+ }
+
+ public static String getGetFileStatusJSONResponse(FileStatus status) {
+ return "{\"FileStatus\":{\"length\":" + status.getLen() + "," +
+ "\"pathSuffix\":\"\",\"type\":\"" + (status.isDirectory() ?
+ "DIRECTORY" :
+ "FILE") + "\"" +
+ ",\"blockSize\":" + status.getBlockSize() + ",\"accessTime\":" +
+ status.getAccessTime() + ",\"modificationTime\":" + status
+ .getModificationTime() + "" +
+ ",\"replication\":" + status.getReplication() + ",\"permission\":\""
+ + status.getPermission() + "\",\"owner\":\"" + status.getOwner()
+ + "\",\"group\":\"" + status.getGroup() + "\"}}";
+ }
+
+ public static String getGetFileStatusJSONResponse() {
+ return getGetFileStatusJSONResponse(4194304);
+ }
+
+ public static String getGetAclStatusJSONResponse() {
+ return "{\n" + " \"AclStatus\": {\n" + " \"entries\": [\n"
+ + " \"user:carla:rw-\", \n" + " \"group::r-x\"\n"
+ + " ], \n" + " \"group\": \"supergroup\", \n"
+ + " \"owner\": \"hadoop\", \n"
+ + " \"permission\":\"775\",\n" + " \"stickyBit\": false\n"
+ + " }\n" + "}";
+ }
+
+ public static String getGetFileStatusJSONResponse(long length) {
+ return "{\"FileStatus\":{\"length\":" + length + "," +
+ "\"pathSuffix\":\"\",\"type\":\"FILE\",\"blockSize\":268435456," +
+ "\"accessTime\":1452103827023,\"modificationTime\":1452103827023," +
+ "\"replication\":0,\"permission\":\"777\"," +
+ "\"owner\":\"NotSupportYet\",\"group\":\"NotSupportYet\"}}";
+ }
+
+ public static String getGetFileStatusJSONResponse(boolean aclBit) {
+ return "{\"FileStatus\":{\"length\":1024," +
+ "\"pathSuffix\":\"\",\"type\":\"FILE\",\"blockSize\":268435456," +
+ "\"accessTime\":1452103827023,\"modificationTime\":1452103827023," +
+ "\"replication\":0,\"permission\":\"777\"," +
+ "\"owner\":\"NotSupportYet\",\"group\":\"NotSupportYet\",\"aclBit\":\""
+ + aclBit + "\"}}";
+ }
+
+ public static String getListFileStatusJSONResponse(int dirSize) {
+ String list = "";
+ for (int i = 0; i < dirSize; ++i) {
+ list += "{\"length\":1024,\"pathSuffix\":\"" + java.util.UUID.randomUUID()
+ + "\",\"type\":\"FILE\",\"blockSize\":268435456," +
+ "\"accessTime\":1452103878833," +
+ "\"modificationTime\":1452103879190,\"replication\":0," +
+ "\"permission\":\"777\",\"owner\":\"NotSupportYet\"," +
+ "\"group\":\"NotSupportYet\"},";
+ }
+
+ list = list.substring(0, list.length() - 1);
+ return "{\"FileStatuses\":{\"FileStatus\":[" + list + "]}}";
+ }
+
+ public static String getListFileStatusJSONResponse(boolean aclBit) {
+ return "{\"FileStatuses\":{\"FileStatus\":[{\"length\":0,\"pathSuffix\":\""
+ + java.util.UUID.randomUUID()
+ + "\",\"type\":\"DIRECTORY\",\"blockSize\":0,"
+ + "\"accessTime\":1481184513488,"
+ + "\"modificationTime\":1481184513488,\"replication\":0,"
+ + "\"permission\":\"770\","
+ + "\"owner\":\"4b27fe1a-d9ab-4a04-ad7a-4bba72cd9e6c\","
+ + "\"group\":\"4b27fe1a-d9ab-4a04-ad7a-4bba72cd9e6c\",\"aclBit\":\""
+ + aclBit + "\"}]}}";
+ }
+
+ public static String getJSONResponse(boolean status) {
+ return "{\"boolean\":" + status + "}";
+ }
+
+ public static String getErrorIllegalArgumentExceptionJSONResponse() {
+ return "{\n" +
+ " \"RemoteException\":\n" +
+ " {\n" +
+ " \"exception\" : \"IllegalArgumentException\",\n" +
+ " \"javaClassName\": \"java.lang.IllegalArgumentException\",\n" +
+ " \"message\" : \"Invalid\"" +
+ " }\n" +
+ "}";
+ }
+
+ public static String getErrorBadOffsetExceptionJSONResponse() {
+ return "{\n" +
+ " \"RemoteException\":\n" +
+ " {\n" +
+ " \"exception\" : \"BadOffsetException\",\n" +
+ " \"javaClassName\": \"org.apache.hadoop.fs.adl"
+ + ".BadOffsetException\",\n" +
+ " \"message\" : \"Invalid\"" +
+ " }\n" +
+ "}";
+ }
+
+ public static String getErrorInternalServerExceptionJSONResponse() {
+ return "{\n" +
+ " \"RemoteException\":\n" +
+ " {\n" +
+ " \"exception\" : \"RuntimeException\",\n" +
+ " \"javaClassName\": \"java.lang.RuntimeException\",\n" +
+ " \"message\" : \"Internal Server Error\"" +
+ " }\n" +
+ "}";
+ }
+
+ public static String getAccessControlException() {
+ return "{\n" + " \"RemoteException\":\n" + " {\n"
+ + " \"exception\" : \"AccessControlException\",\n"
+ + " \"javaClassName\": \"org.apache.hadoop.security"
+ + ".AccessControlException\",\n"
+ + " \"message\" : \"Permission denied: ...\"\n" + " }\n" + "}";
+ }
+
+ public static String getFileNotFoundException() {
+ return "{\n" + " \"RemoteException\":\n" + " {\n"
+ + " \"exception\" : \"FileNotFoundException\",\n"
+ + " \"javaClassName\": \"java.io.FileNotFoundException\",\n"
+ + " \"message\" : \"File does not exist\"\n" + " }\n" + "}";
+ }
+
+ public static byte[] getRandomByteArrayData() {
+ return getRandomByteArrayData(4 * 1024 * 1024);
+ }
+
+ public static byte[] getRandomByteArrayData(int size) {
+ byte[] b = new byte[size];
+ Random rand = new Random();
+ rand.nextBytes(b);
+ return b;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAdlRead.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAdlRead.java
new file mode 100644
index 0000000..172663c
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAdlRead.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.adl;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.adl.common.Parallelized;
+import org.apache.hadoop.fs.adl.common.TestDataForRead;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.ByteArrayInputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Random;
+
+import static org.apache.hadoop.fs.adl.AdlConfKeys.READ_AHEAD_BUFFER_SIZE_KEY;
+
+/**
+ * This class is responsible for stress positional reads vs number of network
+ * calls required by to fetch the amount of data. Test does ensure the data
+ * integrity and order of the data is maintained.
+ */
+@RunWith(Parallelized.class)
+public class TestAdlRead extends AdlMockWebServer {
+
+ private TestDataForRead testData;
+
+ public TestAdlRead(TestDataForRead testData) {
+ Configuration configuration = new Configuration();
+ configuration.setInt(READ_AHEAD_BUFFER_SIZE_KEY, 4 * 1024);
+ setConf(configuration);
+ this.testData = testData;
+ }
+
+ @Parameterized.Parameters(name = "{index}")
+ public static Collection testDataForReadOperation() {
+ return Arrays.asList(new Object[][] {
+
+ //--------------------------
+ // Test Data
+ //--------------------------
+ {new TestDataForRead("Hello World".getBytes(), 2, 1000, true)},
+ {new TestDataForRead(
+ ("the problem you appear to be wrestling with is that this doesn't "
+ + "display very well. ").getBytes(), 2, 1000, true)},
+ {new TestDataForRead(("您的數據是寶貴的資產,以您的組織,並有當前和未來價值。由於這個原因,"
+ + "所有的數據應存儲以供將來分析。今天,這往往是不這樣做," + "因為傳統的分析基礎架構的限制,"
+ + "像模式的預定義,存儲大數據集和不同的數據筒倉的傳播的成本。"
+ + "為了應對這一挑戰,數據湖面概念被引入作為一個企業級存儲庫來存儲所有"
+ + "類型的在一個地方收集到的數據。對於運作和探索性分析的目的,所有類型的" + "數據可以定義需求或模式之前被存儲在數據湖。")
+ .getBytes(), 2, 1000, true)}, {new TestDataForRead(
+ TestADLResponseData.getRandomByteArrayData(4 * 1024), 2, 10, true)},
+ {new TestDataForRead(TestADLResponseData.getRandomByteArrayData(100), 2,
+ 1000, true)}, {new TestDataForRead(
+ TestADLResponseData.getRandomByteArrayData(1 * 1024), 2, 50, true)},
+ {new TestDataForRead(
+ TestADLResponseData.getRandomByteArrayData(8 * 1024), 3, 10,
+ false)}, {new TestDataForRead(
+ TestADLResponseData.getRandomByteArrayData(16 * 1024), 5, 10, false)},
+ {new TestDataForRead(
+ TestADLResponseData.getRandomByteArrayData(32 * 1024), 9, 10,
+ false)}, {new TestDataForRead(
+ TestADLResponseData.getRandomByteArrayData(64 * 1024), 17, 10,
+ false)}});
+ }
+
+ @Test
+ public void testEntireBytes() throws IOException, InterruptedException {
+ getMockServer().setDispatcher(testData.getDispatcher());
+ FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
+ byte[] expectedData = new byte[testData.getActualData().length];
+ int n = 0;
+ int len = expectedData.length;
+ int off = 0;
+ while (n < len) {
+ int count = in.read(expectedData, off + n, len - n);
+ if (count < 0) {
+ throw new EOFException();
+ }
+ n += count;
+ }
+
+ Assert.assertEquals(expectedData.length, testData.getActualData().length);
+ Assert.assertArrayEquals(expectedData, testData.getActualData());
+ in.close();
+ if (testData.isCheckOfNoOfCalls()) {
+ Assert.assertEquals(testData.getExpectedNoNetworkCall(),
+ getMockServer().getRequestCount());
+ }
+ }
+
+ @Test
+ public void testSeekOperation() throws IOException, InterruptedException {
+ getMockServer().setDispatcher(testData.getDispatcher());
+ FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
+ Random random = new Random();
+ for (int i = 0; i < 1000; ++i) {
+ int position = random.nextInt(testData.getActualData().length);
+ in.seek(position);
+ Assert.assertEquals(position, in.getPos());
+ Assert.assertEquals(testData.getActualData()[position] & 0xFF, in.read());
+ }
+ in.close();
+ if (testData.isCheckOfNoOfCalls()) {
+ Assert.assertEquals(testData.getExpectedNoNetworkCall(),
+ getMockServer().getRequestCount());
+ }
+ }
+
+ @Test
+ public void testReadServerCalls() throws IOException, InterruptedException {
+ getMockServer().setDispatcher(testData.getDispatcher());
+ FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
+ byte[] expectedData = new byte[testData.getActualData().length];
+ in.readFully(expectedData);
+ Assert.assertArrayEquals(expectedData, testData.getActualData());
+ Assert.assertEquals(testData.getExpectedNoNetworkCall(),
+ getMockServer().getRequestCount());
+ in.close();
+ }
+
+ @Test
+ public void testReadFully() throws IOException, InterruptedException {
+ getMockServer().setDispatcher(testData.getDispatcher());
+ FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
+ byte[] expectedData = new byte[testData.getActualData().length];
+ in.readFully(expectedData);
+ Assert.assertArrayEquals(expectedData, testData.getActualData());
+
+ in.readFully(0, expectedData);
+ Assert.assertArrayEquals(expectedData, testData.getActualData());
+
+ in.readFully(0, expectedData, 0, expectedData.length);
+ Assert.assertArrayEquals(expectedData, testData.getActualData());
+ in.close();
+ }
+
+ @Test
+ public void testRandomPositionalReadUsingReadFully()
+ throws IOException, InterruptedException {
+ getMockServer().setDispatcher(testData.getDispatcher());
+ FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
+ ByteArrayInputStream actualData = new ByteArrayInputStream(
+ testData.getActualData());
+ Random random = new Random();
+ for (int i = 0; i < testData.getIntensityOfTest(); ++i) {
+ int offset = random.nextInt(testData.getActualData().length);
+ int length = testData.getActualData().length - offset;
+ byte[] expectedData = new byte[length];
+ byte[] actualDataSubset = new byte[length];
+ actualData.reset();
+ actualData.skip(offset);
+ actualData.read(actualDataSubset, 0, length);
+
+ in.readFully(offset, expectedData, 0, length);
+ Assert.assertArrayEquals(expectedData, actualDataSubset);
+ }
+
+ for (int i = 0; i < testData.getIntensityOfTest(); ++i) {
+ int offset = random.nextInt(testData.getActualData().length);
+ int length = random.nextInt(testData.getActualData().length - offset);
+ byte[] expectedData = new byte[length];
+ byte[] actualDataSubset = new byte[length];
+ actualData.reset();
+ actualData.skip(offset);
+ actualData.read(actualDataSubset, 0, length);
+
+ in.readFully(offset, expectedData, 0, length);
+ Assert.assertArrayEquals(expectedData, actualDataSubset);
+ }
+
+ in.close();
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
new file mode 100644
index 0000000..70f2a7f
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.adl;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.commons.lang.builder.EqualsBuilder;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.adl.common.CustomMockTokenProvider;
+import org.apache.hadoop.fs.adl.oauth2.AzureADTokenProvider;
+
+import com.microsoft.azure.datalake.store.oauth2.AccessTokenProvider;
+import com.microsoft.azure.datalake.store.oauth2.ClientCredsTokenProvider;
+import com.microsoft.azure.datalake.store.oauth2.RefreshTokenBasedTokenProvider;
+
+import static org.apache.hadoop.fs.adl.AdlConfKeys.AZURE_AD_CLIENT_ID_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys.AZURE_AD_CLIENT_SECRET_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys.AZURE_AD_REFRESH_TOKEN_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys.AZURE_AD_REFRESH_URL_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY;
+import static org.apache.hadoop.fs.adl.TokenProviderType.*;
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.security.ProviderUtils;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * Test appropriate token provider is loaded as per configuration.
+ */
+public class TestAzureADTokenProvider {
+
+ private static final String CLIENT_ID = "MY_CLIENT_ID";
+ private static final String REFRESH_TOKEN = "MY_REFRESH_TOKEN";
+ private static final String CLIENT_SECRET = "MY_CLIENT_SECRET";
+ private static final String REFRESH_URL = "http://localhost:8080/refresh";
+
+ @Rule
+ public final TemporaryFolder tempDir = new TemporaryFolder();
+
+ @Test
+ public void testRefreshTokenProvider()
+ throws URISyntaxException, IOException {
+ Configuration conf = new Configuration();
+ conf.set(AZURE_AD_CLIENT_ID_KEY, "MY_CLIENTID");
+ conf.set(AZURE_AD_REFRESH_TOKEN_KEY, "XYZ");
+ conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, RefreshToken);
+ conf.set(AZURE_AD_REFRESH_URL_KEY, "http://localhost:8080/refresh");
+
+ URI uri = new URI("adl://localhost:8080");
+ AdlFileSystem fileSystem = new AdlFileSystem();
+ fileSystem.initialize(uri, conf);
+ AccessTokenProvider tokenProvider = fileSystem.getTokenProvider();
+ Assert.assertTrue(tokenProvider instanceof RefreshTokenBasedTokenProvider);
+ }
+
+ @Test
+ public void testClientCredTokenProvider()
+ throws IOException, URISyntaxException {
+ Configuration conf = new Configuration();
+ conf.set(AZURE_AD_CLIENT_ID_KEY, "MY_CLIENTID");
+ conf.set(AZURE_AD_CLIENT_SECRET_KEY, "XYZ");
+ conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, ClientCredential);
+ conf.set(AZURE_AD_REFRESH_URL_KEY, "http://localhost:8080/refresh");
+
+ URI uri = new URI("adl://localhost:8080");
+ AdlFileSystem fileSystem = new AdlFileSystem();
+ fileSystem.initialize(uri, conf);
+ AccessTokenProvider tokenProvider = fileSystem.getTokenProvider();
+ Assert.assertTrue(tokenProvider instanceof ClientCredsTokenProvider);
+ }
+
+ @Test
+ public void testCustomCredTokenProvider()
+ throws URISyntaxException, IOException {
+ Configuration conf = new Configuration();
+ conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
+ CustomMockTokenProvider.class, AzureADTokenProvider.class);
+
+ URI uri = new URI("adl://localhost:8080");
+ AdlFileSystem fileSystem = new AdlFileSystem();
+ fileSystem.initialize(uri, conf);
+ AccessTokenProvider tokenProvider = fileSystem.getTokenProvider();
+ Assert.assertTrue(tokenProvider instanceof SdkTokenProviderAdapter);
+ }
+
+ @Test
+ public void testInvalidProviderConfigurationForType()
+ throws URISyntaxException, IOException {
+ Configuration conf = new Configuration();
+ URI uri = new URI("adl://localhost:8080");
+ AdlFileSystem fileSystem = new AdlFileSystem();
+ try {
+ fileSystem.initialize(uri, conf);
+ Assert.fail("Initialization should have failed due no token provider "
+ + "configuration");
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(
+ e.getMessage().contains("dfs.adls.oauth2.access.token.provider"));
+ }
+ conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
+ CustomMockTokenProvider.class, AzureADTokenProvider.class);
+ fileSystem.initialize(uri, conf);
+ }
+
+ @Test
+ public void testInvalidProviderConfigurationForClassPath()
+ throws URISyntaxException, IOException {
+ Configuration conf = new Configuration();
+ URI uri = new URI("adl://localhost:8080");
+ AdlFileSystem fileSystem = new AdlFileSystem();
+ conf.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
+ "wrong.classpath.CustomMockTokenProvider");
+ try {
+ fileSystem.initialize(uri, conf);
+ Assert.fail("Initialization should have failed due invalid provider "
+ + "configuration");
+ } catch (RuntimeException e) {
+ Assert.assertTrue(
+ e.getMessage().contains("wrong.classpath.CustomMockTokenProvider"));
+ }
+ }
+
+ private CredentialProvider createTempCredProvider(Configuration conf)
+ throws URISyntaxException, IOException {
+ final File file = tempDir.newFile("test.jks");
+ final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
+ file.toURI());
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ jks.toString());
+ return CredentialProviderFactory.getProviders(conf).get(0);
+ }
+
+ @Test
+ public void testRefreshTokenWithCredentialProvider()
+ throws IOException, URISyntaxException {
+ Configuration conf = new Configuration();
+ conf.set(AZURE_AD_CLIENT_ID_KEY, "DUMMY");
+ conf.set(AZURE_AD_REFRESH_TOKEN_KEY, "DUMMY");
+ conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, RefreshToken);
+
+ CredentialProvider provider = createTempCredProvider(conf);
+ provider.createCredentialEntry(AZURE_AD_CLIENT_ID_KEY,
+ CLIENT_ID.toCharArray());
+ provider.createCredentialEntry(AZURE_AD_REFRESH_TOKEN_KEY,
+ REFRESH_TOKEN.toCharArray());
+ provider.flush();
+
+ URI uri = new URI("adl://localhost:8080");
+ AdlFileSystem fileSystem = new AdlFileSystem();
+ fileSystem.initialize(uri, conf);
+ RefreshTokenBasedTokenProvider expected =
+ new RefreshTokenBasedTokenProvider(CLIENT_ID, REFRESH_TOKEN);
+ Assert.assertTrue(EqualsBuilder.reflectionEquals(expected,
+ fileSystem.getTokenProvider()));
+ }
+
+ @Test
+ public void testRefreshTokenWithCredentialProviderFallback()
+ throws IOException, URISyntaxException {
+ Configuration conf = new Configuration();
+ conf.set(AZURE_AD_CLIENT_ID_KEY, CLIENT_ID);
+ conf.set(AZURE_AD_REFRESH_TOKEN_KEY, REFRESH_TOKEN);
+ conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, RefreshToken);
+
+ createTempCredProvider(conf);
+
+ URI uri = new URI("adl://localhost:8080");
+ AdlFileSystem fileSystem = new AdlFileSystem();
+ fileSystem.initialize(uri, conf);
+ RefreshTokenBasedTokenProvider expected =
+ new RefreshTokenBasedTokenProvider(CLIENT_ID, REFRESH_TOKEN);
+ Assert.assertTrue(EqualsBuilder.reflectionEquals(expected,
+ fileSystem.getTokenProvider()));
+ }
+
+ @Test
+ public void testClientCredWithCredentialProvider()
+ throws IOException, URISyntaxException {
+ Configuration conf = new Configuration();
+ conf.set(AZURE_AD_CLIENT_ID_KEY, "DUMMY");
+ conf.set(AZURE_AD_CLIENT_SECRET_KEY, "DUMMY");
+ conf.set(AZURE_AD_REFRESH_URL_KEY, "DUMMY");
+ conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, ClientCredential);
+
+ CredentialProvider provider = createTempCredProvider(conf);
+ provider.createCredentialEntry(AZURE_AD_CLIENT_ID_KEY,
+ CLIENT_ID.toCharArray());
+ provider.createCredentialEntry(AZURE_AD_CLIENT_SECRET_KEY,
+ CLIENT_SECRET.toCharArray());
+ provider.createCredentialEntry(AZURE_AD_REFRESH_URL_KEY,
+ REFRESH_URL.toCharArray());
+ provider.flush();
+
+ URI uri = new URI("adl://localhost:8080");
+ AdlFileSystem fileSystem = new AdlFileSystem();
+ fileSystem.initialize(uri, conf);
+ ClientCredsTokenProvider expected = new ClientCredsTokenProvider(
+ REFRESH_URL, CLIENT_ID, CLIENT_SECRET);
+ Assert.assertTrue(EqualsBuilder.reflectionEquals(expected,
+ fileSystem.getTokenProvider()));
+ }
+
+ @Test
+ public void testClientCredWithCredentialProviderFallback()
+ throws IOException, URISyntaxException {
+ Configuration conf = new Configuration();
+ conf.set(AZURE_AD_CLIENT_ID_KEY, CLIENT_ID);
+ conf.set(AZURE_AD_CLIENT_SECRET_KEY, CLIENT_SECRET);
+ conf.set(AZURE_AD_REFRESH_URL_KEY, REFRESH_URL);
+ conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, ClientCredential);
+
+ createTempCredProvider(conf);
+
+ URI uri = new URI("adl://localhost:8080");
+ AdlFileSystem fileSystem = new AdlFileSystem();
+ fileSystem.initialize(uri, conf);
+ ClientCredsTokenProvider expected = new ClientCredsTokenProvider(
+ REFRESH_URL, CLIENT_ID, CLIENT_SECRET);
+ Assert.assertTrue(EqualsBuilder.reflectionEquals(expected,
+ fileSystem.getTokenProvider()));
+ }
+
+ @Test
+ public void testCredentialProviderPathExclusions() throws Exception {
+ String providerPath =
+ "user:///,jceks://adl/user/hrt_qa/sqoopdbpasswd.jceks," +
+ "jceks://hdfs@nn1.example.com/my/path/test.jceks";
+ Configuration config = new Configuration();
+ config.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ providerPath);
+ String newPath =
+ "user:///,jceks://hdfs@nn1.example.com/my/path/test.jceks";
+
+ excludeAndTestExpectations(config, newPath);
+ }
+
+ @Test
+ public void testExcludeAllProviderTypesFromConfig() throws Exception {
+ String providerPath =
+ "jceks://adl/tmp/test.jceks," +
+ "jceks://adl@/my/path/test.jceks";
+ Configuration config = new Configuration();
+ config.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ providerPath);
+ String newPath = null;
+
+ excludeAndTestExpectations(config, newPath);
+ }
+
+ void excludeAndTestExpectations(Configuration config, String newPath)
+ throws Exception {
+ Configuration conf = ProviderUtils.excludeIncompatibleCredentialProviders(
+ config, AdlFileSystem.class);
+ String effectivePath = conf.get(
+ CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, null);
+ assertEquals(newPath, effectivePath);
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestConcurrentDataReadOperations.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestConcurrentDataReadOperations.java
new file mode 100644
index 0000000..b790562
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestConcurrentDataReadOperations.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.adl;
+
+import com.squareup.okhttp.mockwebserver.Dispatcher;
+import com.squareup.okhttp.mockwebserver.MockResponse;
+import com.squareup.okhttp.mockwebserver.RecordedRequest;
+import okio.Buffer;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * This class is responsible for testing multiple threads trying to access same
+ * or multiple files from the offset.
+ */
+@RunWith(Parameterized.class)
+public class TestConcurrentDataReadOperations extends AdlMockWebServer {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestConcurrentDataReadOperations.class);
+ private static final Object LOCK = new Object();
+ private static FSDataInputStream commonHandle = null;
+ private int concurrencyLevel;
+
+ public TestConcurrentDataReadOperations(int concurrencyLevel) {
+ this.concurrencyLevel = concurrencyLevel;
+ }
+
+ @Parameterized.Parameters(name = "{index}")
+ public static Collection> testDataNumberOfConcurrentRun() {
+ return Arrays.asList(new Object[][] {{1}, {2}, {3}, {4}, {5}});
+ }
+
+ public static byte[] getRandomByteArrayData(int size) {
+ byte[] b = new byte[size];
+ Random rand = new Random();
+ rand.nextBytes(b);
+ return b;
+ }
+
+ private void setDispatcher(final ArrayList testData) {
+ getMockServer().setDispatcher(new Dispatcher() {
+ @Override
+ public MockResponse dispatch(RecordedRequest recordedRequest)
+ throws InterruptedException {
+ CreateTestData currentRequest = null;
+ for (CreateTestData local : testData) {
+ if (recordedRequest.getPath().contains(local.path.toString())) {
+ currentRequest = local;
+ break;
+ }
+ }
+
+ if (currentRequest == null) {
+ new MockResponse().setBody("Request data not found")
+ .setResponseCode(501);
+ }
+
+ if (recordedRequest.getRequestLine().contains("op=GETFILESTATUS")) {
+ return new MockResponse().setResponseCode(200).setBody(
+ TestADLResponseData
+ .getGetFileStatusJSONResponse(currentRequest.data.length));
+ }
+
+ if (recordedRequest.getRequestLine().contains("op=OPEN")) {
+ String request = recordedRequest.getRequestLine();
+ int offset = 0;
+ int byteCount = 0;
+
+ Pattern pattern = Pattern.compile("offset=([0-9]+)");
+ Matcher matcher = pattern.matcher(request);
+ if (matcher.find()) {
+ LOG.debug(matcher.group(1));
+ offset = Integer.parseInt(matcher.group(1));
+ }
+
+ pattern = Pattern.compile("length=([0-9]+)");
+ matcher = pattern.matcher(request);
+ if (matcher.find()) {
+ LOG.debug(matcher.group(1));
+ byteCount = Integer.parseInt(matcher.group(1));
+ }
+
+ Buffer buf = new Buffer();
+ buf.write(currentRequest.data, offset,
+ Math.min(currentRequest.data.length - offset, byteCount));
+ return new MockResponse().setResponseCode(200)
+ .setChunkedBody(buf, 4 * 1024 * 1024);
+ }
+
+ return new MockResponse().setBody("NOT SUPPORTED").setResponseCode(501);
+ }
+ });
+ }
+
+ @Before
+ public void resetHandle() {
+ commonHandle = null;
+ }
+
+ @Test
+ public void testParallelReadOnDifferentStreams()
+ throws IOException, InterruptedException, ExecutionException {
+
+ ArrayList createTestData = new ArrayList();
+
+ Random random = new Random();
+
+ for (int i = 0; i < concurrencyLevel; i++) {
+ CreateTestData testData = new CreateTestData();
+ testData
+ .set(new Path("/test/concurrentRead/" + UUID.randomUUID().toString()),
+ getRandomByteArrayData(random.nextInt(1 * 1024 * 1024)));
+ createTestData.add(testData);
+ }
+
+ setDispatcher(createTestData);
+
+ ArrayList readTestData = new ArrayList();
+ for (CreateTestData local : createTestData) {
+ ReadTestData localReadData = new ReadTestData();
+ localReadData.set(local.path, local.data, 0);
+ readTestData.add(localReadData);
+ }
+
+ runReadTest(readTestData, false);
+ }
+
+ @Test
+ public void testParallelReadOnSameStreams()
+ throws IOException, InterruptedException, ExecutionException {
+ ArrayList createTestData = new ArrayList();
+
+ Random random = new Random();
+
+ for (int i = 0; i < 1; i++) {
+ CreateTestData testData = new CreateTestData();
+ testData
+ .set(new Path("/test/concurrentRead/" + UUID.randomUUID().toString()),
+ getRandomByteArrayData(1024 * 1024));
+ createTestData.add(testData);
+ }
+
+ setDispatcher(createTestData);
+
+ ArrayList readTestData = new ArrayList();
+ ByteArrayInputStream buffered = new ByteArrayInputStream(
+ createTestData.get(0).data);
+
+ ReadTestData readInitially = new ReadTestData();
+ byte[] initialData = new byte[1024 * 1024];
+ buffered.read(initialData);
+
+ readInitially.set(createTestData.get(0).path, initialData, 0);
+ readTestData.add(readInitially);
+ runReadTest(readTestData, false);
+
+ readTestData.clear();
+
+ for (int i = 0; i < concurrencyLevel * 5; i++) {
+ ReadTestData localReadData = new ReadTestData();
+ int offset = random.nextInt((1024 * 1024) - 1);
+ int length = 1024 * 1024 - offset;
+ byte[] expectedData = new byte[length];
+ buffered.reset();
+ buffered.skip(offset);
+ buffered.read(expectedData);
+ localReadData.set(createTestData.get(0).path, expectedData, offset);
+ readTestData.add(localReadData);
+ }
+
+ runReadTest(readTestData, true);
+ }
+
+ void runReadTest(ArrayList testData, boolean useSameStream)
+ throws InterruptedException, ExecutionException {
+
+ ExecutorService executor = Executors.newFixedThreadPool(testData.size());
+ Future[] subtasks = new Future[testData.size()];
+
+ for (int i = 0; i < testData.size(); i++) {
+ subtasks[i] = executor.submit(
+ new ReadConcurrentRunnable(testData.get(i).data, testData.get(i).path,
+ testData.get(i).offset, useSameStream));
+ }
+
+ executor.shutdown();
+
+ // wait until all tasks are finished
+ executor.awaitTermination(120, TimeUnit.SECONDS);
+
+ for (int i = 0; i < testData.size(); ++i) {
+ Assert.assertTrue((Boolean) subtasks[i].get());
+ }
+ }
+
+ class ReadTestData {
+ private Path path;
+ private byte[] data;
+ private int offset;
+
+ public void set(Path filePath, byte[] dataToBeRead, int fromOffset) {
+ this.path = filePath;
+ this.data = dataToBeRead;
+ this.offset = fromOffset;
+ }
+ }
+
+ class CreateTestData {
+ private Path path;
+ private byte[] data;
+
+ public void set(Path filePath, byte[] dataToBeWritten) {
+ this.path = filePath;
+ this.data = dataToBeWritten;
+ }
+ }
+
+ class ReadConcurrentRunnable implements Callable {
+ private Path path;
+ private int offset;
+ private byte[] expectedData;
+ private boolean useSameStream;
+
+ public ReadConcurrentRunnable(byte[] expectedData, Path path, int offset,
+ boolean useSameStream) {
+ this.path = path;
+ this.offset = offset;
+ this.expectedData = expectedData;
+ this.useSameStream = useSameStream;
+ }
+
+ public Boolean call() throws IOException {
+ try {
+ FSDataInputStream in;
+ if (useSameStream) {
+ synchronized (LOCK) {
+ if (commonHandle == null) {
+ commonHandle = getMockAdlFileSystem().open(path);
+ }
+ in = commonHandle;
+ }
+ } else {
+ in = getMockAdlFileSystem().open(path);
+ }
+
+ byte[] actualData = new byte[expectedData.length];
+ in.readFully(offset, actualData);
+ Assert.assertArrayEquals("Path :" + path.toString() + " did not match.",
+ expectedData, actualData);
+ if (!useSameStream) {
+ in.close();
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ return false;
+ }
+ return true;
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java
new file mode 100644
index 0000000..c594c65
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.adl;
+
+import com.squareup.okhttp.mockwebserver.MockResponse;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.adl.common.CustomMockTokenProvider;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.Collection;
+
+import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY;
+
+/**
+ * Test access token provider behaviour with custom token provider and for token
+ * provider cache is enabled.
+ */
+@RunWith(Parameterized.class)
+public class TestCustomTokenProvider extends AdlMockWebServer {
+ private static final long TEN_MINUTES_IN_MILIS = 600000;
+ private int backendCallCount;
+ private int expectedCallbackToAccessToken;
+ private TestableAdlFileSystem[] fileSystems;
+ private Class typeOfTokenProviderClass;
+ private long expiryFromNow;
+ private int fsObjectCount;
+
+ public TestCustomTokenProvider(Class typeOfTokenProviderClass,
+ long expiryFromNow, int fsObjectCount, int backendCallCount,
+ int expectedCallbackToAccessToken)
+ throws IllegalAccessException, InstantiationException, URISyntaxException,
+ IOException {
+ this.typeOfTokenProviderClass = typeOfTokenProviderClass;
+ this.expiryFromNow = expiryFromNow;
+ this.fsObjectCount = fsObjectCount;
+ this.backendCallCount = backendCallCount;
+ this.expectedCallbackToAccessToken = expectedCallbackToAccessToken;
+ }
+
+ @Parameterized.Parameters(name = "{index}")
+ public static Collection testDataForTokenProvider() {
+ return Arrays.asList(new Object[][] {
+ // Data set in order
+ // INPUT - CustomTokenProvider class to load
+ // INPUT - expiry time in milis. Subtract from current time
+ // INPUT - No. of FileSystem object
+ // INPUT - No. of backend calls per FileSystem object
+ // EXPECTED - Number of callbacks to get token after test finished.
+ {CustomMockTokenProvider.class, 0, 1, 1, 1},
+ {CustomMockTokenProvider.class, TEN_MINUTES_IN_MILIS, 1, 1, 1},
+ {CustomMockTokenProvider.class, TEN_MINUTES_IN_MILIS, 2, 1, 2},
+ {CustomMockTokenProvider.class, TEN_MINUTES_IN_MILIS, 10, 10, 10}});
+ }
+
+ /**
+ * Explicitly invoked init so that base class mock server is setup before
+ * test data initialization is done.
+ *
+ * @throws IOException
+ * @throws URISyntaxException
+ */
+ public void init() throws IOException, URISyntaxException {
+ Configuration configuration = new Configuration();
+ configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
+ typeOfTokenProviderClass.getName());
+ fileSystems = new TestableAdlFileSystem[fsObjectCount];
+ URI uri = new URI("adl://localhost:" + getPort());
+
+ for (int i = 0; i < fsObjectCount; ++i) {
+ fileSystems[i] = new TestableAdlFileSystem();
+ fileSystems[i].initialize(uri, configuration);
+
+ ((CustomMockTokenProvider) fileSystems[i].getAzureTokenProvider())
+ .setExpiryTimeInMillisAfter(expiryFromNow);
+ }
+ }
+
+ @Test
+ public void testCustomTokenManagement()
+ throws IOException, URISyntaxException {
+ int accessTokenCallbackDuringExec = 0;
+ init();
+ for (TestableAdlFileSystem tfs : fileSystems) {
+ for (int i = 0; i < backendCallCount; ++i) {
+ getMockServer().enqueue(new MockResponse().setResponseCode(200)
+ .setBody(TestADLResponseData.getGetFileStatusJSONResponse()));
+ FileStatus fileStatus = tfs.getFileStatus(new Path("/test1/test2"));
+ Assert.assertTrue(fileStatus.isFile());
+ Assert.assertEquals("adl://" + getMockServer().getHostName() + ":" +
+ getMockServer().getPort() + "/test1/test2",
+ fileStatus.getPath().toString());
+ Assert.assertEquals(4194304, fileStatus.getLen());
+ Assert.assertEquals(ADL_BLOCK_SIZE, fileStatus.getBlockSize());
+ Assert.assertEquals(1, fileStatus.getReplication());
+ Assert
+ .assertEquals(new FsPermission("777"), fileStatus.getPermission());
+ Assert.assertEquals("NotSupportYet", fileStatus.getOwner());
+ Assert.assertEquals("NotSupportYet", fileStatus.getGroup());
+ }
+
+ accessTokenCallbackDuringExec += ((CustomMockTokenProvider) tfs
+ .getAzureTokenProvider()).getAccessTokenRequestCount();
+ }
+
+ Assert.assertEquals(expectedCallbackToAccessToken,
+ accessTokenCallbackDuringExec);
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java
new file mode 100644
index 0000000..78ef931
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl;
+
+import com.squareup.okhttp.mockwebserver.MockResponse;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Time;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE;
+
+/**
+ * This class is responsible for testing local getFileStatus implementation
+ * to cover correct parsing of successful and error JSON response
+ * from the server.
+ * Adls GetFileStatus operation is in detail covered in
+ * org.apache.hadoop.fs.adl.live testing package.
+ */
+public class TestGetFileStatus extends AdlMockWebServer {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestGetFileStatus.class);
+
+ @Test
+ public void getFileStatusReturnsAsExpected()
+ throws URISyntaxException, IOException {
+ getMockServer().enqueue(new MockResponse().setResponseCode(200)
+ .setBody(TestADLResponseData.getGetFileStatusJSONResponse()));
+ long startTime = Time.monotonicNow();
+ FileStatus fileStatus = getMockAdlFileSystem()
+ .getFileStatus(new Path("/test1/test2"));
+ long endTime = Time.monotonicNow();
+ LOG.debug("Time : " + (endTime - startTime));
+ Assert.assertTrue(fileStatus.isFile());
+ Assert.assertEquals("adl://" + getMockServer().getHostName() + ":" +
+ getMockServer().getPort() + "/test1/test2",
+ fileStatus.getPath().toString());
+ Assert.assertEquals(4194304, fileStatus.getLen());
+ Assert.assertEquals(ADL_BLOCK_SIZE, fileStatus.getBlockSize());
+ Assert.assertEquals(1, fileStatus.getReplication());
+ Assert.assertEquals(new FsPermission("777"), fileStatus.getPermission());
+ Assert.assertEquals("NotSupportYet", fileStatus.getOwner());
+ Assert.assertEquals("NotSupportYet", fileStatus.getGroup());
+ }
+
+ @Test
+ public void getFileStatusAclBit()
+ throws URISyntaxException, IOException {
+ // With ACLBIT set to true
+ getMockServer().enqueue(new MockResponse().setResponseCode(200)
+ .setBody(TestADLResponseData.getGetFileStatusJSONResponse(true)));
+ long startTime = Time.monotonicNow();
+ FileStatus fileStatus = getMockAdlFileSystem()
+ .getFileStatus(new Path("/test1/test2"));
+ long endTime = Time.monotonicNow();
+ LOG.debug("Time : " + (endTime - startTime));
+ Assert.assertTrue(fileStatus.isFile());
+ Assert.assertEquals(true, fileStatus.getPermission().getAclBit());
+
+ // With ACLBIT set to false
+ getMockServer().enqueue(new MockResponse().setResponseCode(200)
+ .setBody(TestADLResponseData.getGetFileStatusJSONResponse(false)));
+ startTime = Time.monotonicNow();
+ fileStatus = getMockAdlFileSystem()
+ .getFileStatus(new Path("/test1/test2"));
+ endTime = Time.monotonicNow();
+ LOG.debug("Time : " + (endTime - startTime));
+ Assert.assertTrue(fileStatus.isFile());
+ Assert.assertEquals(false, fileStatus.getPermission().getAclBit());
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java
new file mode 100644
index 0000000..dac8886
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl;
+
+import com.squareup.okhttp.mockwebserver.MockResponse;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.Time;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+/**
+ * This class is responsible for testing local listStatus implementation to
+ * cover correct parsing of successful and error JSON response from the server.
+ * Adls ListStatus functionality is in detail covered in
+ * org.apache.hadoop.fs.adl.live testing package.
+ */
+public class TestListStatus extends AdlMockWebServer {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestListStatus.class);
+
+ @Test
+ public void listStatusReturnsAsExpected() throws IOException {
+ getMockServer().enqueue(new MockResponse().setResponseCode(200)
+ .setBody(TestADLResponseData.getListFileStatusJSONResponse(10)));
+ long startTime = Time.monotonicNow();
+ FileStatus[] ls = getMockAdlFileSystem()
+ .listStatus(new Path("/test1/test2"));
+ long endTime = Time.monotonicNow();
+ LOG.debug("Time : " + (endTime - startTime));
+ Assert.assertEquals(10, ls.length);
+
+ getMockServer().enqueue(new MockResponse().setResponseCode(200)
+ .setBody(TestADLResponseData.getListFileStatusJSONResponse(200)));
+ startTime = Time.monotonicNow();
+ ls = getMockAdlFileSystem().listStatus(new Path("/test1/test2"));
+ endTime = Time.monotonicNow();
+ LOG.debug("Time : " + (endTime - startTime));
+ Assert.assertEquals(200, ls.length);
+
+ getMockServer().enqueue(new MockResponse().setResponseCode(200)
+ .setBody(TestADLResponseData.getListFileStatusJSONResponse(2048)));
+ startTime = Time.monotonicNow();
+ ls = getMockAdlFileSystem().listStatus(new Path("/test1/test2"));
+ endTime = Time.monotonicNow();
+ LOG.debug("Time : " + (endTime - startTime));
+ Assert.assertEquals(2048, ls.length);
+ }
+
+ @Test
+ public void listStatusOnFailure() throws IOException {
+ getMockServer().enqueue(new MockResponse().setResponseCode(403).setBody(
+ TestADLResponseData.getErrorIllegalArgumentExceptionJSONResponse()));
+ FileStatus[] ls = null;
+ long startTime = Time.monotonicNow();
+ try {
+ ls = getMockAdlFileSystem().listStatus(new Path("/test1/test2"));
+ } catch (IOException e) {
+ Assert.assertTrue(e.getMessage().contains("Invalid"));
+ }
+ long endTime = Time.monotonicNow();
+ LOG.debug("Time : " + (endTime - startTime));
+
+ // SDK may increase number of retry attempts before error is propagated
+ // to caller. Adding max 10 error responses in the queue to align with SDK.
+ for (int i = 0; i < 10; ++i) {
+ getMockServer().enqueue(new MockResponse().setResponseCode(500).setBody(
+ TestADLResponseData.getErrorInternalServerExceptionJSONResponse()));
+ }
+
+ startTime = Time.monotonicNow();
+ try {
+ ls = getMockAdlFileSystem().listStatus(new Path("/test1/test2"));
+ } catch (IOException e) {
+ Assert.assertTrue(e.getMessage().contains("Internal Server Error"));
+ }
+ endTime = Time.monotonicNow();
+ LOG.debug("Time : " + (endTime - startTime));
+ }
+
+ @Test
+ public void listStatusAclBit()
+ throws URISyntaxException, IOException {
+ // With ACLBIT set to true
+ getMockServer().enqueue(new MockResponse().setResponseCode(200)
+ .setBody(TestADLResponseData.getListFileStatusJSONResponse(true)));
+ FileStatus[] ls = null;
+ long startTime = Time.monotonicNow();
+ ls = getMockAdlFileSystem()
+ .listStatus(new Path("/test1/test2"));
+ long endTime = Time.monotonicNow();
+ LOG.debug("Time : " + (endTime - startTime));
+ for (int i = 0; i < ls.length; i++) {
+ Assert.assertTrue(ls[i].isDirectory());
+ Assert.assertEquals(true, ls[i].getPermission().getAclBit());
+ }
+
+ // With ACLBIT set to false
+ ls = null;
+ getMockServer().enqueue(new MockResponse().setResponseCode(200)
+ .setBody(TestADLResponseData.getListFileStatusJSONResponse(false)));
+ startTime = Time.monotonicNow();
+ ls = getMockAdlFileSystem()
+ .listStatus(new Path("/test1/test2"));
+ endTime = Time.monotonicNow();
+ LOG.debug("Time : " + (endTime - startTime));
+ for (int i = 0; i < ls.length; i++) {
+ Assert.assertTrue(ls[i].isDirectory());
+ Assert.assertEquals(false, ls[i].getPermission().getAclBit());
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java
new file mode 100644
index 0000000..908f8b8
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.adl;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY;
+
+/**
+ * This class verifies path conversion to SDK.
+ */
+public class TestRelativePathFormation {
+
+ @Test
+ public void testToRelativePath() throws URISyntaxException, IOException {
+ AdlFileSystem fs = new AdlFileSystem();
+ Configuration configuration = new Configuration();
+ configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
+ "org.apache.hadoop.fs.adl.common.CustomMockTokenProvider");
+
+ fs.initialize(new URI("adl://temp.account.net"), configuration);
+
+ Assert.assertEquals("/usr", fs.toRelativeFilePath(new Path("/usr")));
+ Assert.assertEquals("/usr",
+ fs.toRelativeFilePath(new Path("adl://temp.account.net/usr")));
+
+ // When working directory is set.
+ fs.setWorkingDirectory(new Path("/a/b/"));
+ Assert.assertEquals("/usr", fs.toRelativeFilePath(new Path("/usr")));
+ Assert.assertEquals("/a/b/usr", fs.toRelativeFilePath(new Path("usr")));
+ Assert.assertEquals("/usr",
+ fs.toRelativeFilePath(new Path("adl://temp.account.net/usr")));
+ Assert.assertEquals("/usr",
+ fs.toRelativeFilePath(new Path("wasb://temp.account.net/usr")));
+ }
+
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
new file mode 100644
index 0000000..4cabaa3
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.adl;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .ADL_DEBUG_OVERRIDE_LOCAL_USER_AS_OWNER;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .ADL_DEBUG_SET_LOCAL_USER_AS_OWNER_DEFAULT;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .ADL_ENABLEUPN_FOR_OWNERGROUP_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .ADL_EXPERIMENT_POSITIONAL_READ_DEFAULT;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .ADL_EXPERIMENT_POSITIONAL_READ_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_REPLICATION_FACTOR;
+import static org.apache.hadoop.fs.adl.AdlConfKeys.AZURE_AD_CLIENT_ID_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys.AZURE_AD_CLIENT_SECRET_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys.AZURE_AD_REFRESH_TOKEN_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys.AZURE_AD_REFRESH_URL_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .DEFAULT_READ_AHEAD_BUFFER_SIZE;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .DEFAULT_WRITE_AHEAD_BUFFER_SIZE;
+import static org.apache.hadoop.fs.adl.AdlConfKeys.LATENCY_TRACKER_DEFAULT;
+import static org.apache.hadoop.fs.adl.AdlConfKeys.LATENCY_TRACKER_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys.READ_AHEAD_BUFFER_SIZE_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .TOKEN_PROVIDER_TYPE_CLIENT_CRED;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+ .TOKEN_PROVIDER_TYPE_REFRESH_TOKEN;
+import static org.apache.hadoop.fs.adl.AdlConfKeys.WRITE_BUFFER_SIZE_KEY;
+
+/**
+ * Validate configuration keys defined for adl storage file system instance.
+ */
+public class TestValidateConfiguration {
+
+ @Test
+ public void validateConfigurationKeys() {
+ Assert
+ .assertEquals("dfs.adls.oauth2.refresh.url", AZURE_AD_REFRESH_URL_KEY);
+ Assert.assertEquals("dfs.adls.oauth2.access.token.provider",
+ AZURE_AD_TOKEN_PROVIDER_CLASS_KEY);
+ Assert.assertEquals("dfs.adls.oauth2.client.id", AZURE_AD_CLIENT_ID_KEY);
+ Assert.assertEquals("dfs.adls.oauth2.refresh.token",
+ AZURE_AD_REFRESH_TOKEN_KEY);
+ Assert
+ .assertEquals("dfs.adls.oauth2.credential", AZURE_AD_CLIENT_SECRET_KEY);
+ Assert.assertEquals("adl.debug.override.localuserasfileowner",
+ ADL_DEBUG_OVERRIDE_LOCAL_USER_AS_OWNER);
+
+ Assert.assertEquals("dfs.adls.oauth2.access.token.provider.type",
+ AZURE_AD_TOKEN_PROVIDER_TYPE_KEY);
+
+ Assert.assertEquals("adl.feature.client.cache.readahead",
+ READ_AHEAD_BUFFER_SIZE_KEY);
+
+ Assert.assertEquals("adl.feature.client.cache.drop.behind.writes",
+ WRITE_BUFFER_SIZE_KEY);
+
+ Assert.assertEquals("RefreshToken", TOKEN_PROVIDER_TYPE_REFRESH_TOKEN);
+
+ Assert.assertEquals("ClientCredential", TOKEN_PROVIDER_TYPE_CLIENT_CRED);
+
+ Assert.assertEquals("adl.dfs.enable.client.latency.tracker",
+ LATENCY_TRACKER_KEY);
+
+ Assert.assertEquals(true, LATENCY_TRACKER_DEFAULT);
+
+ Assert.assertEquals(true, ADL_EXPERIMENT_POSITIONAL_READ_DEFAULT);
+
+ Assert.assertEquals("adl.feature.experiment.positional.read.enable",
+ ADL_EXPERIMENT_POSITIONAL_READ_KEY);
+
+ Assert.assertEquals(1, ADL_REPLICATION_FACTOR);
+ Assert.assertEquals(256 * 1024 * 1024, ADL_BLOCK_SIZE);
+ Assert.assertEquals(false, ADL_DEBUG_SET_LOCAL_USER_AS_OWNER_DEFAULT);
+ Assert.assertEquals(4 * 1024 * 1024, DEFAULT_READ_AHEAD_BUFFER_SIZE);
+ Assert.assertEquals(4 * 1024 * 1024, DEFAULT_WRITE_AHEAD_BUFFER_SIZE);
+
+ Assert.assertEquals("adl.feature.ownerandgroup.enableupn",
+ ADL_ENABLEUPN_FOR_OWNERGROUP_KEY);
+ Assert.assertEquals(false,
+ ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT);
+ }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/dtp/DtpHttp2Handler.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestableAdlFileSystem.java
similarity index 65%
rename from hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/dtp/DtpHttp2Handler.java
rename to hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestableAdlFileSystem.java
index 5b6f279..4acb39b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/dtp/DtpHttp2Handler.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestableAdlFileSystem.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -14,21 +14,17 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
+ *
*/
-package org.apache.hadoop.hdfs.server.datanode.web.dtp;
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import io.netty.handler.codec.http2.Http2ConnectionHandler;
+package org.apache.hadoop.fs.adl;
/**
- * The HTTP/2 handler.
+ * Mock adl file storage subclass to mock adl storage on local http service.
*/
-@InterfaceAudience.Private
-public class DtpHttp2Handler extends Http2ConnectionHandler {
-
- public DtpHttp2Handler() {
- super(true, new DtpHttp2FrameListener());
- ((DtpHttp2FrameListener) decoder().listener()).encoder(encoder());
+public class TestableAdlFileSystem extends AdlFileSystem {
+ @Override
+ protected String getTransportScheme() {
+ return "http";
}
}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/CustomMockTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/CustomMockTokenProvider.java
new file mode 100644
index 0000000..c48ca0e
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/CustomMockTokenProvider.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.adl.common;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.adl.oauth2.AzureADTokenProvider;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.Random;
+
+/**
+ * Custom token management without cache enabled.
+ */
+public class CustomMockTokenProvider extends AzureADTokenProvider {
+ private Random random;
+ private long expiryTime;
+ private int accessTokenRequestCount = 0;
+
+ @Override
+ public void initialize(Configuration configuration) throws IOException {
+ random = new Random();
+ }
+
+ @Override
+ public String getAccessToken() throws IOException {
+ accessTokenRequestCount++;
+ return String.valueOf(random.nextInt());
+ }
+
+ @Override
+ public Date getExpiryTime() {
+ Date before10Min = new Date();
+ before10Min.setTime(expiryTime);
+ return before10Min;
+ }
+
+ public void setExpiryTimeInMillisAfter(long timeInMillis) {
+ expiryTime = System.currentTimeMillis() + timeInMillis;
+ }
+
+ public int getAccessTokenRequestCount() {
+ return accessTokenRequestCount;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/ExpectedResponse.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/ExpectedResponse.java
new file mode 100644
index 0000000..dc8577d
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/ExpectedResponse.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.common;
+
+import com.squareup.okhttp.mockwebserver.MockResponse;
+
+import java.util.ArrayList;
+
+/**
+ * Supporting class to hold expected MockResponse object along with parameters
+ * for validation in test methods.
+ */
+public class ExpectedResponse {
+ private MockResponse response;
+ private ArrayList expectedQueryParameters = new ArrayList();
+ private int expectedBodySize;
+ private String httpRequestType;
+
+ public int getExpectedBodySize() {
+ return expectedBodySize;
+ }
+
+ public String getHttpRequestType() {
+ return httpRequestType;
+ }
+
+ public ArrayList getExpectedQueryParameters() {
+ return expectedQueryParameters;
+ }
+
+ public MockResponse getResponse() {
+ return response;
+ }
+
+ ExpectedResponse set(MockResponse mockResponse) {
+ this.response = mockResponse;
+ return this;
+ }
+
+ ExpectedResponse addExpectedQueryParam(String param) {
+ expectedQueryParameters.add(param);
+ return this;
+ }
+
+ ExpectedResponse addExpectedBodySize(int bodySize) {
+ this.expectedBodySize = bodySize;
+ return this;
+ }
+
+ ExpectedResponse addExpectedHttpRequestType(String expectedHttpRequestType) {
+ this.httpRequestType = expectedHttpRequestType;
+ return this;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/Parallelized.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/Parallelized.java
new file mode 100644
index 0000000..b08a892
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/Parallelized.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.common;
+
+import org.junit.runners.Parameterized;
+import org.junit.runners.model.RunnerScheduler;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Provided for convenience to execute parametrized test cases concurrently.
+ */
+public class Parallelized extends Parameterized {
+
+ public Parallelized(Class classObj) throws Throwable {
+ super(classObj);
+ setScheduler(new ThreadPoolScheduler());
+ }
+
+ private static class ThreadPoolScheduler implements RunnerScheduler {
+ private ExecutorService executor;
+
+ public ThreadPoolScheduler() {
+ int numThreads = 10;
+ executor = Executors.newFixedThreadPool(numThreads);
+ }
+
+ public void finished() {
+ executor.shutdown();
+ try {
+ executor.awaitTermination(10, TimeUnit.MINUTES);
+ } catch (InterruptedException exc) {
+ throw new RuntimeException(exc);
+ }
+ }
+
+ public void schedule(Runnable childStatement) {
+ executor.submit(childStatement);
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/TestDataForRead.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/TestDataForRead.java
new file mode 100644
index 0000000..509b3f0
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/TestDataForRead.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.common;
+
+import com.squareup.okhttp.mockwebserver.Dispatcher;
+import com.squareup.okhttp.mockwebserver.MockResponse;
+import com.squareup.okhttp.mockwebserver.RecordedRequest;
+import okio.Buffer;
+import org.apache.hadoop.fs.adl.TestADLResponseData;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Supporting class for mock test to validate Adls read operation.
+ */
+public class TestDataForRead {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestDataForRead.class);
+
+ private byte[] actualData;
+ private ArrayList responses;
+ private Dispatcher dispatcher;
+ private int intensityOfTest;
+ private boolean checkOfNoOfCalls;
+ private int expectedNoNetworkCall;
+
+ public TestDataForRead(final byte[] actualData, int expectedNoNetworkCall,
+ int intensityOfTest, boolean checkOfNoOfCalls) {
+
+ this.checkOfNoOfCalls = checkOfNoOfCalls;
+ this.actualData = actualData;
+ responses = new ArrayList();
+ this.expectedNoNetworkCall = expectedNoNetworkCall;
+ this.intensityOfTest = intensityOfTest;
+
+ dispatcher = new Dispatcher() {
+ @Override
+ public MockResponse dispatch(RecordedRequest recordedRequest)
+ throws InterruptedException {
+
+ if (recordedRequest.getRequestLine().contains("op=GETFILESTATUS")) {
+ return new MockResponse().setResponseCode(200).setBody(
+ TestADLResponseData
+ .getGetFileStatusJSONResponse(actualData.length));
+ }
+
+ if (recordedRequest.getRequestLine().contains("op=OPEN")) {
+ String request = recordedRequest.getRequestLine();
+ int offset = 0;
+ int byteCount = 0;
+
+ Pattern pattern = Pattern.compile("offset=([0-9]+)");
+ Matcher matcher = pattern.matcher(request);
+ if (matcher.find()) {
+ LOG.debug(matcher.group(1));
+ offset = Integer.parseInt(matcher.group(1));
+ }
+
+ pattern = Pattern.compile("length=([0-9]+)");
+ matcher = pattern.matcher(request);
+ if (matcher.find()) {
+ LOG.debug(matcher.group(1));
+ byteCount = Integer.parseInt(matcher.group(1));
+ }
+
+ Buffer buf = new Buffer();
+ buf.write(actualData, offset,
+ Math.min(actualData.length - offset, byteCount));
+ return new MockResponse().setResponseCode(200)
+ .setChunkedBody(buf, 4 * 1024 * 1024);
+ }
+
+ return new MockResponse().setBody("NOT SUPPORTED").setResponseCode(501);
+ }
+ };
+ }
+
+ public boolean isCheckOfNoOfCalls() {
+ return checkOfNoOfCalls;
+ }
+
+ public int getExpectedNoNetworkCall() {
+ return expectedNoNetworkCall;
+ }
+
+ public int getIntensityOfTest() {
+ return intensityOfTest;
+ }
+
+ public byte[] getActualData() {
+ return actualData;
+ }
+
+ public ArrayList getResponses() {
+ return responses;
+ }
+
+ public Dispatcher getDispatcher() {
+ return dispatcher;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java
new file mode 100644
index 0000000..7d6153d
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.live;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.adl.AdlFileSystem;
+import org.apache.hadoop.util.ReflectionUtils;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * Configure Adl storage file system.
+ */
+public final class AdlStorageConfiguration {
+ static final String CONTRACT_XML = "adls.xml";
+
+ private static final String CONTRACT_ENABLE_KEY =
+ "dfs.adl.test.contract.enable";
+ private static final boolean CONTRACT_ENABLE_DEFAULT = false;
+
+ private static final String FILE_SYSTEM_KEY =
+ String.format("test.fs.%s.name", AdlFileSystem.SCHEME);
+
+ private static final String FILE_SYSTEM_IMPL_KEY =
+ String.format("fs.%s.impl", AdlFileSystem.SCHEME);
+ private static final Class> FILE_SYSTEM_IMPL_DEFAULT =
+ AdlFileSystem.class;
+
+ private static boolean isContractTestEnabled = false;
+ private static Configuration conf = null;
+
+ private AdlStorageConfiguration() {
+ }
+
+ public synchronized static Configuration getConfiguration() {
+ Configuration newConf = new Configuration();
+ newConf.addResource(CONTRACT_XML);
+ return newConf;
+ }
+
+ public synchronized static boolean isContractTestEnabled() {
+ if (conf == null) {
+ conf = getConfiguration();
+ }
+
+ isContractTestEnabled = conf.getBoolean(CONTRACT_ENABLE_KEY,
+ CONTRACT_ENABLE_DEFAULT);
+ return isContractTestEnabled;
+ }
+
+ public synchronized static FileSystem createStorageConnector()
+ throws URISyntaxException, IOException {
+ if (conf == null) {
+ conf = getConfiguration();
+ }
+
+ if (!isContractTestEnabled()) {
+ return null;
+ }
+
+ String fileSystem = conf.get(FILE_SYSTEM_KEY);
+ if (fileSystem == null || fileSystem.trim().length() == 0) {
+ throw new IOException("Default file system not configured.");
+ }
+
+ Class> clazz = conf.getClass(FILE_SYSTEM_IMPL_KEY,
+ FILE_SYSTEM_IMPL_DEFAULT);
+ FileSystem fs = (FileSystem) ReflectionUtils.newInstance(clazz, conf);
+ fs.initialize(new URI(fileSystem), conf);
+ return fs;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageContract.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageContract.java
new file mode 100644
index 0000000..262b636
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageContract.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.live;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+class AdlStorageContract extends AbstractFSContract {
+ private FileSystem fs;
+
+ protected AdlStorageContract(Configuration conf) {
+ super(conf);
+ try {
+ fs = AdlStorageConfiguration.createStorageConnector();
+ } catch (URISyntaxException e) {
+ throw new IllegalStateException("Can not initialize ADL FileSystem. "
+ + "Please check test.fs.adl.name property.", e);
+ } catch (IOException e) {
+ throw new IllegalStateException("Can not initialize ADL FileSystem.", e);
+ }
+ this.setConf(AdlStorageConfiguration.getConfiguration());
+ }
+
+ @Override
+ public String getScheme() {
+ return "adl";
+ }
+
+ @Override
+ public FileSystem getTestFileSystem() throws IOException {
+ return this.fs;
+ }
+
+ @Override
+ public Path getTestPath() {
+ return new Path("/test");
+ }
+
+ @Override
+ public boolean isEnabled() {
+ return AdlStorageConfiguration.isContractTestEnabled();
+ }
+
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractAppendLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractAppendLive.java
new file mode 100644
index 0000000..ffe6dd3
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractAppendLive.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.live;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractAppendTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.junit.Test;
+
+/**
+ * Test Append on Adl file system.
+ */
+public class TestAdlContractAppendLive extends AbstractContractAppendTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration configuration) {
+ return new AdlStorageContract(configuration);
+ }
+
+ @Override
+ @Test
+ public void testRenameFileBeingAppended() throws Throwable {
+ ContractTestUtils.unsupported("Skipping since renaming file in append "
+ + "mode not supported in Adl");
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractConcatLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractConcatLive.java
new file mode 100644
index 0000000..60d30ac
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractConcatLive.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.live;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractContractConcatTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.Test;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
+
+/**
+ * Test concat on Adl file system.
+ */
+public class TestAdlContractConcatLive extends AbstractContractConcatTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration configuration) {
+ return new AdlStorageContract(configuration);
+ }
+
+ @Test
+ public void testConcatMissingTarget() throws Throwable {
+ Path testPath = path("test");
+ Path zeroByteFile = new Path(testPath, "zero.txt");
+ Path target = new Path(testPath, "target");
+ touch(getFileSystem(), zeroByteFile);
+ // Concat on missing target is allowed on Adls file system.
+ getFileSystem().concat(target, new Path[] {zeroByteFile});
+ }
+}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractCreateLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractCreateLive.java
new file mode 100644
index 0000000..06347e9
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractCreateLive.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.live;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Test creating files, overwrite options.
+ */
+public class TestAdlContractCreateLive extends AbstractContractCreateTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration configuration) {
+ return new AdlStorageContract(configuration);
+ }
+}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractDeleteLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractDeleteLive.java
new file mode 100644
index 0000000..6961f15
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractDeleteLive.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.live;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Test delete contract test.
+ */
+public class TestAdlContractDeleteLive extends AbstractContractDeleteTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration configuration) {
+ return new AdlStorageContract(configuration);
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractGetFileStatusLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractGetFileStatusLive.java
new file mode 100644
index 0000000..d50dd68
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractGetFileStatusLive.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.live;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Test getFileStatus contract test.
+ */
+public class TestAdlContractGetFileStatusLive extends
+ AbstractContractGetFileStatusTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration configuration) {
+ return new AdlStorageContract(configuration);
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractMkdirLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractMkdirLive.java
new file mode 100644
index 0000000..5e760c5
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractMkdirLive.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.live;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Test Mkdir contract on Adl storage file system.
+ */
+public class TestAdlContractMkdirLive extends AbstractContractMkdirTest {
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new AdlStorageContract(conf);
+ }
+}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractOpenLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractOpenLive.java
new file mode 100644
index 0000000..7a35d2c
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractOpenLive.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.live;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Test OPEN - read API.
+ */
+public class TestAdlContractOpenLive extends AbstractContractOpenTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration configuration) {
+ return new AdlStorageContract(configuration);
+ }
+}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractRenameLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractRenameLive.java
new file mode 100644
index 0000000..d72d35e
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractRenameLive.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.live;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Test rename contract test cases on Adl file system.
+ */
+public class TestAdlContractRenameLive extends AbstractContractRenameTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration configuration) {
+ return new AdlStorageContract(configuration);
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractRootDirLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractRootDirLive.java
new file mode 100644
index 0000000..8ebc632
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractRootDirLive.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.live;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Test operation on root level.
+ */
+public class TestAdlContractRootDirLive
+ extends AbstractContractRootDirectoryTest {
+ @Override
+ protected AbstractFSContract createContract(Configuration configuration) {
+ return new AdlStorageContract(configuration);
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractSeekLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractSeekLive.java
new file mode 100644
index 0000000..62423b6
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractSeekLive.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.live;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Test seek operation on Adl file system.
+ */
+public class TestAdlContractSeekLive extends AbstractContractSeekTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration configuration) {
+ return new AdlStorageContract(configuration);
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlDifferentSizeWritesLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlDifferentSizeWritesLive.java
new file mode 100644
index 0000000..5421e0b
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlDifferentSizeWritesLive.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.live;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.adl.common.Parallelized;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Random;
+import java.util.UUID;
+
+import static org.apache.hadoop.fs.adl.AdlConfKeys.WRITE_BUFFER_SIZE_KEY;
+
+/**
+ * Verify data integrity with different data sizes with buffer size.
+ */
+@RunWith(Parallelized.class)
+public class TestAdlDifferentSizeWritesLive {
+ private static Random rand = new Random();
+ private int totalSize;
+ private int chunkSize;
+
+ public TestAdlDifferentSizeWritesLive(int totalSize, int chunkSize) {
+ this.totalSize = totalSize;
+ this.chunkSize = chunkSize;
+ }
+
+ public static byte[] getRandomByteArrayData(int size) {
+ byte[] b = new byte[size];
+ rand.nextBytes(b);
+ return b;
+ }
+
+ @Parameterized.Parameters(name = "{index}: Data Size [{0}] ; Chunk Size "
+ + "[{1}]")
+ public static Collection testDataForIntegrityTest() {
+ return Arrays.asList(
+ new Object[][] {{4 * 1024, 1 * 1024}, {4 * 1024, 7 * 1024},
+ {4 * 1024, 10}, {2 * 1024, 10}, {1 * 1024, 10}, {100, 1},
+ {4 * 1024, 1 * 1024}, {7 * 1024, 2 * 1024}, {9 * 1024, 2 * 1024},
+ {10 * 1024, 3 * 1024}, {10 * 1024, 1 * 1024},
+ {10 * 1024, 8 * 1024}});
+ }
+
+ @BeforeClass
+ public static void cleanUpParent() throws IOException, URISyntaxException {
+ if (AdlStorageConfiguration.isContractTestEnabled()) {
+ Path path = new Path("/test/dataIntegrityCheck/");
+ FileSystem fs = AdlStorageConfiguration.createStorageConnector();
+ fs.delete(path, true);
+ }
+ }
+
+ @Before
+ public void setup() throws Exception {
+ org.junit.Assume
+ .assumeTrue(AdlStorageConfiguration.isContractTestEnabled());
+ }
+
+ @Test
+ public void testDataIntegrity() throws IOException {
+ Path path = new Path(
+ "/test/dataIntegrityCheck/" + UUID.randomUUID().toString());
+ FileSystem fs = null;
+ AdlStorageConfiguration.getConfiguration()
+ .setInt(WRITE_BUFFER_SIZE_KEY, 4 * 1024);
+ try {
+ fs = AdlStorageConfiguration.createStorageConnector();
+ } catch (URISyntaxException e) {
+ throw new IllegalStateException("Can not initialize ADL FileSystem. "
+ + "Please check test.fs.adl.name property.", e);
+ }
+ byte[] expectedData = getRandomByteArrayData(totalSize);
+
+ FSDataOutputStream out = fs.create(path, true);
+ int iteration = totalSize / chunkSize;
+ int reminderIteration = totalSize % chunkSize;
+ int offset = 0;
+ for (int i = 0; i < iteration; ++i) {
+ out.write(expectedData, offset, chunkSize);
+ offset += chunkSize;
+ }
+
+ out.write(expectedData, offset, reminderIteration);
+ out.close();
+
+ byte[] actualData = new byte[totalSize];
+ FSDataInputStream in = fs.open(path);
+ in.readFully(0, actualData);
+ in.close();
+ Assert.assertArrayEquals(expectedData, actualData);
+ Assert.assertTrue(fs.delete(path, true));
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileContextCreateMkdirLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileContextCreateMkdirLive.java
new file mode 100644
index 0000000..5166de1
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileContextCreateMkdirLive.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.live;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DelegateToFileSystem;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileContextCreateMkdirBaseTest;
+import org.apache.hadoop.fs.FileContextTestHelper;
+import org.apache.hadoop.fs.FileSystem;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+
+import java.net.URI;
+import java.util.UUID;
+
+/**
+ * Test file context Create/Mkdir operation.
+ */
+public class TestAdlFileContextCreateMkdirLive
+ extends FileContextCreateMkdirBaseTest {
+ private static final String KEY_FILE_SYSTEM = "test.fs.adl.name";
+
+ @BeforeClass
+ public static void skipTestCheck() {
+ Assume.assumeTrue(AdlStorageConfiguration.isContractTestEnabled());
+ }
+
+ @Override
+ public void setUp() throws Exception {
+ Configuration conf = AdlStorageConfiguration.getConfiguration();
+ String fileSystem = conf.get(KEY_FILE_SYSTEM);
+ if (fileSystem == null || fileSystem.trim().length() == 0) {
+ throw new Exception("Default file system not configured.");
+ }
+ URI uri = new URI(fileSystem);
+ FileSystem fs = AdlStorageConfiguration.createStorageConnector();
+ fc = FileContext.getFileContext(
+ new DelegateToFileSystem(uri, fs, conf, fs.getScheme(), false) {
+ }, conf);
+ super.setUp();
+ }
+
+ @Override
+ protected FileContextTestHelper createFileContextHelper() {
+ // On Windows, root directory path is created from local running directory.
+ // Adl does not support ':' as part of the path which results in failure.
+ return new FileContextTestHelper(UUID.randomUUID().toString());
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileContextMainOperationsLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileContextMainOperationsLive.java
new file mode 100644
index 0000000..ee10da7
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileContextMainOperationsLive.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.live;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.*;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.UUID;
+
+import static org.apache.hadoop.util.Shell.WINDOWS;
+
+/**
+ * Run collection of tests for the {@link FileContext}.
+ */
+public class TestAdlFileContextMainOperationsLive
+ extends FileContextMainOperationsBaseTest {
+
+ private static final String KEY_FILE_SYSTEM = "test.fs.adl.name";
+
+ @BeforeClass
+ public static void skipTestCheck() {
+ Assume.assumeTrue(AdlStorageConfiguration.isContractTestEnabled());
+ }
+
+ @Override
+ public void setUp() throws Exception {
+ Configuration conf = AdlStorageConfiguration.getConfiguration();
+ String fileSystem = conf.get(KEY_FILE_SYSTEM);
+ if (fileSystem == null || fileSystem.trim().length() == 0) {
+ throw new Exception("Default file system not configured.");
+ }
+ URI uri = new URI(fileSystem);
+ FileSystem fs = AdlStorageConfiguration.createStorageConnector();
+ fc = FileContext.getFileContext(
+ new DelegateToFileSystem(uri, fs, conf, fs.getScheme(), false) {
+ }, conf);
+ super.setUp();
+ }
+
+ @Override
+ protected FileContextTestHelper createFileContextHelper() {
+ // On Windows, root directory path is created from local running directory.
+ // Adl does not support ':' as part of the path which results in failure.
+ // return new FileContextTestHelper(GenericTestUtils
+ // .getRandomizedTestDir()
+ // .getAbsolutePath().replaceAll(":",""));
+ return new FileContextTestHelper(UUID.randomUUID().toString());
+ }
+
+ @Override
+ protected boolean listCorruptedBlocksSupported() {
+ return false;
+ }
+
+ @Override
+ public void testWorkingDirectory() throws Exception {
+ if (WINDOWS) {
+ // TODO :Fix is required in Hadoop shell to support windows permission
+ // set.
+ // The test is failing with NPE on windows platform only, with Linux
+ // platform test passes.
+ Assume.assumeTrue(false);
+ } else {
+ super.testWorkingDirectory();
+ }
+ }
+
+ @Override
+ public void testUnsupportedSymlink() throws IOException {
+ Assume.assumeTrue(false);
+ }
+
+ @Test
+ public void testSetVerifyChecksum() throws IOException {
+ Assume.assumeTrue(false);
+ }
+}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileSystemContractLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileSystemContractLive.java
new file mode 100644
index 0000000..88bacd9
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileSystemContractLive.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.live;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileSystemContractBaseTest;
+import org.apache.hadoop.fs.Path;
+import org.junit.Assume;
+import org.junit.Before;
+
+import java.io.IOException;
+
+/**
+ * Test Base contract tests on Adl file system.
+ */
+public class TestAdlFileSystemContractLive extends FileSystemContractBaseTest {
+ private FileSystem adlStore;
+
+ @Override
+ protected void setUp() throws Exception {
+ adlStore = AdlStorageConfiguration.createStorageConnector();
+ if (AdlStorageConfiguration.isContractTestEnabled()) {
+ fs = adlStore;
+ }
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ if (AdlStorageConfiguration.isContractTestEnabled()) {
+ cleanup();
+ adlStore = null;
+ fs = null;
+ }
+ }
+
+ private void cleanup() throws IOException {
+ adlStore.delete(new Path("/test"), true);
+ }
+
+ @Override
+ protected void runTest() throws Throwable {
+ if (AdlStorageConfiguration.isContractTestEnabled()) {
+ super.runTest();
+ }
+ }
+
+ @Before
+ public void skipTestCheck() {
+ Assume.assumeTrue(AdlStorageConfiguration.isContractTestEnabled());
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlInternalCreateNonRecursive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlInternalCreateNonRecursive.java
new file mode 100644
index 0000000..7e11a54
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlInternalCreateNonRecursive.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.live;
+
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.adl.common.Parallelized;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.UUID;
+
+/**
+ * Test createNonRecursive API.
+ */
+@RunWith(Parallelized.class)
+public class TestAdlInternalCreateNonRecursive {
+ private Path inputFileName;
+ private FsPermission inputPermission;
+ private boolean inputOverride;
+ private boolean inputFileAlreadyExist;
+ private boolean inputParentAlreadyExist;
+ private Class expectedExceptionType;
+ private FileSystem adlStore;
+
+ public TestAdlInternalCreateNonRecursive(String testScenario, String fileName,
+ FsPermission permission, boolean override, boolean fileAlreadyExist,
+ boolean parentAlreadyExist, Class exceptionType) {
+
+ // Random parent path for each test so that parallel execution does not fail
+ // other running test.
+ inputFileName = new Path(
+ "/test/createNonRecursive/" + UUID.randomUUID().toString(), fileName);
+ inputPermission = permission;
+ inputFileAlreadyExist = fileAlreadyExist;
+ inputOverride = override;
+ inputParentAlreadyExist = parentAlreadyExist;
+ expectedExceptionType = exceptionType;
+ }
+
+ @Parameterized.Parameters(name = "{0}")
+ public static Collection adlCreateNonRecursiveTestData()
+ throws UnsupportedEncodingException {
+ /*
+ Test Data
+ File name, Permission, Override flag, File already exist, Parent
+ already exist
+ shouldCreateSucceed, expectedExceptionIfFileCreateFails
+
+ File already exist and Parent already exist are mutually exclusive.
+ */
+ return Arrays.asList(new Object[][] {
+ {"CNR - When file do not exist.", UUID.randomUUID().toString(),
+ FsPermission.getFileDefault(), false, false, true, null},
+ {"CNR - When file exist. Override false", UUID.randomUUID().toString(),
+ FsPermission.getFileDefault(), false, true, true,
+ FileAlreadyExistsException.class},
+ {"CNR - When file exist. Override true", UUID.randomUUID().toString(),
+ FsPermission.getFileDefault(), true, true, true, null},
+
+ //TODO: This test is skipped till the fixes are not made it to prod.
+ /*{ "CNR - When parent do no exist.", UUID.randomUUID().toString(),
+ FsPermission.getFileDefault(), false, false, true, false,
+ IOException.class }*/});
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ Assume.assumeTrue(AdlStorageConfiguration.isContractTestEnabled());
+ adlStore = AdlStorageConfiguration.createStorageConnector();
+ }
+
+ @Test
+ public void testCreateNonRecursiveFunctionality() throws IOException {
+ if (inputFileAlreadyExist) {
+ FileSystem.create(adlStore, inputFileName, inputPermission);
+ }
+
+ // Mutually exclusive to inputFileAlreadyExist
+ if (inputParentAlreadyExist) {
+ adlStore.mkdirs(inputFileName.getParent());
+ } else {
+ adlStore.delete(inputFileName.getParent(), true);
+ }
+
+ try {
+ adlStore.createNonRecursive(inputFileName, inputPermission, inputOverride,
+ CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT,
+ adlStore.getDefaultReplication(inputFileName),
+ adlStore.getDefaultBlockSize(inputFileName), null);
+ } catch (IOException e) {
+
+ if (expectedExceptionType == null) {
+ throw e;
+ }
+
+ Assert.assertEquals(expectedExceptionType, e.getClass());
+ return;
+ }
+
+ if (expectedExceptionType != null) {
+ Assert.fail("CreateNonRecursive should have failed with exception "
+ + expectedExceptionType.getName());
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlPermissionLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlPermissionLive.java
new file mode 100644
index 0000000..dd7c10d
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlPermissionLive.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+
+package org.apache.hadoop.fs.adl.live;
+
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.adl.common.Parallelized;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.junit.*;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.UUID;
+
+/**
+ * Test ACL permission on file/folder on Adl file system.
+ */
+@RunWith(Parallelized.class)
+public class TestAdlPermissionLive {
+
+ private static Path testRoot = new Path("/test");
+ private FsPermission permission;
+ private Path path;
+ private FileSystem adlStore;
+
+ public TestAdlPermissionLive(FsPermission testPermission) {
+ permission = testPermission;
+ }
+
+ @Parameterized.Parameters(name = "{0}")
+ public static Collection adlCreateNonRecursiveTestData()
+ throws UnsupportedEncodingException {
+ /*
+ Test Data
+ File/Folder name, User permission, Group permission, Other Permission,
+ Parent already exist
+ shouldCreateSucceed, expectedExceptionIfFileCreateFails
+ */
+ final Collection