commons-lang
commons-lang
${commons-lang.version}
diff --git a/common/src/java/org/apache/hadoop/hive/common/io/crypto/CipherSuite.java b/common/src/java/org/apache/hadoop/hive/common/io/crypto/CipherSuite.java
new file mode 100644
index 0000000..1b6a69a
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/io/crypto/CipherSuite.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common.io.crypto;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * Defines properties of a CipherSuite. Modeled after the ciphers in
+ * {@link javax.crypto.Cipher}.
+ */
+@InterfaceAudience.Private
+public enum CipherSuite {
+ UNKNOWN("Unknown", 0),
+ AES_CTR_NOPADDING("AES/CTR/NoPadding", 16);
+
+ private final String name;
+ private final int algoBlockSize;
+
+ private Integer unknownValue = null;
+
+ CipherSuite(String name, int algoBlockSize) {
+ this.name = name;
+ this.algoBlockSize = algoBlockSize;
+ }
+
+ public void setUnknownValue(int unknown) {
+ this.unknownValue = unknown;
+ }
+
+ public int getUnknownValue() {
+ return unknownValue;
+ }
+
+ /**
+ * @return name of cipher suite, as in {@link javax.crypto.Cipher}
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * @return size of an algorithm block in bytes
+ */
+ public int getAlgorithmBlockSize() {
+ return algoBlockSize;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder("{");
+ builder.append("name: " + name);
+ builder.append(", algorithmBlockSize: " + algoBlockSize);
+ if (unknownValue != null) {
+ builder.append(", unknownValue: " + unknownValue);
+ }
+ builder.append("}");
+ return builder.toString();
+ }
+
+ public static void checkName(String name) {
+ CipherSuite[] suites = CipherSuite.values();
+ for (CipherSuite suite : suites) {
+ if (suite.getName().equals(name)) {
+ return;
+ }
+ }
+ throw new IllegalArgumentException("Invalid cipher suite name: " + name);
+ }
+
+ /**
+ * Convert to CipherSuite from name, {@link #algoBlockSize} is fixed for
+ * certain cipher suite, just need to compare the name.
+ * @param name cipher suite name
+ * @return CipherSuite cipher suite
+ */
+ public static CipherSuite convert(String name) {
+ CipherSuite[] suites = CipherSuite.values();
+ for (CipherSuite suite : suites) {
+ if (suite.getName().equals(name)) {
+ return suite;
+ }
+ }
+ throw new IllegalArgumentException("Invalid cipher suite name: " + name);
+ }
+
+ /**
+ * Returns suffix of cipher suite configuration.
+ * @return String configuration suffix
+ */
+ public String getConfigSuffix() {
+ String[] parts = name.split("/");
+ StringBuilder suffix = new StringBuilder();
+ for (String part : parts) {
+ suffix.append(".").append(part.toLowerCase());
+ }
+
+ return suffix.toString();
+ }
+}
diff --git a/common/src/java/org/apache/hadoop/hive/common/io/crypto/CryptoCodec.java b/common/src/java/org/apache/hadoop/hive/common/io/crypto/CryptoCodec.java
new file mode 100644
index 0000000..41fb84f
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/io/crypto/CryptoCodec.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hive.common.io.crypto;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configurable;
+
+/**
+ * A common interface for a cryptographic algorithm.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class CryptoCodec implements Configurable {
+
+ /**
+ * @return the CipherSuite for this codec.
+ */
+ public abstract CipherSuite getCipherSuite();
+
+ /**
+ * Return this codec's algorithm
+ */
+ public abstract String getAlgorithm();
+
+ /**
+ * Return the key length required by this cipher, in bytes
+ */
+ public abstract int getKeyLength();
+
+ /**
+ * Return the expected initialization vector length, in bytes, or 0 if not applicable
+ */
+ public abstract int getIvLength();
+
+ /**
+ * Get an encryptor for encrypting data.
+ */
+ public abstract Encryptor createEncryptor();
+
+ /**
+ * Return a decryptor for decrypting data.
+ */
+ public abstract Decryptor createDecryptor();
+
+}
diff --git a/common/src/java/org/apache/hadoop/hive/common/io/crypto/CryptoCodecFactory.java b/common/src/java/org/apache/hadoop/hive/common/io/crypto/CryptoCodecFactory.java
new file mode 100644
index 0000000..b326803
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/io/crypto/CryptoCodecFactory.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hive.common.io.crypto;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CryptoCodecFactory {
+ public static Logger LOG = LoggerFactory.getLogger(CryptoCodecFactory.class);
+
+ /**
+ * Get crypto codec for specified algorithm/mode/padding.
+ *
+ * @param conf
+ * the configuration
+ * @param CipherSuite
+ * algorithm/mode/padding
+ * @return CryptoCodec the codec object. Null value will be returned if no
+ * crypto codec classes with cipher suite configured.
+ */
+ public static CryptoCodec getInstance(Configuration conf,
+ CipherSuite cipherSuite) {
+ Class extends CryptoCodec> clazz = getCodecClass(conf, cipherSuite);
+ if (clazz == null) {
+ return null;
+ }
+ CryptoCodec codec = null;
+ try {
+ CryptoCodec c = ReflectionUtils.newInstance(clazz, conf);
+ if (c.getCipherSuite().getName().equals(cipherSuite.getName())) {
+ if (codec == null) {
+ LOG.debug("Using crypto codec {}.", clazz.getName());
+ codec = c;
+ }
+ } else {
+ LOG.warn("Crypto codec {} doesn't meet the cipher suite {}.",
+ clazz.getName(), cipherSuite.getName());
+ }
+ } catch (Exception e) {
+ LOG.warn("Crypto codec {} is not available.", clazz.getName());
+ }
+
+ if (codec != null) {
+ return codec;
+ }
+
+ throw new RuntimeException("No available crypto codec which meets " +
+ "the cipher suite " + cipherSuite.getName() + ".");
+ }
+
+ /**
+ * Get crypto codec for algorithm/mode/padding in config value
+ * hive.security.crypto.cipher.suite
+ *
+ * @param conf
+ * the configuration
+ * @return CryptoCodec the codec object Null value will be returned if no
+ * crypto codec classes with cipher suite configured.
+ */
+ public static CryptoCodec getInstance(Configuration conf) {
+ String name = conf.get(ConfVars.HIVE_SECURITY_CRYPTO_CIPHER_SUITE.varname,
+ ConfVars.HIVE_SECURITY_CRYPTO_CIPHER_SUITE.getDefaultValue());
+ return getInstance(conf, CipherSuite.convert(name));
+ }
+
+ private static Class extends CryptoCodec> getCodecClass(
+ Configuration conf, CipherSuite cipherSuite) {
+ Class extends CryptoCodec> result = null;
+ String codecString = conf.get(ConfVars.HIVE_SECURITY_CRYPTO_CODEC.varname,
+ ConfVars.HIVE_SECURITY_CRYPTO_CODEC.getDefaultValue());
+
+ try {
+ result = (Class extends CryptoCodec>) conf.getClassByName(codecString);
+ } catch (ClassCastException e) {
+ LOG.warn("Class " + codecString + " is not a CryptoCodec.");
+ } catch (ClassNotFoundException e) {
+ LOG.warn("Crypto codec " + codecString + " not found.");
+ }
+
+ return result;
+ }
+}
diff --git a/common/src/java/org/apache/hadoop/hive/common/io/crypto/Decryptor.java b/common/src/java/org/apache/hadoop/hive/common/io/crypto/Decryptor.java
new file mode 100644
index 0000000..fd96105
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/io/crypto/Decryptor.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.io.crypto;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Decryptors apply a cipher to an InputStream to recover plaintext.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class Decryptor {
+
+ /**
+ * Get the secret key
+ */
+ public abstract Key getKey();
+
+ /**
+ * Set the secret key
+ * @param key
+ */
+ public abstract void setKey(Key key);
+
+ /**
+ * Get the expected length for the initialization vector
+ * @return the expected length for the initialization vector
+ */
+ public abstract int getIvLength();
+
+ /**
+ * Get the cipher's internal block size
+ * @return the cipher's internal block size
+ */
+ public abstract int getBlockSize();
+
+ /**
+ * Set the initialization vector
+ * @param iv
+ */
+ public abstract void setIv(byte[] iv);
+
+ /**
+ * Reset state, reinitialize with the key and iv
+ */
+ public abstract void reset();
+
+ /**
+ * Create a stream for decryption
+ * @param in
+ */
+ public abstract InputStream createDecryptionStream(InputStream in);
+
+ /**
+ * Decrypt a stream of ciphertext
+ * @param in
+ * @param out
+ */
+ public void decrypt(InputStream in, OutputStream out, int outLen)
+ throws IOException {
+ InputStream is = createDecryptionStream(in);
+ byte buf[] = new byte[8*1024];
+ long remaining = outLen;
+ try {
+ while (remaining > 0) {
+ int toRead = (int)(remaining < buf.length ? remaining : buf.length);
+ int read = is.read(buf, 0, toRead);
+ if (read < 0) {
+ break;
+ }
+ out.write(buf, 0, read);
+ remaining -= read;
+ }
+ } finally {
+ is.close();
+ }
+ }
+
+ /**
+ * Decrypt a stream to a array of byte
+ * @param in
+ * @param out
+ */
+ public void decrypt(InputStream in, byte[] dest, int destOffset,
+ int destSize) throws IOException {
+ InputStream is = createDecryptionStream(in);
+ try {
+ IOUtils.readFully(is, dest, destOffset, destSize);
+ } finally {
+ is.close();
+ }
+ }
+}
diff --git a/common/src/java/org/apache/hadoop/hive/common/io/crypto/Encryptor.java b/common/src/java/org/apache/hadoop/hive/common/io/crypto/Encryptor.java
new file mode 100644
index 0000000..5bce6dc
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/io/crypto/Encryptor.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.io.crypto;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Encryptors apply a cipher to an OutputStream to produce ciphertext.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class Encryptor {
+ public static Logger LOG = LoggerFactory.getLogger(Encryptor.class);
+
+ /**
+ * Get the secret key
+ */
+ public abstract Key getKey();
+
+ /**
+ * Set the secret key
+ * @param key
+ */
+ public abstract void setKey(Key key);
+
+ /**
+ * Get the expected length for the initialization vector
+ * @return the expected length for the initialization vector
+ */
+ public abstract int getIvLength();
+
+ /**
+ * Get the cipher's internal block size
+ * @return the cipher's internal block size
+ */
+ public abstract int getBlockSize();
+
+ /**
+ * Get the initialization vector
+ */
+ public abstract byte[] getIv();
+
+ /**
+ * Set the initialization vector
+ * @param iv
+ */
+ public abstract void setIv(byte[] iv);
+
+ /**
+ * Reset state, reinitialize with the key and iv
+ */
+ public abstract void reset();
+
+ /**
+ * Create a stream for encryption
+ * @param out
+ */
+ public abstract OutputStream createEncryptionStream(OutputStream out);
+
+ /**
+ * Encrypt a stream of plaintext
+ * @param in
+ * @param out
+ */
+ public void encrypt(InputStream in, OutputStream out) throws IOException {
+ OutputStream os = createEncryptionStream(out);
+ try {
+ IOUtils.copy(in, os);
+ } finally {
+ os.close();
+ }
+ }
+
+ /**
+ * Encrypt a array of byte of plaintext
+ * @param src
+ * @param offset
+ * @param length
+ * @param out
+ */
+ public void encrypt(byte[] src, int offset, int length,
+ OutputStream out) throws IOException {
+ OutputStream os = createEncryptionStream(out);
+ try {
+ os.write(src, offset, length);
+ } finally {
+ os.close();
+ }
+ }
+}
diff --git a/common/src/java/org/apache/hadoop/hive/common/io/crypto/Key.java b/common/src/java/org/apache/hadoop/hive/common/io/crypto/Key.java
new file mode 100644
index 0000000..f056669
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/io/crypto/Key.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.hive.common.io.crypto;
+
+import java.util.Arrays;
+
+/**
+ * Key is class contains key name and key material.
+ *
+ */
+public class Key {
+ private final String name;
+ private final byte[] material;
+
+ public Key(String name, byte[] material) {
+ this.name = name;
+ this.material = Arrays.copyOf(material, material.length);;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public byte[] getMaterial() {
+ return material;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder buf = new StringBuilder();
+ buf.append("key");
+ buf.append("=");
+ if (material == null) {
+ buf.append("null");
+ } else {
+ for(byte b: material) {
+ buf.append(' ');
+ int right = b & 0xff;
+ if (right < 0x10) {
+ buf.append('0');
+ }
+ buf.append(Integer.toHexString(right));
+ }
+ }
+ return buf.toString();
+ }
+}
diff --git a/common/src/java/org/apache/hadoop/hive/common/io/crypto/aes/AesDecryptor.java b/common/src/java/org/apache/hadoop/hive/common/io/crypto/aes/AesDecryptor.java
new file mode 100644
index 0000000..1479be5
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/io/crypto/aes/AesDecryptor.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.io.crypto.aes;
+
+import java.io.InputStream;
+import java.security.InvalidAlgorithmParameterException;
+import java.security.InvalidKeyException;
+import java.util.Arrays;
+
+import javax.crypto.Cipher;
+import javax.crypto.spec.IvParameterSpec;
+import javax.crypto.spec.SecretKeySpec;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hive.common.io.crypto.Decryptor;
+import org.apache.hadoop.hive.common.io.crypto.Key;
+
+import com.google.common.base.Preconditions;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class AesDecryptor extends Decryptor {
+
+ private final Cipher cipher;
+ private Key key;
+ private byte[] iv;
+ private boolean initialized = false;
+
+ public AesDecryptor(Cipher cipher) {
+ this.cipher = cipher;
+ }
+
+ public AesDecryptor(Cipher cipher, Key key, byte[] iv) {
+ this.cipher = cipher;
+ this.key = key;
+ this.iv = Arrays.copyOf(iv, iv.length);
+ }
+
+ public Cipher getCipher() {
+ return cipher;
+ }
+
+ @Override
+ public Key getKey() {
+ return key;
+ }
+
+ @Override
+ public void setKey(Key key) {
+ Preconditions.checkNotNull(key, "Key cannot be null");
+ if (key != null) {
+ Preconditions.checkArgument(key.getMaterial().length == JceAesCtrCryptoCodec.KEY_LENGTH,
+ "Invalid key length");
+ }
+ this.key = key;
+ }
+
+ @Override
+ public int getIvLength() {
+ return JceAesCtrCryptoCodec.IV_LENGTH;
+ }
+
+ @Override
+ public int getBlockSize() {
+ return JceAesCtrCryptoCodec.BLOCK_SIZE;
+ }
+
+ @Override
+ public void setIv(byte[] iv) {
+ Preconditions.checkNotNull(iv, "IV cannot be null");
+ Preconditions.checkArgument(iv.length == JceAesCtrCryptoCodec.IV_LENGTH,
+ "Invalid IV length");
+ this.iv = Arrays.copyOf(iv, iv.length);
+ }
+
+ @Override
+ public void reset() {
+ init();
+ }
+
+ @Override
+ public InputStream createDecryptionStream(InputStream in) {
+ if (!initialized) {
+ init();
+ }
+ return new javax.crypto.CipherInputStream(in, cipher);
+ }
+
+ protected void init() {
+ try {
+ if (iv == null) {
+ throw new NullPointerException("IV is null");
+ }
+ cipher.init(javax.crypto.Cipher.DECRYPT_MODE,
+ new SecretKeySpec(key.getMaterial(), "AES"), new IvParameterSpec(iv));
+ } catch (InvalidKeyException e) {
+ throw new RuntimeException(e);
+ } catch (InvalidAlgorithmParameterException e) {
+ throw new RuntimeException(e);
+ }
+ initialized = true;
+ }
+
+}
diff --git a/common/src/java/org/apache/hadoop/hive/common/io/crypto/aes/AesEncryptor.java b/common/src/java/org/apache/hadoop/hive/common/io/crypto/aes/AesEncryptor.java
new file mode 100644
index 0000000..3d5cce1
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/io/crypto/aes/AesEncryptor.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.io.crypto.aes;
+
+import java.io.OutputStream;
+import java.security.InvalidAlgorithmParameterException;
+import java.security.InvalidKeyException;
+import java.security.SecureRandom;
+import java.util.Arrays;
+
+import javax.crypto.Cipher;
+import javax.crypto.spec.IvParameterSpec;
+import javax.crypto.spec.SecretKeySpec;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hive.common.io.crypto.Encryptor;
+import org.apache.hadoop.hive.common.io.crypto.Key;
+
+import com.google.common.base.Preconditions;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class AesEncryptor extends Encryptor {
+
+ private final Cipher cipher;
+ private final SecureRandom rng;
+ private Key key;
+ private byte[] iv;
+ private boolean initialized = false;
+
+ public AesEncryptor(Cipher cipher, SecureRandom rng) {
+ this.cipher = cipher;
+ this.rng = rng;
+ }
+
+ public AesEncryptor(Cipher cipher, SecureRandom rng, Key key, byte[] iv) {
+ this.cipher = cipher;
+ this.rng = rng;
+ this.key = key;
+ this.iv = Arrays.copyOf(iv, iv.length);
+ }
+
+ public Cipher getCipher() {
+ return cipher;
+ }
+
+ @Override
+ public Key getKey() {
+ return key;
+ }
+
+ @Override
+ public void setKey(Key key) {
+ Preconditions.checkNotNull(key, "Key cannot be null");
+ if (key != null) {
+ Preconditions.checkArgument(key.getMaterial().length == JceAesCtrCryptoCodec.KEY_LENGTH,
+ "Invalid key length");
+ }
+ this.key = key;
+ }
+
+ @Override
+ public int getIvLength() {
+ return JceAesCtrCryptoCodec.IV_LENGTH;
+ }
+
+ @Override
+ public int getBlockSize() {
+ return JceAesCtrCryptoCodec.BLOCK_SIZE;
+ }
+
+ @Override
+ public byte[] getIv() {
+ return iv;
+ }
+
+ @Override
+ public void setIv(byte[] iv) {
+ if (iv != null) {
+ Preconditions.checkArgument(iv.length == JceAesCtrCryptoCodec.IV_LENGTH,
+ "Invalid IV length");
+ }
+ this.iv = Arrays.copyOf(iv, iv.length);
+ }
+
+ @Override
+ public void reset() {
+ init();
+ }
+
+ @Override
+ public OutputStream createEncryptionStream(OutputStream out) {
+ if (!initialized) {
+ init();
+ }
+ return new javax.crypto.CipherOutputStream(out, cipher);
+ }
+
+ protected void init() {
+ try {
+ if (iv == null) {
+ iv = new byte[getIvLength()];
+ rng.nextBytes(iv);
+ }
+ cipher.init(javax.crypto.Cipher.ENCRYPT_MODE,
+ new SecretKeySpec(key.getMaterial(), "AES"), new IvParameterSpec(iv));
+ } catch (InvalidKeyException e) {
+ throw new RuntimeException(e);
+ } catch (InvalidAlgorithmParameterException e) {
+ throw new RuntimeException(e);
+ }
+ initialized = true;
+ }
+}
diff --git a/common/src/java/org/apache/hadoop/hive/common/io/crypto/aes/JceAesCtrCryptoCodec.java b/common/src/java/org/apache/hadoop/hive/common/io/crypto/aes/JceAesCtrCryptoCodec.java
new file mode 100644
index 0000000..493a02b
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/io/crypto/aes/JceAesCtrCryptoCodec.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.io.crypto.aes;
+
+import java.security.GeneralSecurityException;
+import java.security.SecureRandom;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.io.crypto.CipherSuite;
+import org.apache.hadoop.hive.common.io.crypto.CryptoCodec;
+import org.apache.hadoop.hive.common.io.crypto.Decryptor;
+import org.apache.hadoop.hive.common.io.crypto.Encryptor;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * AES-128, provided by the JCE
+ *
+ * Algorithm instances are pooled for reuse, so the cipher provider and mode
+ * are configurable but fixed at instantiation.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class JceAesCtrCryptoCodec extends CryptoCodec {
+ private static final Log LOG = LogFactory.getLog(JceAesCtrCryptoCodec.class);
+ protected static final CipherSuite SUITE = CipherSuite.AES_CTR_NOPADDING;
+
+ public static final int KEY_LENGTH = 16;
+ public static final int KEY_LENGTH_BITS = KEY_LENGTH * 8;
+ public static final int BLOCK_SIZE = 16;
+ public static final int IV_LENGTH = 16;
+
+ public static final String CIPHER_PROVIDER_KEY = "hive.security.crypto.jce.provider";
+ public static final String RNG_ALGORITHM_KEY = "hive.security.crypto.rng.algorithm";
+ public static final String RNG_PROVIDER_KEY = "hive.security.crypto.rng.provider";
+
+ private Configuration conf;
+ private String rngAlgorithm;
+ private String cipherProvider;
+ private SecureRandom rng;
+
+ public JceAesCtrCryptoCodec() {
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ // The JCE provider, null if default
+ cipherProvider = conf.get(CIPHER_PROVIDER_KEY);
+ // RNG algorithm
+ rngAlgorithm = conf.get(RNG_ALGORITHM_KEY, "SHA1PRNG");
+ // RNG provider, null if default
+ String rngProvider = conf.get(RNG_PROVIDER_KEY);
+ try {
+ if (rngProvider != null) {
+ rng = SecureRandom.getInstance(rngAlgorithm, rngProvider);
+ } else {
+ rng = SecureRandom.getInstance(rngAlgorithm);
+ }
+ } catch (GeneralSecurityException e) {
+ LOG.warn("Could not instantiate specified RNG, falling back to default", e);
+ rng = new SecureRandom();
+ }
+ }
+
+ @Override
+ public CipherSuite getCipherSuite() {
+ return SUITE;
+ }
+
+ @Override
+ public String getAlgorithm() {
+ return "AES";
+ }
+
+ @Override
+ public int getKeyLength() {
+ return KEY_LENGTH;
+ }
+
+ @Override
+ public int getIvLength() {
+ return IV_LENGTH;
+ }
+
+ @Override
+ public Encryptor createEncryptor() {
+ return new AesEncryptor(getJCECipherInstance(), rng);
+ }
+
+ @Override
+ public Decryptor createDecryptor() {
+ return new AesDecryptor(getJCECipherInstance());
+ }
+
+ @VisibleForTesting
+ SecureRandom getRNG() {
+ return rng;
+ }
+
+ private javax.crypto.Cipher getJCECipherInstance() {
+ try {
+ if (cipherProvider != null) {
+ return javax.crypto.Cipher.getInstance(SUITE.getName(), cipherProvider);
+ }
+ return javax.crypto.Cipher.getInstance(SUITE.getName());
+ } catch (GeneralSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 7f4afd9..0e95d87 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1729,7 +1729,13 @@
"When auto reducer parallelism is enabled this factor will be used to over-partition data in shuffle edges."),
TEZ_MIN_PARTITION_FACTOR("hive.tez.min.partition.factor", 0.25f,
"When auto reducer parallelism is enabled this factor will be used to put a lower limit to the number\n" +
- "of reducers that tez specifies.")
+ "of reducers that tez specifies."),
+ HIVE_SECURITY_CRYPTO_CIPHER_SUITE("hive.security.crypto.cipher.suite",
+ "AES/CTR/NoPadding",
+ "Now just support AES/CTR/NoPadding"),
+ HIVE_SECURITY_CRYPTO_CODEC("hive.security.crypto.codec",
+ "hive.security.crypto.codec.aes.JceAesCryptoCodec",
+ "The crypto codec should match crypto cipher suite.")
;
public final String varname;
diff --git a/common/src/test/org/apache/hadoop/hive/common/io/crypto/TestCipherSuite.java b/common/src/test/org/apache/hadoop/hive/common/io/crypto/TestCipherSuite.java
new file mode 100644
index 0000000..d47f328
--- /dev/null
+++ b/common/src/test/org/apache/hadoop/hive/common/io/crypto/TestCipherSuite.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.io.crypto;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestCipherSuite {
+ @Test
+ public void testAesCtr() throws Exception {
+ CipherSuite SUITE = CipherSuite.AES_CTR_NOPADDING;
+ Assert.assertEquals("AES/CTR/NoPadding", SUITE.getName());
+ Assert.assertEquals(16, SUITE.getAlgorithmBlockSize());
+
+ String trueAlgoName = "AES/CTR/NoPadding";
+ String wrongAlogName = "AES/CTR/NoPadding/Wrong";
+ CipherSuite.checkName(trueAlgoName);
+ try {
+ CipherSuite.checkName(wrongAlogName);
+ Assert.fail("wrong cipher algorithm");
+ } catch (Exception e) {
+ // no-op
+ }
+
+ SUITE = CipherSuite.convert(trueAlgoName);
+ Assert.assertEquals("AES/CTR/NoPadding", SUITE.getName());
+ Assert.assertEquals(16, SUITE.getAlgorithmBlockSize());
+
+ try {
+ SUITE = CipherSuite.convert(wrongAlogName);
+ Assert.fail("wrong cipher algorithm");
+ } catch (Exception e) {
+ // no-op
+ }
+ }
+}
diff --git a/common/src/test/org/apache/hadoop/hive/common/io/crypto/TestKey.java b/common/src/test/org/apache/hadoop/hive/common/io/crypto/TestKey.java
new file mode 100644
index 0000000..a5c7dc5
--- /dev/null
+++ b/common/src/test/org/apache/hadoop/hive/common/io/crypto/TestKey.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common.io.crypto;
+
+import java.security.SecureRandom;
+import java.util.Arrays;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestKey {
+
+ @Test
+ public void testKey() throws Exception {
+ SecureRandom rng = SecureRandom.getInstance("SHA1PRNG");
+ byte[] keyBytes = new byte[16];
+ rng.nextBytes(keyBytes);
+ Key key = new Key("key01", keyBytes);
+ Assert.assertEquals("key01", key.getName());
+ Assert.assertTrue(Arrays.equals(keyBytes, key.getMaterial()));
+ }
+}
diff --git a/common/src/test/org/apache/hadoop/hive/common/io/crypto/aes/TestAesCtrCryptoCodec.java b/common/src/test/org/apache/hadoop/hive/common/io/crypto/aes/TestAesCtrCryptoCodec.java
new file mode 100644
index 0000000..cdaa762
--- /dev/null
+++ b/common/src/test/org/apache/hadoop/hive/common/io/crypto/aes/TestAesCtrCryptoCodec.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common.io.crypto.aes;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.security.SecureRandom;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.io.crypto.CipherSuite;
+import org.apache.hadoop.hive.common.io.crypto.CryptoCodec;
+import org.apache.hadoop.hive.common.io.crypto.CryptoCodecFactory;
+import org.apache.hadoop.hive.common.io.crypto.Decryptor;
+import org.apache.hadoop.hive.common.io.crypto.Encryptor;
+import org.apache.hadoop.hive.common.io.crypto.Key;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestAesCtrCryptoCodec {
+ private SecureRandom rng;
+ private Key key;
+ private final byte[] iv = new byte[16];
+
+ @Before
+ public void setUp() throws Exception{
+ rng = SecureRandom.getInstance("SHA1PRNG");
+ byte[] keyBytes = new byte[16];
+ rng.nextBytes(keyBytes);
+ key = new Key("key1", keyBytes);
+ rng.nextBytes(iv);
+ }
+
+ @Test
+ public void testBasic() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set("hive.security.crypto.codec", JceAesCtrCryptoCodec.class.getName());
+ conf.set("hive.security.crypto.cipher.suite", "AES/CTR/NoPadding");
+ CryptoCodec codec = CryptoCodecFactory.getInstance(conf);
+ Assert.assertEquals(CipherSuite.AES_CTR_NOPADDING, codec.getCipherSuite());
+ Assert.assertEquals("AES", codec.getAlgorithm());
+ Assert.assertEquals(JceAesCtrCryptoCodec.KEY_LENGTH, codec.getKeyLength());
+ Assert.assertEquals(JceAesCtrCryptoCodec.IV_LENGTH, codec.getIvLength());
+ }
+
+ @Test
+ public void testEncrypt() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set("hive.security.crypto.codec", JceAesCtrCryptoCodec.class.getName());
+ conf.set("hive.security.crypto.cipher.suite", "AES/CTR/NoPadding");
+ CryptoCodec codec = CryptoCodecFactory.getInstance(conf);
+
+ Encryptor encryptor = codec.createEncryptor();
+ encryptor.setKey(key);
+ encryptor.setIv(iv);
+ String text = "TestAesCtrCryptoCodec#testEncrypt";
+ ByteArrayInputStream in = new ByteArrayInputStream(text.getBytes());
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ encryptor.encrypt(in, out);
+ byte[] encryptedText = out.toByteArray();
+ Assert.assertNotSame(text, new String(encryptedText));
+
+ Decryptor decryptor = codec.createDecryptor();
+ decryptor.setKey(key);
+ decryptor.setIv(iv);
+ in = new ByteArrayInputStream(encryptedText);
+ out = new ByteArrayOutputStream();
+ decryptor.decrypt(in, out, text.getBytes().length);
+ String decryptedText = new String(out.toByteArray());
+ Assert.assertEquals(text, decryptedText);
+ }
+}
diff --git a/common/src/test/org/apache/hadoop/hive/common/io/crypto/aes/TestAesEncryptor.java b/common/src/test/org/apache/hadoop/hive/common/io/crypto/aes/TestAesEncryptor.java
new file mode 100644
index 0000000..6aec4a3
--- /dev/null
+++ b/common/src/test/org/apache/hadoop/hive/common/io/crypto/aes/TestAesEncryptor.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common.io.crypto.aes;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.security.SecureRandom;
+
+import javax.crypto.Cipher;
+
+import org.apache.hadoop.hive.common.io.crypto.Decryptor;
+import org.apache.hadoop.hive.common.io.crypto.Encryptor;
+import org.apache.hadoop.hive.common.io.crypto.Key;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestAesEncryptor {
+ private Cipher cipher;
+ private SecureRandom rng;
+ private Key key;
+ private final byte[] iv = new byte[16];
+
+ @Before
+ public void setUp() throws Exception{
+ cipher = Cipher.getInstance("AES/CTR/NoPadding");
+ rng = SecureRandom.getInstance("SHA1PRNG");
+ byte[] keyBytes = new byte[16];
+ rng.nextBytes(keyBytes);
+ key = new Key("key1", keyBytes);
+ rng.nextBytes(iv);
+ }
+
+ @Test
+ public void testEncryptStream() throws Exception {
+ Encryptor encryptor = new AesEncryptor(cipher, rng);
+ encryptor.setKey(key);
+ encryptor.setIv(iv);
+
+ String text = "TestAesEncryptor#testEncryptStream";
+ ByteArrayInputStream in = new ByteArrayInputStream(text.getBytes());
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ encryptor.encrypt(in, out);
+ byte[] encryptedText = out.toByteArray();
+ Assert.assertNotSame(text, new String(encryptedText));
+
+ Decryptor decryptor = new AesDecryptor(cipher);
+ decryptor.setKey(key);
+ decryptor.setIv(iv);
+ in = new ByteArrayInputStream(encryptedText);
+ out = new ByteArrayOutputStream();
+ decryptor.decrypt(in, out, text.getBytes().length);
+ String decryptedText = new String(out.toByteArray());
+ Assert.assertEquals(text, decryptedText);
+ }
+
+ @Test
+ public void testEncryptByteArray() throws Exception {
+ Encryptor encryptor = new AesEncryptor(cipher, rng);
+ encryptor.setKey(key);
+ encryptor.setIv(iv);
+
+ String text = "TestAesEncryptor#testEncryptByteArray";
+ byte[] textBytes = text.getBytes();
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ encryptor.encrypt(textBytes, 0, textBytes.length, out);
+ byte[] encryptedText = out.toByteArray();
+ Assert.assertNotSame(text, new String(encryptedText));
+
+ Decryptor decryptor = new AesDecryptor(cipher);
+ decryptor.setKey(key);
+ decryptor.setIv(iv);
+ InputStream in = new ByteArrayInputStream(encryptedText);
+ byte[] decryptedBytes = new byte[textBytes.length];
+ decryptor.decrypt(in, decryptedBytes, 0, decryptedBytes.length);
+ String decryptedText = new String(decryptedBytes);
+ Assert.assertEquals(text, decryptedText);
+ }
+}