diff --git a/pom.xml b/pom.xml index c8d1b87..4f103f3 100644 --- a/pom.xml +++ b/pom.xml @@ -113,7 +113,7 @@ 2.1.6 0.20.2 1.2.1 - 2.4.0 + 3.0.0-SNAPSHOT ${basedir}/${hive.path.to.root}/testutils/hadoop 0.98.3-hadoop1 0.98.3-hadoop2 diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 0533ae8..5bc052f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -225,7 +225,7 @@ public static Schema getSchema(BaseSemanticAnalyzer sem, HiveConf conf) { String tableName = "result"; List lst = null; try { - lst = MetaStoreUtils.getFieldsFromDeserializer(tableName, td.getDeserializer()); + lst = MetaStoreUtils.getFieldsFromDeserializer(tableName, td.getDeserializer(conf)); } catch (Exception e) { LOG.warn("Error getting schema: " + org.apache.hadoop.util.StringUtils.stringifyException(e)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index cd017d8..caa4140 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -173,6 +173,7 @@ import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveV1Authorizer; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.aes.HiveSerdeKeyManagement; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe; import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; @@ -3795,6 +3796,12 @@ private int createTable(Hive db, CreateTableDesc crtTbl) throws HiveException { Table tbl = db.newTable(crtTbl.getTableName()); if (crtTbl.getTblProps() != null) { + //set up table for properties if encryption is needed + try { + HiveSerdeKeyManagement.setupTableForEncryption(conf, crtTbl.getTblProps()); + } catch (Exception e) { + throw new HiveException(e); + } tbl.getTTable().getParameters().putAll(crtTbl.getTblProps()); } @@ -3971,6 +3978,12 @@ private int createTableLike(Hive db, CreateTableLikeDesc crtTbl) throws HiveExce tbl=db.newTable(targetTableName); if (crtTbl.getTblProps() != null) { + //set up table for properties if encryption is needed + try { + HiveSerdeKeyManagement.setupTableForEncryption(conf, crtTbl.getTblProps()); + } catch (Exception e) { + throw new HiveException(e); + } tbl.getTTable().getParameters().putAll(crtTbl.getTblProps()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java index 1dde78e..84ff363 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java @@ -291,7 +291,7 @@ protected void initializeOp(Configuration hconf) throws HiveException { statsFromRecordWriter = new boolean[numFiles]; serializer = (Serializer) conf.getTableInfo().getDeserializerClass().newInstance(); - serializer.initialize(null, conf.getTableInfo().getProperties()); + serializer.initialize(hconf, conf.getTableInfo().getProperties()); outputClass = serializer.getSerializedClass(); // Timeout is chosen to make sure that even if one iteration takes more than diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index e6b4422..9a40380 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -5948,7 +5948,7 @@ private Operator genFileSinkPlan(String dest, QB qb, Operator input) try { StructObjectInspector rowObjectInspector = (StructObjectInspector) table_desc - .getDeserializer().getObjectInspector(); + .getDeserializer(conf).getObjectInspector(); List fields = rowObjectInspector .getAllStructFieldRefs(); for (int i = 0; i < fields.size(); i++) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java index 39f1793..1e48002 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.Properties; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils; @@ -79,8 +80,12 @@ public TableDesc( * Return a deserializer object corresponding to the tableDesc. */ public Deserializer getDeserializer() throws Exception { + return getDeserializer(null); + } + + public Deserializer getDeserializer(Configuration conf) throws Exception { Deserializer de = getDeserializerClass().newInstance(); - SerDeUtils.initializeSerDe(de, null, properties, null); + SerDeUtils.initializeSerDe(de, conf, properties, null); return de; } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/AbstractFieldRewriter.java b/serde/src/java/org/apache/hadoop/hive/serde2/AbstractFieldRewriter.java index d6a0fcf..b1799fa 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/AbstractFieldRewriter.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/AbstractFieldRewriter.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.serde2; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import java.io.IOException; @@ -27,8 +28,8 @@ public class AbstractFieldRewriter implements FieldRewriter { @Override - public void init(List columnNames, List columnTypes, Properties properties) - throws IOException { + public void init(List columnNames, List columnTypes, Properties properties, + Configuration conf) throws IOException { } @Override diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/FieldRewriter.java b/serde/src/java/org/apache/hadoop/hive/serde2/FieldRewriter.java index e75bf6b..92a5542 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/FieldRewriter.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/FieldRewriter.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.serde2; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import java.io.IOException; @@ -32,8 +33,8 @@ */ public interface FieldRewriter { - void init(List columnNames, List columnTypes, Properties properties) - throws IOException; + void init(List columnNames, List columnTypes, Properties properties, + Configuration conf) throws IOException; void encode(int index, ByteStream.Input input, ByteStream.Output output) throws IOException; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/aes/AESConstants.java b/serde/src/java/org/apache/hadoop/hive/serde2/aes/AESConstants.java new file mode 100644 index 0000000..2a6b939 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/aes/AESConstants.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.aes; + +public class AESConstants { + public static final int KEY_LENGTH = 128; + + public static final int IV_LENGTH = 128; + +// public static final String ALGORITHM = "AES/CBC/PKCS5PADDING"; + public static final String ALGORITHM = "AES/CTR/NOPADDING"; + + public static final String PROVIDER = "SunJCE"; +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/aes/AESRewriter.java b/serde/src/java/org/apache/hadoop/hive/serde2/aes/AESRewriter.java new file mode 100644 index 0000000..da1a972 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/aes/AESRewriter.java @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.aes; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion; +import org.apache.hadoop.crypto.key.kms.KMSClientProvider; +import org.apache.hadoop.hive.serde2.AbstractFieldRewriter; +import org.apache.hadoop.hive.serde2.ByteStream; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.security.authorize.AuthorizationException; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.List; +import java.util.Properties; + +import javax.crypto.Cipher; + +public class AESRewriter extends AbstractFieldRewriter { + private Cipher cipher; + private byte[] keyBytes; + private byte[] ivBytes; + private boolean isEncode = true; + private static Log LOG = LogFactory.getLog(AESRewriter.class); + + @Override + public void init(List columnNames, List columnTypes, Properties properties, + Configuration conf) throws IOException { + String keyNames = properties.getProperty(HiveSerdeKeyManagement.HIVE_ENCRYPT_KEYNAMES); + + if (keyNames == null || keyNames.isEmpty()) { + isEncode = false; + LOG.warn("Please set " + HiveSerdeKeyManagement.HIVE_ENCRYPT_KEYNAMES); + return; + } + + String kmsUri = conf.get(HiveSerdeKeyManagement.KMS_URI); + if (kmsUri == null) { + isEncode = false; + LOG.warn("Please set " + HiveSerdeKeyManagement.KMS_URI); + return; + } + + KeyVersion kv = null; + try { + URL url = new URL(kmsUri); + URI uri = HiveSerdeKeyManagement.createKMSUri(url); + KMSClientProvider kp = new KMSClientProvider(uri, conf); + // now just support all columns share one key + String keyName = AESUtil.getKeys(keyNames).get(0); + kv = kp.getCurrentKey(keyName); + if (kv == null) { + throw new IOException("Cant get the key when do ser/deser"); + } + } catch (URISyntaxException e) { + throw new IOException("Bad configuration of " + HiveSerdeKeyManagement.KMS_URI + + " at " + kmsUri, e); + } catch (AuthorizationException e) { + isEncode = false; + return; + } + + try { + cipher = CipherFactory.getInstance(AESConstants.ALGORITHM, AESConstants.PROVIDER); + } catch (Exception e) { + throw new IOException("Failed to initiate cipher object.", e); + } + + keyBytes = kv.getMaterial(); + ivBytes = AESUtil.decodeBytes(properties.getProperty(HiveSerdeKeyManagement.HIVE_ENCRYPT_IV)); + } + + @Override + public void encode(int index, ByteStream.Input input, ByteStream.Output output) + throws IOException { + try { + if (!isEncode) { + output.write(input.toBytes()); + return; + } + cipher.init(Cipher.ENCRYPT_MODE, AESUtil.convert2SecretKey(keyBytes), + AESUtil.convert2IvSpec(ivBytes)); + byte[] encryptedBytes = cipher.doFinal(input.toBytes()); + byte[] wrappedBytes = Base64.encodeBase64(encryptedBytes); + output.write(wrappedBytes); + } catch (Exception e) { + throw new IOException(e.getMessage()); + } + } + + @Override + public void decode(int index, ByteStream.Input input, ByteStream.Output output) + throws IOException { + try { + if (!isEncode) { + output.write(input.toBytes()); + return; + } + cipher.init(Cipher.DECRYPT_MODE, AESUtil.convert2SecretKey(keyBytes), + AESUtil.convert2IvSpec(ivBytes)); + byte[] unwrappedBytes = Base64.decodeBase64(input.toBytes()); + byte[] plaintextBytes = cipher.doFinal(unwrappedBytes); + output.write(plaintextBytes); + } catch (Exception e) { + throw new IOException(e.getMessage()); + } + } +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/aes/AESUtil.java b/serde/src/java/org/apache/hadoop/hive/serde2/aes/AESUtil.java new file mode 100644 index 0000000..0106afe --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/aes/AESUtil.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.aes; + +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.Charset; +import java.security.DigestException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import javax.crypto.SecretKey; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +import org.apache.commons.codec.binary.Base64; + +public class AESUtil { + /** + * Generate 256bit key/iv for AES-256 or 128bit key/iv for AES-128 from a string + * + * @param password + * @param cryptographicLength + * @return the key/iv byte array which is the length according to cryptographicLength + * @throws CryptoException + */ + public static byte[] hash(String password, int cryptographicLength) throws CryptoException { + try { + if(cryptographicLength == 256) + return sha256(password.getBytes("UTF-8")); + else + return sha1(password.getBytes("UTF-8")); + } catch (UnsupportedEncodingException ex) { + throw new CryptoException(ex); + } + } + + public static byte[] sha1(byte[] input) throws CryptoException { + try { + MessageDigest sha; + sha = MessageDigest.getInstance("SHA-1"); + sha.update(input); + byte[] result = sha.digest(); + return Arrays.copyOf(result, 16); + } catch (NoSuchAlgorithmException e) { + throw new CryptoException("Cannot generate native password from input key", e); + } + } + + public static byte[] sha256(byte[] input) throws CryptoException { + try { + final byte[] result = new byte[32]; + MessageDigest sha256; + sha256 = MessageDigest.getInstance("SHA-256"); + sha256.update(input); + sha256.digest(result, 0, result.length); + return result; + } catch (NoSuchAlgorithmException e) { + throw new CryptoException("Cannot generate native password from input key", e); + } catch (DigestException e) { + throw new CryptoException("Cannot generate native password from input key", e); + } + } + + // length in bits + public static byte[] randomBytes(int length) { + Random rand = new Random(); + byte[] result = new byte[length / 8]; + rand.nextBytes(result); + return result; + } + + public static byte[] randomBytes(byte[] key) { + Random rand = new Random(); + rand.nextBytes(key); + return key; + } + + public static SecretKey convert2SecretKey(byte[] key) { + return new SecretKeySpec(key, "AES"); + } + + public static IvParameterSpec convert2IvSpec(byte[] iv) { + return new IvParameterSpec(iv); + } + + public static String encodeBytes(byte[] bytes) throws UnsupportedEncodingException { + return bytes == null ? null : new String(Base64.encodeBase64(bytes), Charset.forName("UTF-8")); + } + + public static byte[] decodeBytes(String str) throws UnsupportedEncodingException { + return Base64.decodeBase64(str.getBytes(Charset.forName("UTF-8"))); + } + + // support every column has a key in future + public static List getKeys(String keyNames) { + List keys = new ArrayList(); + for (String key : keyNames.split(",")) { + keys.add(key.trim()); + } + return keys; + } +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/aes/CipherFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/aes/CipherFactory.java new file mode 100644 index 0000000..237232d --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/aes/CipherFactory.java @@ -0,0 +1,56 @@ +package org.apache.hadoop.hive.serde2.aes; + +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; + +import javax.crypto.Cipher; +import javax.crypto.NoSuchPaddingException; + +public class CipherFactory { + + /** + * Create an instance of this Cipher class, according to the algorithm string + * + * @param transformation + * @return cipher instance + * @throws NoSuchAlgorithmException + * @throws NoSuchProviderException + * @throws NoSuchPaddingException + * @throws InstantiationException + * @throws IllegalAccessException + */ + public static Cipher getInstance(String transformation) + throws NoSuchAlgorithmException, NoSuchProviderException, + NoSuchPaddingException, InstantiationException, IllegalAccessException { + return getInstance(transformation, null); + } + + /** + * Create an instance of this Cipher class, according to the algorithm string + * and provider. + * + * @param transformation + * @param provider + * @return + * @throws NoSuchAlgorithmException + * @throws NoSuchProviderException + * @throws NoSuchPaddingException + * @throws InstantiationException + * @throws IllegalAccessException + */ + public static Cipher getInstance(String transformation, String provider) + throws NoSuchAlgorithmException, NoSuchProviderException, + NoSuchPaddingException, InstantiationException, IllegalAccessException { + Cipher cipher = null; + try { + if (provider == null || provider.isEmpty()) { + cipher = Cipher.getInstance(transformation); + } else { + cipher = Cipher.getInstance(transformation, provider); + } + } catch (NoSuchProviderException e) { + cipher = Cipher.getInstance(transformation); + } + return cipher; + } +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/aes/CryptoException.java b/serde/src/java/org/apache/hadoop/hive/serde2/aes/CryptoException.java new file mode 100644 index 0000000..7007ff2 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/aes/CryptoException.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.aes; + +/** + * CryptoException is the base exception class for cryptographic related classes. + */ +public class CryptoException extends Exception { + private static final long serialVersionUID = 1L; + + /** + * Constructs a new exception with null as its detail message. + * The cause is not initialized, and may subsequently be initialized by a + * call to {@link #initCause}. + */ + public CryptoException() { + super(); + } + + /** + * Constructs a new exception with the specified detail message. The + * cause is not initialized, and may subsequently be initialized by + * a call to {@link #initCause}. + * + * @param message the detail message. The detail message is saved for + * later retrieval by the {@link #getMessage()} method. + */ + public CryptoException(String message) { + super(message); + } + + /** + * Constructs a new exception with the specified detail message and cause. + *

+ * Note that the detail message associated with cause is + * not automatically incorporated in this exception's detail message. + * + * @param message + * the detail message (which is saved for later retrieval by the + * {@link #getMessage()} method). + * @param cause + * the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A null value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + * @since 1.4 + */ + public CryptoException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructs a new exception with the specified cause and a detail message + * of (cause==null ? null : cause.toString()) (which typically + * contains the class and detail message of cause). This + * constructor is useful for exceptions that are little more than wrappers + * for other throwables (for example, + * {@link java.security.PrivilegedActionException}). + * + * @param cause + * the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A null value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + * @since 1.4 + */ + public CryptoException(Throwable cause) { + super(cause); + } +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/aes/HiveSerdeKeyManagement.java b/serde/src/java/org/apache/hadoop/hive/serde2/aes/HiveSerdeKeyManagement.java new file mode 100644 index 0000000..0596794 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/aes/HiveSerdeKeyManagement.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.aes; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.security.NoSuchAlgorithmException; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.KeyProvider; +import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion; +import org.apache.hadoop.crypto.key.kms.KMSClientProvider; +import org.apache.hadoop.security.authorize.AuthorizationException; + +public class HiveSerdeKeyManagement { + public static final String HIVE_ENCRYPT_KEYNAMES = "hive.encrypt.keynames"; + public static final String HIVE_ENCRYPT_IV = "hive.encrypt.iv"; + public static final String KMS_URI = "hadoop.security.kms.uri"; + + private static Log LOG = LogFactory.getLog(HiveSerdeKeyManagement.class); + + public static void setupTableForEncryption(Configuration conf, Map tblProps) + throws IOException { + if(tblProps == null) { + return; + } + + String keyNames = tblProps.get(HIVE_ENCRYPT_KEYNAMES); + if (keyNames == null || keyNames.isEmpty()) { + return; + } + + String kmsUri = conf.get(KMS_URI); + if (kmsUri == null) { + LOG.warn("Please set " + KMS_URI + " if you want to enable encryption"); + return; + } + + // 1. create key using kms + try { + URL url = new URL(kmsUri); + URI uri = createKMSUri(url); + KMSClientProvider kp = new KMSClientProvider(uri, conf); + for (String keyName : AESUtil.getKeys(keyNames)) { + KeyVersion kv = kp.getCurrentKey(keyName); + if (kv == null) { + kv = kp.createKey(keyName, new KeyProvider.Options(conf)); + } + } + } catch (URISyntaxException e) { + throw new IOException("Bad configuration of " + KMS_URI + " at " + kmsUri, e); + } catch (AuthorizationException e) { + throw new IOException("Current user has no permission to get/create key", e); + } catch (NoSuchAlgorithmException e) { + throw new IOException("No such algorithm when create key", e); + } + + // 2. generate iv and set to table properties + byte[] ivBytes = AESUtil.randomBytes(AESConstants.IV_LENGTH); + tblProps.put(HIVE_ENCRYPT_IV, AESUtil.encodeBytes(ivBytes)); + } + + public static URI createKMSUri(URL kmsUrl) throws URISyntaxException { + String str = kmsUrl.toString(); + str = str.replaceFirst("://", "@"); + return new URI("kms://" + str); + } +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java index edea98d..5dca32c 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java @@ -390,7 +390,7 @@ private static FieldRewriter createRewriter(String encoderClass, Properties prop try { FieldRewriter rewriter = (FieldRewriter) ReflectionUtils.newInstance(Class.forName(encoderClass), job); - rewriter.init(parameters.columnNames, parameters.columnTypes, properties); + rewriter.init(parameters.columnNames, parameters.columnTypes, properties, job); return rewriter; } catch (Exception e) { throw new SerDeException(e); diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java index f354fb7..ecc81fb 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java @@ -163,6 +163,7 @@ public static String getMajorVersion() { case 1: return "0.20S"; case 2: + case 3: return "0.23"; default: throw new IllegalArgumentException("Unrecognized Hadoop major version number: " + vers);