diff --git data/files/datasets/testdataset/cleanup.hive.sql data/files/datasets/testdataset/cleanup.hive.sql new file mode 100644 index 0000000000..89275fbc0c --- /dev/null +++ data/files/datasets/testdataset/cleanup.hive.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS testdataset; diff --git data/files/datasets/testdataset/load.hive.sql data/files/datasets/testdataset/load.hive.sql new file mode 100644 index 0000000000..05c1bca2ee --- /dev/null +++ data/files/datasets/testdataset/load.hive.sql @@ -0,0 +1,13 @@ +set hive.stats.dbclass=fs; +-- +-- Table testdataset +-- +DROP TABLE IF EXISTS testdataset; + +CREATE TABLE testdataset (key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/testdataset.txt" INTO TABLE testdataset; + +ANALYZE TABLE testdataset COMPUTE STATISTICS; + +ANALYZE TABLE testdataset COMPUTE STATISTICS FOR COLUMNS key,value; diff --git data/files/testdataset.txt data/files/testdataset.txt new file mode 100644 index 0000000000..9825414ecf --- /dev/null +++ data/files/testdataset.txt @@ -0,0 +1,500 @@ +238val_238 +86val_86 +311val_311 +27val_27 +165val_165 +409val_409 +255val_255 +278val_278 +98val_98 +484val_484 +265val_265 +193val_193 +401val_401 +150val_150 +273val_273 +224val_224 +369val_369 +66val_66 +128val_128 +213val_213 +146val_146 +406val_406 +429val_429 +374val_374 +152val_152 +469val_469 +145val_145 +495val_495 +37val_37 +327val_327 +281val_281 +277val_277 +209val_209 +15val_15 +82val_82 +403val_403 +166val_166 +417val_417 +430val_430 +252val_252 +292val_292 +219val_219 +287val_287 +153val_153 +193val_193 +338val_338 +446val_446 +459val_459 +394val_394 +237val_237 +482val_482 +174val_174 +413val_413 +494val_494 +207val_207 +199val_199 +466val_466 +208val_208 +174val_174 +399val_399 +396val_396 +247val_247 +417val_417 +489val_489 +162val_162 +377val_377 +397val_397 +309val_309 +365val_365 +266val_266 +439val_439 +342val_342 +367val_367 +325val_325 +167val_167 +195val_195 +475val_475 +17val_17 +113val_113 +155val_155 +203val_203 +339val_339 +0val_0 +455val_455 +128val_128 +311val_311 +316val_316 +57val_57 +302val_302 +205val_205 +149val_149 +438val_438 +345val_345 +129val_129 +170val_170 +20val_20 +489val_489 +157val_157 +378val_378 +221val_221 +92val_92 +111val_111 +47val_47 +72val_72 +4val_4 +280val_280 +35val_35 +427val_427 +277val_277 +208val_208 +356val_356 +399val_399 +169val_169 +382val_382 +498val_498 +125val_125 +386val_386 +437val_437 +469val_469 +192val_192 +286val_286 +187val_187 +176val_176 +54val_54 +459val_459 +51val_51 +138val_138 +103val_103 +239val_239 +213val_213 +216val_216 +430val_430 +278val_278 +176val_176 +289val_289 +221val_221 +65val_65 +318val_318 +332val_332 +311val_311 +275val_275 +137val_137 +241val_241 +83val_83 +333val_333 +180val_180 +284val_284 +12val_12 +230val_230 +181val_181 +67val_67 +260val_260 +404val_404 +384val_384 +489val_489 +353val_353 +373val_373 +272val_272 +138val_138 +217val_217 +84val_84 +348val_348 +466val_466 +58val_58 +8val_8 +411val_411 +230val_230 +208val_208 +348val_348 +24val_24 +463val_463 +431val_431 +179val_179 +172val_172 +42val_42 +129val_129 +158val_158 +119val_119 +496val_496 +0val_0 +322val_322 +197val_197 +468val_468 +393val_393 +454val_454 +100val_100 +298val_298 +199val_199 +191val_191 +418val_418 +96val_96 +26val_26 +165val_165 +327val_327 +230val_230 +205val_205 +120val_120 +131val_131 +51val_51 +404val_404 +43val_43 +436val_436 +156val_156 +469val_469 +468val_468 +308val_308 +95val_95 +196val_196 +288val_288 +481val_481 +457val_457 +98val_98 +282val_282 +197val_197 +187val_187 +318val_318 +318val_318 +409val_409 +470val_470 +137val_137 +369val_369 +316val_316 +169val_169 +413val_413 +85val_85 +77val_77 +0val_0 +490val_490 +87val_87 +364val_364 +179val_179 +118val_118 +134val_134 +395val_395 +282val_282 +138val_138 +238val_238 +419val_419 +15val_15 +118val_118 +72val_72 +90val_90 +307val_307 +19val_19 +435val_435 +10val_10 +277val_277 +273val_273 +306val_306 +224val_224 +309val_309 +389val_389 +327val_327 +242val_242 +369val_369 +392val_392 +272val_272 +331val_331 +401val_401 +242val_242 +452val_452 +177val_177 +226val_226 +5val_5 +497val_497 +402val_402 +396val_396 +317val_317 +395val_395 +58val_58 +35val_35 +336val_336 +95val_95 +11val_11 +168val_168 +34val_34 +229val_229 +233val_233 +143val_143 +472val_472 +322val_322 +498val_498 +160val_160 +195val_195 +42val_42 +321val_321 +430val_430 +119val_119 +489val_489 +458val_458 +78val_78 +76val_76 +41val_41 +223val_223 +492val_492 +149val_149 +449val_449 +218val_218 +228val_228 +138val_138 +453val_453 +30val_30 +209val_209 +64val_64 +468val_468 +76val_76 +74val_74 +342val_342 +69val_69 +230val_230 +33val_33 +368val_368 +103val_103 +296val_296 +113val_113 +216val_216 +367val_367 +344val_344 +167val_167 +274val_274 +219val_219 +239val_239 +485val_485 +116val_116 +223val_223 +256val_256 +263val_263 +70val_70 +487val_487 +480val_480 +401val_401 +288val_288 +191val_191 +5val_5 +244val_244 +438val_438 +128val_128 +467val_467 +432val_432 +202val_202 +316val_316 +229val_229 +469val_469 +463val_463 +280val_280 +2val_2 +35val_35 +283val_283 +331val_331 +235val_235 +80val_80 +44val_44 +193val_193 +321val_321 +335val_335 +104val_104 +466val_466 +366val_366 +175val_175 +403val_403 +483val_483 +53val_53 +105val_105 +257val_257 +406val_406 +409val_409 +190val_190 +406val_406 +401val_401 +114val_114 +258val_258 +90val_90 +203val_203 +262val_262 +348val_348 +424val_424 +12val_12 +396val_396 +201val_201 +217val_217 +164val_164 +431val_431 +454val_454 +478val_478 +298val_298 +125val_125 +431val_431 +164val_164 +424val_424 +187val_187 +382val_382 +5val_5 +70val_70 +397val_397 +480val_480 +291val_291 +24val_24 +351val_351 +255val_255 +104val_104 +70val_70 +163val_163 +438val_438 +119val_119 +414val_414 +200val_200 +491val_491 +237val_237 +439val_439 +360val_360 +248val_248 +479val_479 +305val_305 +417val_417 +199val_199 +444val_444 +120val_120 +429val_429 +169val_169 +443val_443 +323val_323 +325val_325 +277val_277 +230val_230 +478val_478 +178val_178 +468val_468 +310val_310 +317val_317 +333val_333 +493val_493 +460val_460 +207val_207 +249val_249 +265val_265 +480val_480 +83val_83 +136val_136 +353val_353 +172val_172 +214val_214 +462val_462 +233val_233 +406val_406 +133val_133 +175val_175 +189val_189 +454val_454 +375val_375 +401val_401 +421val_421 +407val_407 +384val_384 +256val_256 +26val_26 +134val_134 +67val_67 +384val_384 +379val_379 +18val_18 +462val_462 +492val_492 +100val_100 +298val_298 +9val_9 +341val_341 +498val_498 +146val_146 +458val_458 +362val_362 +186val_186 +285val_285 +348val_348 +167val_167 +18val_18 +273val_273 +183val_183 +281val_281 +344val_344 +97val_97 +469val_469 +315val_315 +84val_84 +28val_28 +37val_37 +448val_448 +152val_152 +348val_348 +307val_307 +194val_194 +414val_414 +477val_477 +222val_222 +126val_126 +90val_90 +169val_169 +403val_403 +400val_400 +200val_200 +97val_97 diff --git data/scripts/q_test_init_src.sql data/scripts/q_test_init_src.sql index 2a62d29e34..765d6c7269 100644 --- data/scripts/q_test_init_src.sql +++ data/scripts/q_test_init_src.sql @@ -7,4 +7,3 @@ LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.txt" OVERWRITE INTO TABLE ANALYZE TABLE src COMPUTE STATISTICS; ANALYZE TABLE src COMPUTE STATISTICS FOR COLUMNS key,value; - diff --git itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestDatasetCliDriver.java itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestDatasetCliDriver.java new file mode 100644 index 0000000000..ef070ae9fb --- /dev/null +++ itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestDatasetCliDriver.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.cli; + +import java.io.File; +import java.util.List; + +import org.apache.hadoop.hive.cli.control.CliAdapter; +import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class TestDatasetCliDriver { + + static CliAdapter adapter = new CliConfigs.DatasetTestConfig().getCliAdapter(); + + @Parameters(name = "{0}") + public static List getParameters() throws Exception { + return adapter.getParameters(); + } + + @ClassRule + public static TestRule cliClassRule = adapter.buildClassRule(); + + @Rule + public TestRule cliTestRule = adapter.buildTestRule(); + + private String name; + private File qfile; + + public TestDatasetCliDriver(String name, File qfile) { + this.name = name; + this.qfile = qfile; + } + + @Test + public void testCliDriver() throws Exception { + adapter.runTest(name, qfile); + } + +} diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java index 72336e2624..d3291a17aa 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java @@ -37,6 +37,10 @@ import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.FsType; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; +import org.apache.hadoop.hive.ql.dataset.Dataset; +import org.apache.hadoop.hive.ql.dataset.DatasetCollection; +import org.apache.hadoop.hive.ql.dataset.DatasetParser; + import com.google.common.base.Splitter; import com.google.common.collect.Sets; @@ -70,9 +74,10 @@ // moved...this may change private Set includeQueryFileNames; private Class cliAdapter; + private DatasetCollection datasets; public AbstractCliConfig(Class adapter) { - cliAdapter=adapter; + cliAdapter = adapter; clusterType = MiniClusterType.none; queryFile = getSysPropValue("qfile"); queryFileRegex = getSysPropValue("qfile_regex"); @@ -85,7 +90,7 @@ private static String getHiveRoot() { try { candidateSiblings.add(new File(System.getProperty("hive.root")).getCanonicalPath()); } catch (IOException e) { - throw new RuntimeException("error getting hive.root",e); + throw new RuntimeException("error getting hive.root", e); } } candidateSiblings.add(new File(".").getAbsolutePath()); @@ -161,7 +166,6 @@ protected void excludeQuery(String qFile) { excludedQueryFileNames.add(qFile); } - private static final Splitter TEST_SPLITTER = Splitter.onPattern("[, ]").trimResults().omitEmptyStrings(); @@ -261,9 +265,25 @@ public boolean accept(File filePath) { testFiles.remove(new File(queryDir, qFileName)); } + prepareDatasets(testFiles); + return testFiles; } + private void prepareDatasets(Set testFiles) { + DatasetParser parser = new DatasetParser(); + + for (File testFile : testFiles) { + parser.parse(testFile); + } + + this.datasets = parser.getDatasets(); + } + + public DatasetCollection getDatasets() { + return datasets; + } + private void prepareDirs() throws Exception { File hiveRootDir = new File(HIVE_ROOT); if (!hiveRootDir.exists()) { diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java index dd6f15c3be..29d8031160 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java @@ -227,7 +227,6 @@ public EncryptedHDFSCliConfig() { setInitScript("q_test_init_src.sql"); setCleanupScript("q_test_cleanup_src.sql"); - setClusterType(MiniClusterType.mr); setFsType(QTestUtil.FsType.encrypted_hdfs); if (getClusterType() == MiniClusterType.tez) { @@ -476,6 +475,27 @@ public BeeLineConfig() { } } + public static class DatasetTestConfig extends AbstractCliConfig { + public DatasetTestConfig() { + super(CoreCliDriver.class); + try { + setQueryDir("ql/src/test/queries/clientpositive/dataset"); + + setResultsDir("ql/src/test/results/clientpositive/dataset"); + setLogDir("itests/qtest/target/qfile-results/dataset"); + + // we won't need these any more: HIVE-18051 + // setInitScript("q_test_init_src.sql"); + // setCleanupScript("q_test_cleanup_src.sql"); + + setHiveConfDir(""); + setClusterType(MiniClusterType.none); + } catch (Exception e) { + throw new RuntimeException("can't construct cliconfig", e); + } + } + } + public static class AccumuloCliConfig extends AbstractCliConfig { public AccumuloCliConfig() { super(CoreAccumuloCliDriver.class); diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java index d97ab3bf98..584207ed8f 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java @@ -22,9 +22,6 @@ import java.util.concurrent.TimeUnit; -import com.google.common.base.Stopwatch; -import com.google.common.base.Strings; -import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType; import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; @@ -36,6 +33,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Stopwatch; +import com.google.common.base.Strings; + public class CoreCliDriver extends CliAdapter { private static final Logger LOG = LoggerFactory.getLogger(CoreCliDriver.class); @@ -48,13 +48,15 @@ public CoreCliDriver(AbstractCliConfig testCliConfig) { @Override @BeforeClass public void beforeClass() { - String message = "Starting " + CoreCliDriver.class.getName() + " run at " + System.currentTimeMillis(); + String message = + "Starting " + CoreCliDriver.class.getName() + " run at " + System.currentTimeMillis(); LOG.info(message); System.err.println(message); - final MiniClusterType miniMR =cliConfig.getClusterType(); + final MiniClusterType miniMR = cliConfig.getClusterType(); final String hiveConfDir = cliConfig.getHiveConfDir(); final String initScript = cliConfig.getInitScript(); final String cleanupScript = cliConfig.getCleanupScript(); + try { final String hadoopVer = cliConfig.getHadoopVersion(); @@ -62,7 +64,8 @@ public void beforeClass() { @Override public QTestUtil invokeInternal() throws Exception { return new QTestUtil((cliConfig.getResultsDir()), (cliConfig.getLogDir()), miniMR, - hiveConfDir, hadoopVer, initScript, cleanupScript, true, cliConfig.getFsType()); + hiveConfDir, hadoopVer, initScript, cleanupScript, true, cliConfig.getFsType(), + cliConfig.getDatasets()); } }.invoke("QtestUtil instance created", LOG, true); @@ -87,7 +90,7 @@ public Void invokeInternal() throws Exception { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); System.err.flush(); - throw new RuntimeException("Unexpected exception in static initialization",e); + throw new RuntimeException("Unexpected exception in static initialization", e); } } @@ -148,8 +151,9 @@ public Void invokeInternal() throws Exception { } } - static String debugHint = "\nSee ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, " - + "or check ./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific test cases logs."; + static String debugHint = + "\nSee ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, " + + "or check ./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific test cases logs."; @Override public void runTest(String tname, String fname, String fpath) throws Exception { @@ -178,17 +182,16 @@ public void runTest(String tname, String fname, String fpath) throws Exception { QTestProcessExecResult result = qt.checkCliDriverResults(fname); if (result.getReturnCode() != 0) { failed = true; - String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? - debugHint : "\r\n" + result.getCapturedOutput(); + String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? debugHint + : "\r\n" + result.getCapturedOutput(); qt.failedDiff(result.getReturnCode(), fname, message); } - } - catch (Exception e) { + } catch (Exception e) { failed = true; qt.failed(e, fname, debugHint); } finally { - String message = "Done query" + fname + ". succeeded=" + !failed + ", skipped=" + skipped + - ". ElapsedTime(ms)=" + sw.stop().elapsed(TimeUnit.MILLISECONDS); + String message = "Done query" + fname + ". succeeded=" + !failed + ", skipped=" + skipped + + ". ElapsedTime(ms)=" + sw.stop().elapsed(TimeUnit.MILLISECONDS); LOG.info(message); System.err.println(message); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index f79dbac573..c51ee7b5da 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -52,6 +52,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.Deque; import java.util.EnumSet; @@ -102,6 +103,8 @@ import org.apache.hadoop.hive.llap.io.api.LlapProxy; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.Index; +import org.apache.hadoop.hive.ql.dataset.Dataset; +import org.apache.hadoop.hive.ql.dataset.DatasetCollection; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.Utilities; @@ -163,6 +166,7 @@ private String testWarehouse; private final String testFiles; + private final File datasetDir; protected final String outDir; protected final String logDir; private final TreeMap qMap; @@ -174,8 +178,9 @@ private final Set qNoSessionReuseQuerySet; private final Set qJavaVersionSpecificOutput; private static final String SORT_SUFFIX = ".sorted"; - private final Set srcTables; + private static Set srcTables; private final Set srcUDFs; + private final DatasetCollection datasets; private final MiniClusterType clusterType; private final FsType fsType; private ParseDriver pd; @@ -209,7 +214,15 @@ private HBaseTestingUtility utility; public static Set getSrcTables() { - HashSet srcTables = new HashSet(); + if (srcTables == null){ + initSrcTables(); + } + + return Collections.unmodifiableSet(srcTables); + } + + private static void initSrcTables(){ + srcTables = new HashSet(); // FIXME: moved default value to here...for now // i think this features is never really used from the command line String defaultTestSrcTables = "src,src1,srcbucket,srcbucket2,src_json,src_thrift," + @@ -224,9 +237,16 @@ if (srcTables.isEmpty()) { throw new RuntimeException("Source tables cannot be empty"); } - return srcTables; + } + + private CliDriver getCliDriver() { + if(cliDriver == null){ + cliDriver = new CliDriver(); + } + return cliDriver; } + /** * Returns the default UDF names which should not be removed when resetting the test database * @return The list of the UDF names not to remove @@ -529,18 +549,18 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String confDir, String hadoopVer, String initScript, String cleanupScript, boolean withLlapIo) throws Exception { this(outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript, - withLlapIo, null); + withLlapIo, null, null); } public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String confDir, String hadzoopVer, String initScript, String cleanupScript, - boolean withLlapIo, FsType fsType) + boolean withLlapIo, FsType fsType, DatasetCollection datasets) throws Exception { LOG.info("Setting up QTestUtil with outDir={}, logDir={}, clusterType={}, confDir={}," + " hadoopVer={}, initScript={}, cleanupScript={}, withLlapIo={}," + - " fsType={}" + " fsType={}, datasets={}" , outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript, - withLlapIo, fsType); + withLlapIo, fsType, datasets); Preconditions.checkNotNull(clusterType, "ClusterType cannot be null"); if (fsType != null) { this.fsType = fsType; @@ -549,9 +569,11 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, } this.outDir = outDir; this.logDir = logDir; - this.srcTables=getSrcTables(); + initSrcTables(); this.srcUDFs = getSrcUDFs(); - + this.datasets = datasets == null ? new DatasetCollection() : datasets; + srcTables.addAll(datasets.getTables()); + // HIVE-14443 move this fall-back logic to CliConfigs if (confDir != null && !confDir.isEmpty()) { HiveConf.setHiveSiteLocation(new URL("file://"+ new File(confDir).toURI().getPath() + "/hive-site.xml")); @@ -595,6 +617,11 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, } testFiles = dataDir; + // Use path relative to dataDir directory if it is not specified + datasetDir = conf.get("test.data.set.files") == null + ? new File(new File(dataDir).getAbsolutePath() + "/datasets") + : new File(conf.get("test.data.set.files")); + // Use the current directory if it is not specified String scriptsDir = conf.get("test.data.scripts"); if (scriptsDir == null) { @@ -1022,23 +1049,10 @@ public void cleanUp(String tname) throws Exception { clearUDFsCreatedDuringTests(); clearKeysCreatedInTests(); - File cleanupFile = new File(cleanupScript); - if (cleanupFile.isFile()) { - String cleanupCommands = readEntireFileIntoString(cleanupFile); - LOG.info("Cleanup (" + cleanupScript + "):\n" + cleanupCommands); - if(cliDriver == null) { - cliDriver = new CliDriver(); - } - SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", true); - int result = cliDriver.processLine(cleanupCommands); - if (result != 0) { - LOG.error("Failed during cleanup processLine with code={}. Ignoring", result); - // TODO Convert this to an Assert.fail once HIVE-14682 is fixed - } - SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", false); - } else { - LOG.info("No cleanup script detected. Skipping."); - } + SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", true); + cleanupFromFile(); + cleanupDatasets(); + SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", false); // delete any contents in the warehouse dir Path p = new Path(testWarehouse); @@ -1059,6 +1073,22 @@ public void cleanUp(String tname) throws Exception { FunctionRegistry.unregisterTemporaryUDF("test_error"); } + private void cleanupFromFile() throws IOException { + File cleanupFile = new File(cleanupScript); + if (cleanupFile.isFile()) { + String cleanupCommands = readEntireFileIntoString(cleanupFile); + LOG.info("Cleanup (" + cleanupScript + "):\n" + cleanupCommands); + + int result = getCliDriver().processLine(cleanupCommands); + if (result != 0) { + LOG.error("Failed during cleanup processLine with code={}. Ignoring", result); + // TODO Convert this to an Assert.fail once HIVE-14682 is fixed + } + } else { + LOG.info("No cleanup script detected. Skipping."); + } + } + protected void runCreateTableCmd(String createTableCmd) throws Exception { int ecode = 0; ecode = drv.run(createTableCmd).getResponseCode(); @@ -1091,29 +1121,67 @@ public void createSources(String tname) throws Exception { startSessionState(canReuseSession); } - if(cliDriver == null) { - cliDriver = new CliDriver(); - } - cliDriver.processLine("set test.data.dir=" + testFiles + ";"); + getCliDriver().processLine("set test.data.dir=" + testFiles + ";"); + + conf.setBoolean("hive.test.init.phase", true); + + initFromScript(); + initDatasets(); + + conf.setBoolean("hive.test.init.phase", false); + } + + private void initFromScript() throws IOException { File scriptFile = new File(this.initScript); if (!scriptFile.isFile()) { LOG.info("No init script detected. Skipping"); return; } - conf.setBoolean("hive.test.init.phase", true); - + String initCommands = readEntireFileIntoString(scriptFile); LOG.info("Initial setup (" + initScript + "):\n" + initCommands); - int result = cliDriver.processLine(initCommands); + int result = getCliDriver().processLine(initCommands); LOG.info("Result from cliDrriver.processLine in createSources=" + result); if (result != 0) { Assert.fail("Failed during createSources processLine with code=" + result); } + } - conf.setBoolean("hive.test.init.phase", false); + private void initDatasets() throws IOException { + for (String table : datasets.getTables()) { + File tableFile = new File(new File(datasetDir, table), Dataset.INIT_FILE_NAME); + String commands = readEntireFileIntoString(tableFile); + + int result = getCliDriver().processLine(commands); + LOG.info("Result from cliDrriver.processLine in initFromDatasets=" + result); + if (result != 0) { + Assert.fail("Failed during initFromDatasets processLine with code=" + result); + } + + srcTables.add(table); + } } + private void cleanupDatasets() throws IOException { + for (String table : datasets.getTables()) { + File cleanupFile = new File(new File(datasetDir, table), Dataset.CLEANUP_FILE_NAME); + if (cleanupFile.isFile()) { + String cleanupCommands = readEntireFileIntoString(cleanupFile); + + LOG.info("Cleanup (" + table + "):\n" + cleanupCommands); + + int result = getCliDriver().processLine(cleanupCommands); + if (result != 0) { + LOG.error("Failed during cleanup processLine with code={}. Ignoring", result); + } + } else { + LOG.info( + String.format("No cleanup script detected for dataset table: '%s'. Skipping.", table)); + } + } + } + public void init() throws Exception { // Create remote dirs once. @@ -1135,7 +1203,7 @@ public void init() throws Exception { public void init(String tname) throws Exception { cleanUp(tname); createSources(tname); - cliDriver.processCmd("set hive.cli.print.header=true;"); + getCliDriver().processCmd("set hive.cli.print.header=true;"); } public void cliInit(String tname) throws Exception { @@ -1201,12 +1269,10 @@ public String cliInit(String tname, boolean recreate) throws Exception { } SessionState.start(ss); - cliDriver = new CliDriver(); - if (tname.equals("init_file.q")) { ss.initFiles.add(AbstractCliConfig.HIVE_ROOT + "/data/scripts/test_init_file.sql"); } - cliDriver.processInitFiles(ss); + getCliDriver().processInitFiles(ss); return outf.getAbsolutePath(); } @@ -1259,7 +1325,7 @@ public int executeAdhocCommand(String q) { String q1 = q.split(";")[0] + ";"; LOG.debug("Executing " + q1); - return cliDriver.processLine(q1); + return getCliDriver().processLine(q1); } public int executeOne(String tname) { @@ -1274,7 +1340,7 @@ public int executeOne(String tname) { qMap.put(tname, qrest); System.out.println("Executing " + q1); - return cliDriver.processLine(q1); + return getCliDriver().processLine(q1); } public int execute(String tname) { @@ -1319,7 +1385,7 @@ private int executeClientInternal(String commands) { if (isCommandUsedForTesting(command)) { rc = executeTestCommand(command); } else { - rc = cliDriver.processLine(command); + rc = getCliDriver().processLine(command); } if (rc != 0 && !ignoreErrors()) { diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/dataset/Dataset.java itests/util/src/main/java/org/apache/hadoop/hive/ql/dataset/Dataset.java new file mode 100644 index 0000000000..2e439653df --- /dev/null +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/dataset/Dataset.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.dataset; + +import java.util.Set; + +/** + * Dataset interface for classes which represent a dataset: a table or a set tables. + * + */ +public interface Dataset { + final String INIT_FILE_NAME = "load.hive.sql"; + final String CLEANUP_FILE_NAME = "cleanup.hive.sql"; + + Set getTables(); +} diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/dataset/DatasetCollection.java itests/util/src/main/java/org/apache/hadoop/hive/ql/dataset/DatasetCollection.java new file mode 100644 index 0000000000..3a28791945 --- /dev/null +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/dataset/DatasetCollection.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.dataset; + +import java.util.HashSet; +import java.util.Set; + +/** + * DatasetCollection. Wraps a set of tables defined in a Dataset. + * + */ +public class DatasetCollection implements Dataset { + private Set tables = new HashSet(); + + public void add(Dataset dataset) { + tables.addAll(dataset.getTables()); + } + + public void add(String dataset) { + tables.add(dataset); + } + + public Set getTables() { + return tables; + } +} diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/dataset/DatasetParser.java itests/util/src/main/java/org/apache/hadoop/hive/ql/dataset/DatasetParser.java new file mode 100644 index 0000000000..bb4237b6a8 --- /dev/null +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/dataset/DatasetParser.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.dataset; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +public class DatasetParser { + + private DatasetCollection datasets = new DatasetCollection(); + public static final String DATASET_PREFIX = "--! qt:dataset:"; + + public void parse(File file) { + try (BufferedReader br = new BufferedReader(new FileReader(file))) { + for (String line = br.readLine(); line != null; line = br.readLine()) { + if (line.trim().startsWith(DATASET_PREFIX)) { + Set strDatasets = parseDatasetsFromLine(line); + + for (String strDataset : strDatasets) { + datasets.add(strDataset); + } + } + } + } catch (IOException e) { + throw new RuntimeException("cannot find qfile while parsing for datasets", e); + } + } + + public DatasetCollection getDatasets() { + return datasets; + } + + public static Set parseDatasetsFromLine(String input) { + Set datasets = new HashSet(); + + input = input.substring(DATASET_PREFIX.length()); + if (!input.trim().isEmpty()) { + datasets.addAll(Arrays.asList(input.split(","))); + } + + return datasets; + } + +} diff --git itests/util/src/test/java/org/apache/hadoop/hive/ql/dataset/TestDatasetParser.java itests/util/src/test/java/org/apache/hadoop/hive/ql/dataset/TestDatasetParser.java new file mode 100644 index 0000000000..1df7e96563 --- /dev/null +++ itests/util/src/test/java/org/apache/hadoop/hive/ql/dataset/TestDatasetParser.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.dataset; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.junit.Assert; +import org.junit.Test; + +public class TestDatasetParser { + + @Test + public void testParseOne() { + Set expected = new HashSet(Arrays.asList("mydataset")); + Assert.assertEquals(expected, DatasetParser + .parseDatasetsFromLine(String.format("%smydataset", DatasetParser.DATASET_PREFIX))); + } + + @Test + public void testParseMultiple() { + Set expected = new HashSet(Arrays.asList("one", "two", "three")); + Assert.assertEquals(expected, DatasetParser + .parseDatasetsFromLine(String.format("%sone,two,three", DatasetParser.DATASET_PREFIX))); + } + + @Test + public void testParseUnique() { + Set expected = new HashSet(Arrays.asList("one", "two")); + Assert.assertEquals(expected, DatasetParser + .parseDatasetsFromLine(String.format("%sone,one,two", DatasetParser.DATASET_PREFIX))); + } + + @Test + public void testParseNone() { + Assert.assertTrue(DatasetParser + .parseDatasetsFromLine(String.format("%s", DatasetParser.DATASET_PREFIX)).isEmpty()); + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java index 107ce683d1..8437f47834 100644 --- ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java +++ ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java @@ -28,14 +28,14 @@ import org.apache.hadoop.security.UserGroupInformation; /** - * Implementation of a pre execute hook that prevents modifications - * of read-only tables used by the test framework + * Implementation of a pre execute hook that prevents modifications of read-only tables used by the + * test framework */ public class EnforceReadOnlyTables implements ExecuteWithHookContext { - private static final Set READ_ONLY_TABLES = new HashSet(); + private final Set READ_ONLY_TABLES = new HashSet(); - static { + public EnforceReadOnlyTables() { for (String srcTable : System.getProperty("test.src.tables", "").trim().split(",")) { srcTable = srcTable.trim(); if (!srcTable.isEmpty()) { @@ -53,25 +53,23 @@ public void run(HookContext hookContext) throws Exception { Set inputs = hookContext.getInputs(); Set outputs = hookContext.getOutputs(); UserGroupInformation ugi = hookContext.getUgi(); - this.run(ss,inputs,outputs,ugi); + this.run(ss, inputs, outputs, ugi); } - public void run(SessionState sess, Set inputs, - Set outputs, UserGroupInformation ugi) - throws Exception { + public void run(SessionState sess, Set inputs, Set outputs, + UserGroupInformation ugi) throws Exception { // Don't enforce during test driver setup or shutdown. - if (sess.getConf().getBoolean("hive.test.init.phase", false) || - sess.getConf().getBoolean("hive.test.shutdown.phase", false)) { + if (sess.getConf().getBoolean("hive.test.init.phase", false) + || sess.getConf().getBoolean("hive.test.shutdown.phase", false)) { return; } - for (WriteEntity w: outputs) { - if ((w.getTyp() == WriteEntity.Type.TABLE) || - (w.getTyp() == WriteEntity.Type.PARTITION)) { + for (WriteEntity w : outputs) { + if ((w.getTyp() == WriteEntity.Type.TABLE) || (w.getTyp() == WriteEntity.Type.PARTITION)) { Table t = w.getTable(); if (DEFAULT_DATABASE_NAME.equalsIgnoreCase(t.getDbName()) && READ_ONLY_TABLES.contains(t.getTableName())) { - throw new RuntimeException ("Cannot overwrite read-only table: " + t.getTableName()); + throw new RuntimeException("Cannot overwrite read-only table: " + t.getTableName()); } } } diff --git ql/src/test/queries/clientpositive/dataset/testdataset.q ql/src/test/queries/clientpositive/dataset/testdataset.q new file mode 100644 index 0000000000..96b78adfd0 --- /dev/null +++ ql/src/test/queries/clientpositive/dataset/testdataset.q @@ -0,0 +1,2 @@ +--! qt:dataset:testdataset +select count(*) from testdataset; diff --git ql/src/test/results/clientpositive/dataset/testdataset.q.out ql/src/test/results/clientpositive/dataset/testdataset.q.out new file mode 100644 index 0000000000..2482befdb3 --- /dev/null +++ ql/src/test/results/clientpositive/dataset/testdataset.q.out @@ -0,0 +1,9 @@ +PREHOOK: query: select count(*) from testdataset +PREHOOK: type: QUERY +PREHOOK: Input: default@testdataset +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from testdataset +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testdataset +#### A masked pattern was here #### +500