diff --git bin/beeline.cmd bin/beeline.cmd deleted file mode 100644 index 971e20b..0000000 --- bin/beeline.cmd +++ /dev/null @@ -1,66 +0,0 @@ -@echo off -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -SetLocal EnableDelayedExpansion - -pushd %CD%\.. -if not defined HIVE_HOME ( - set HIVE_HOME=%CD% -) -popd - -if "%HADOOP_BIN_PATH:~-1%" == "\" ( - set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% -) - -if not defined JAVA_HOME ( - echo Error: JAVA_HOME is not set. - goto :eof -) - -@rem get the hadoop envrionment -if not exist %HADOOP_HOME%\libexec\hadoop-config.cmd ( - @echo +================================================================+ - @echo ^| Error: HADOOP_HOME is not set correctly ^| - @echo +----------------------------------------------------------------+ - @echo ^| Please set your HADOOP_HOME variable to the absolute path of ^| - @echo ^| the directory that contains \libexec\hadoop-config.cmd ^| - @echo +================================================================+ - exit /b 1 -) -@rem supress the HADOOP_HOME warnings in 1.x.x -set HADOOP_HOME_WARN_SUPPRESS=true - -@rem include only the beeline client jar and its dependencies -pushd %HIVE_HOME%\lib -for /f %%a IN ('dir /b hive-beeline-**.jar') do ( - set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%HIVE_HOME%\lib\%%a -) -for /f %%a IN ('dir /b super-csv-**.jar') do ( - set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%HIVE_HOME%\lib\%%a -) -for /f %%a IN ('dir /b jline-**.jar') do ( - set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%HIVE_HOME%\lib\%%a -) -for /f %%a IN ('dir /b hive-jdbc-**-standalone.jar') do ( - set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%HIVE_HOME%\lib\%%a -) -popd -set HADOOP_USER_CLASSPATH_FIRST=true -call %HADOOP_HOME%\libexec\hadoop-config.cmd - -call %JAVA_HOME%\bin\java %JAVA_HEAP_MAX% %HADOOP_OPTS% -classpath %CLASSPATH% org.apache.hive.beeline.BeeLine %* - -endlocal diff --git bin/derbyserver.cmd bin/derbyserver.cmd deleted file mode 100644 index 386df80..0000000 --- bin/derbyserver.cmd +++ /dev/null @@ -1,60 +0,0 @@ -@echo off -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -@rem -setlocal enabledelayedexpansion - -set hadoop-config-script=%HADOOP_HOME%\libexec\yarn-config.cmd -call %hadoop-config-script% - -pushd %HIVE_HOME%\lib -for /f %%a IN ('dir /b derby*.jar') do ( - call :SetClasspath %HIVE_HOME%\lib\%%a -) -popd - -set CLASS=org.apache.derby.drda.NetworkServerControl - -if [%1]==[catservicexml] goto :derbyservice_catservice - -:derbyserver - - if "%1" == "--config" ( - shift - set HADOOP_CONF_DIR=%2 - shift - - if exist %HADOOP_CONF_DIR%\hadoop-env.cmd ( - call %HADOOP_CONF_DIR%\hadoop-env.cmd - ) - ) - - call %JAVA% %JAVA_HEAP_MAX% %HADOOP_OPTS% -classpath %CLASSPATH% %CLASS% start -h 0.0.0.0 -noSecurityManager -goto :EOF - -:SetClasspath - set CLASSPATH=%CLASSPATH%;%1 -goto :EOF - -:derbyservice_catservice -@echo ^ -@echo ^derbyserver^ -@echo ^derbyserver^ -@echo ^Derby Service^ -@echo ^%JAVA_HOME%\bin\java^ -@echo ^%JAVA_HEAP_MAX% %HADOOP_OPTS% -classpath %CLASSPATH% %CLASS% start -h 0.0.0.0 -noSecurityManager^ -@echo ^ -goto :EOF -endlocal diff --git bin/ext/cleardanglingscratchdir.cmd bin/ext/cleardanglingscratchdir.cmd deleted file mode 100644 index 1bb0453..0000000 --- bin/ext/cleardanglingscratchdir.cmd +++ /dev/null @@ -1,34 +0,0 @@ -@echo off -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. - -set CLASS=org.apache.hadoop.hive.ql.session.ClearDanglingScratchDir -set HIVE_OPTS= - -pushd %HIVE_LIB% -for /f %%a IN ('dir /b hive-exec-*.jar') do ( - set JAR=%HIVE_LIB%\%%a -) -popd - -if [%1]==[cleardanglingscratchdir_help] goto :cleardanglingscratchdir_help - -:cleardanglingscratchdir - call %HIVE_BIN_PATH%\ext\util\execHiveCmd.cmd %CLASS% -goto :EOF - -:cleardanglingscratchdir_help - echo "usage hive --service cleardanglingscratchdir" -goto :EOF diff --git bin/ext/cli.cmd bin/ext/cli.cmd deleted file mode 100644 index 74a1596..0000000 --- bin/ext/cli.cmd +++ /dev/null @@ -1,58 +0,0 @@ -@echo off -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. - -if [%1]==[cli_help] goto :cli_help - -:cli - call :update_cli - call %HIVE_BIN_PATH%\ext\util\execHiveCmd.cmd %CLASS% -goto :EOF - -:cli_help - call :update_cli - set HIVEARGS=--help - call :cli -goto :EOF - -:update_cli - if [%USE_DEPRECATED_CLI%] == [] ( - set USE_DEPRECATED_CLI=true - ) - - if /I "%USE_DEPRECATED_CLI%" == "true" ( - call :old_cli - ) else ( - call :new_cli - ) -goto :EOF - -:old_cli - set CLASS=org.apache.hadoop.hive.cli.CliDriver - pushd %HIVE_LIB% - for /f %%a IN ('dir /b hive-cli-*.jar') do ( - set JAR=%HIVE_LIB%\%%a - ) - popd -goto :EOF - -:new_cli - set CLASS=org.apache.hive.beeline.cli.HiveCli - pushd %HIVE_LIB% - for /f %%a IN ('dir /b hive-beeline-*.jar') do ( - set JAR=%HIVE_LIB%\%%a - ) - popd -goto :EOF diff --git bin/ext/debug.cmd bin/ext/debug.cmd deleted file mode 100644 index 2baf831..0000000 --- bin/ext/debug.cmd +++ /dev/null @@ -1,110 +0,0 @@ -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -@echo off - -set RECURSIVE= -set PORT= -set MAINSUSPEND= -set CHILDSUSPEND= -set SWAPSUSPEND= -set HIVE_MAIN_CLIENT_DEBUG_OPTS= -set HIVE_CHILD_CLIENT_DEBUG_OPTS= -if %1==HELP ( - goto :debug_help -) else ( - call :get_debug_params %* -) - -@rem must use java 1.5 or later prereq -:ParseDebugArgs - :ProcessDebugArgsLoop - if [%1]==[] goto :EndProcessDebug - - set params=%1 - set temp=%params:~0, 8% - - @rem trim off the --debug[ if it is the 1st param - if %temp%==--debug[ ( - set params=%params:--debug[=% - ) - - @rem trim off the ] if necessary on the value of the param - set value=%2 - set value=%value:]=% - - if %params%==recursive ( - set RECURSIVE=%value% - shift - shift - goto :ProcessDebugArgsLoop - ) - - if %params%==port ( - set PORT=%value% - shift - shift - goto :ProcessDebugArgsLoop - ) - - if %params%==mainSuspend ( - set MAINSUSPEND=%value% - shift - shift - goto :ProcessDebugArgsLoop - ) - if %params%==childSuspend ( - set CHILDSUSPEND=%value% - shift - shift - goto :ProcessDebugArgsLoop - ) - if %params%==swapSuspend ( - set childTemp=%CHILDSUSPEND% - set CHILDSUSPEND=%MAINSUSPEND% - set MAINSUSPEND=%childTemp% - shift - goto :ProcessDebugArgsLoop - ) - - shift - goto :ProcessDebugArgsLoop - - :EndProcessDebug -goto :EOF - -:set_debug_defaults - set RECURSIVE="y" - set PORT=address=8000 - set MAINSUSPEND=suspend=y - set CHILDSUSPEND=suspend=n -goto :EOF - -:get_debug_params - call :set_debug_defaults - call :ParseDebugArgs %* - set HIVE_MAIN_CLIENT_DEBUG_OPTS= -XX:+UseParallelGC -Xdebug -Xrunjdwp:transport=dt_socket,server=y,%PORT%,%MAINSUSPEND% - set HIVE_CHILD_CLIENT_DEBUG_OPTS= -XX:+UseParallelGC -Xdebug -Xrunjdwp:transport=dt_socket,server=y,%CHILDSUSPEND% -goto :EOF - -:debug_help - echo "Allows to debug Hive by connecting to it via JDI API" - echo "Usage: hive --debug[:comma-separated parameters list]" - echo "Parameters:" - echo "recursive= Should child JVMs also be started in debug mode. Default: y" - echo "port= Port on which main JVM listens for debug connection. Default: 8000" - echo "mainSuspend= Should main JVM wait with execution for the debugger to connect. Default: y" - echo "childSuspend= Should child JVMs wait with execution for the debugger to connect. Default: n" - echo "swapSuspend Swaps suspend options between main and child JVMs" -goto :EOF diff --git bin/ext/hbaseimport.cmd bin/ext/hbaseimport.cmd deleted file mode 100644 index ff69007..0000000 --- bin/ext/hbaseimport.cmd +++ /dev/null @@ -1,35 +0,0 @@ -@echo off -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. - -set CLASS=org.apache.hadoop.hive.metastore.hbase.HBaseImport -set HIVE_OPTS= -set HADOOP_CLASSPATH= - -pushd %HIVE_LIB% -for /f %%a IN ('dir /b hive-exec-*.jar') do ( - set JAR=%HIVE_LIB%\%%a -) -popd - -if [%1]==[hbaseimport_help] goto :hbaseimport_help - -:hbaseimport - call %HIVE_BIN_PATH%\ext\util\execHiveCmd.cmd %CLASS% -goto :EOF - -:hbaseimport_help - echo "usage hive --hbaseimport" -goto :EOF diff --git bin/ext/help.cmd bin/ext/help.cmd deleted file mode 100644 index abd429d..0000000 --- bin/ext/help.cmd +++ /dev/null @@ -1,30 +0,0 @@ -@echo off -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. - -:help - echo "Usage ./hive --service serviceName " - echo "Service List: $SERVICE_LIST" - echo "Parameters parsed:" - echo " --auxpath : Auxiliary jars " - echo " --config : Hive configuration directory" - echo " --service : Starts specific service/component. cli is default" - echo "Parameters used:" - echo " HADOOP_HOME or HADOOP_PREFIX : Hadoop install directory" - echo " HIVE_OPT : Hive options" - echo "For help on a particular service:" - echo " ./hive --service serviceName --help" - echo "Debug help: ./hive --debug --help" -goto :EOF diff --git bin/ext/hiveserver2.cmd bin/ext/hiveserver2.cmd deleted file mode 100644 index 26c954c..0000000 --- bin/ext/hiveserver2.cmd +++ /dev/null @@ -1,139 +0,0 @@ -@echo off -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. - -set CLASS=org.apache.hive.service.server.HiveServer2 -pushd %HIVE_LIB% -for /f %%a IN ('dir /b hive-service-*.jar') do ( - set JAR=%HIVE_LIB%\%%a -) -popd - -if defined HBASE_HOME ( - dir %HBASE_HOME%\lib > nul - if %errorlevel%==0 ( - pushd %HBASE_HOME%\lib - set HIVE_HBASE_PATH= - for /f %%a IN ('dir /b hbase-server-**-hadoop2.jar') do ( - call :AddToHiveHbasePath %HBASE_HOME%\lib\%%a - ) - for /f %%a IN ('dir /b hbase-client-**-hadoop2.jar') do ( - call :AddToHiveHbasePath %HBASE_HOME%\lib\%%a - ) - for /f %%a IN ('dir /b hbase-protocol-**-hadoop2.jar') do ( - call :AddToHiveHbasePath %HBASE_HOME%\lib\%%a - ) - for /f %%a IN ('dir /b htrace-core-**.jar') do ( - call :AddToHiveHbasePath %HBASE_HOME%\lib\%%a - ) - for /f %%a IN ('dir /b hbase-common-**-hadoop2.jar') do ( - call :AddToHiveHbasePath %HBASE_HOME%\lib\%%a - ) - for /f %%a IN ('dir /b hbase-hadoop-compat-**-hadoop2.jar') do ( - call :AddToHiveHbasePath %HBASE_HOME%\lib\%%a - ) - ) - popd - if defined HBASE_CONF_DIR ( - dir %HBASE_CONF_DIR% > nul - if %errorlevel%==0 ( - call :AddToHiveHbasePath %HBASE_CONF_DIR% - ) - ) - ) -) - -@rem add auxilary jars such as serdes -if not defined HIVE_AUX_JARS_PATH goto :AddMiscAuxLibDir - -setLocal EnableDelayedExpansion -:auxJarLoop - for /f "delims=," %%a in ("!HIVE_AUX_JARS_PATH!") do ( - set auxjar=%%a - if exist %%a ( - if exist "%%a\nul" ( - @rem %%a is a dir - pushd %%a - for /f %%b IN ('dir /b *.jar') do ( - call :AddToAuxJavaParam %%a\%%b - ) - popd - ) else ( - @rem %%a is a file - call :AddToAuxJavaParam %%a - ) - ) - ) - :striploop - set stripchar=!HIVE_AUX_JARS_PATH:~0,1! - set HIVE_AUX_JARS_PATH=!HIVE_AUX_JARS_PATH:~1! - if "!HIVE_AUX_JARS_PATH!" EQU "" goto auxJarLoopEnd - if "!stripchar!" NEQ "," goto striploop - goto auxJarLoop - -:auxJarLoopEnd - -:AddMiscAuxLibDir -@rem adding jars from hcatalog\share\hcatalog directory -if exist %HIVE_HOME%\hcatalog\share\hcatalog ( - pushd %HIVE_HOME%\hcatalog\share\hcatalog - for /f %%a IN ('dir /b *.jar') do ( - call :AddToAuxJavaParam %HIVE_HOME%\hcatalog\share\hcatalog\%%a - ) - popd -) - -if [%1]==[hiveserver2_help] goto :hiveserver2_help - -if [%1]==[hiveserver2_catservice] goto :hiveserver2_catservice - -:hiveserver2 - - @rem hadoop 20 or newer - skip the aux_jars option and hiveconf - call %HIVE_BIN_PATH%\ext\util\execHiveCmd.cmd %CLASS% -goto :EOF - -:hiveserver2_help - set HIVEARGS=-h - goto :hiveserver2 -goto :EOF - -:hiveserver2_catservice -@echo ^ -@echo ^HiveServer2^ -@echo ^HiveServer2^ -@echo ^Hadoop HiveServer2 Service^ -@echo ^%JAVA_HOME%\bin\java^ -@echo ^%JAVA_HEAP_MAX% -XX:MaxPermSize=512m %HADOOP_OPTS% -classpath %CLASSPATH%;%HIVE_HBASE_PATH%;%HIVE_HOME%\hcatalog\share\hcatalog\* %CLASS% -hiveconf hive.hadoop.classpath=%HIVE_LIB%\*;%HIVE_HOME%\hcatalog\share\hcatalog\* -hiveconf hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory -hiveconf hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator -hiveconf hive.metastore.uris=" " -hiveconf hive.aux.jars.path=%AUX_JAVA_PARAM% %HIVE_OPTS%^ -@echo ^ -goto :EOF - -:AddToHiveHbasePath -if not defined HIVE_HBASE_PATH ( - set HIVE_HBASE_PATH=%1 - ) else ( - set HIVE_HBASE_PATH=%HIVE_HBASE_PATH%;%1 - ) -) -goto :EOF - -:AddToAuxJavaParam -if not defined AUX_JAVA_PARAM ( - set AUX_JAVA_PARAM=file:///%1 - ) else ( - set AUX_JAVA_PARAM=%AUX_JAVA_PARAM%,file:///%1 - ) -) -goto :EOF \ No newline at end of file diff --git bin/ext/jar.cmd bin/ext/jar.cmd deleted file mode 100644 index e5ab875..0000000 --- bin/ext/jar.cmd +++ /dev/null @@ -1,43 +0,0 @@ -@echo off -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. - -if [%1]==[jar_help] goto :jar_help - -:jar - - set RUNJAR=%1 - shift - - set RUNCLASS=%1 - shift - - if "%RUNJAR%"== ""( - echo "RUNJAR not specified" - exit 3 - ) - - if "%RUNCLASS%" == "" ( - echo "RUNCLASS not specified" - exit 3 - ) - @rem hadoop 20 or newer - skip the aux_jars option and hiveconf - %HADOOP% jar %$RUNJAR% %RUNCLASS% %HIVE_OPTS% %* -goto :EOF - -:jar_help - echo "Used for applications that require Hadoop and Hive classpath and environment." - echo "./hive --service jar HIVE_OPTS " -goto :EOF \ No newline at end of file diff --git bin/ext/lineage.cmd bin/ext/lineage.cmd deleted file mode 100644 index 03e4f56..0000000 --- bin/ext/lineage.cmd +++ /dev/null @@ -1,30 +0,0 @@ -@echo off -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. - -set CLASS=org.apache.hadoop.hive.ql.tools.LineageInfo - -if [%1]==[lineage_help] goto :lineage_help - -:lineage - if not exist %HIVE_LIB%/hive-exec-*.jar ( - echo "Missing Hive exec Jar" - ) - call %HIVE_BIN_PATH%\ext\util\execHiveCmd.cmd %CLASS% -goto :EOF - -:lineage_help - echo "usage hive.cmd 'hql' " -goto :EOF \ No newline at end of file diff --git bin/ext/metastore.cmd bin/ext/metastore.cmd deleted file mode 100644 index 68d17ab..0000000 --- bin/ext/metastore.cmd +++ /dev/null @@ -1,47 +0,0 @@ -@echo off -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. - -set CLASS=org.apache.hadoop.hive.metastore.HiveMetaStore -pushd %HIVE_LIB% -for /f %%a IN ('dir /b hive-metastore-*.jar') do ( - set JAR=%HIVE_LIB%\%%a -) -popd -if [%1]==[metastore_help] goto :metastore_help - -if [%1]==[metastore_catservice] goto :metastore_catservice - -:metastore - echo "Starting Hive Metastore Server" - @rem hadoop 20 or newer - skip the aux_jars option and hiveconf - set HADOOP_OPTS=%HIVE_METASTORE_HADOOP_OPTS% %HADOOP_OPTS% - call %HIVE_BIN_PATH%\ext\util\execHiveCmd.cmd %CLASS% -goto :EOF - -:metastore_help - set HIVEARGS=-h - call :metastore -goto :EOF - -:metastore_catservice -@echo ^ -@echo ^Metastore^ -@echo ^Metastore^ -@echo ^Hadoop Metastore Service^ -@echo ^%JAVA_HOME%\bin\java^ -@echo ^%JAVA_HEAP_MAX% %HADOOP_OPTS% %AUX_PARAM% -classpath %CLASSPATH% %CLASS% %HIVE_OPTS%^ -@echo ^ -goto :EOF diff --git bin/ext/orcfiledump.cmd bin/ext/orcfiledump.cmd deleted file mode 100644 index ff4b410..0000000 --- bin/ext/orcfiledump.cmd +++ /dev/null @@ -1,35 +0,0 @@ -@echo off -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. - -set CLASS=org.apache.orc.tools.FileDump -set HIVE_OPTS= -set HADOOP_CLASSPATH= - -pushd %HIVE_LIB% -for /f %%a IN ('dir /b hive-exec-*.jar') do ( - set JAR=%HIVE_LIB%\%%a -) -popd - -if [%1]==[orcfiledump_help] goto :orcfiledump_help - -:orcfiledump - call %HIVE_BIN_PATH%\ext\util\execHiveCmd.cmd %CLASS% -goto :EOF - -:orcfiledump_help - echo "usage hive --orcfiledump [-d] [--rowindex=_csv_col_ids_] [-t] [-j] [-p] [--recover] [--skip-dump] [--backup-path=_path_for_backup_] " -goto :EOF diff --git bin/ext/rcfilecat.cmd bin/ext/rcfilecat.cmd deleted file mode 100644 index 324e926..0000000 --- bin/ext/rcfilecat.cmd +++ /dev/null @@ -1,34 +0,0 @@ -@echo off -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. - -set CLASS=org.apache.hadoop.hive.cli.RCFileCat -set HIVE_OPTS= - -pushd %HIVE_LIB% -for /f %%a IN ('dir /b hive-cli-*.jar') do ( - set JAR=%HIVE_LIB%\%%a -) -popd - -if [%1]==[rcfilecat_help] goto :rcfilecat_help - -:rcfilecat - call %HIVE_BIN_PATH%\ext\util\execHiveCmd.cmd %CLASS% -goto :EOF - -:rcfilecat_help - echo "usage ./hive rcfilecat [--start='startoffset'] [--length='len'] " -goto :EOF diff --git bin/ext/schemaTool.cmd bin/ext/schemaTool.cmd deleted file mode 100644 index c85886d..0000000 --- bin/ext/schemaTool.cmd +++ /dev/null @@ -1,33 +0,0 @@ -@echo off -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. - -set CLASS=org.apache.hive.beeline.HiveSchemaTool -set HIVE_OPTS= -pushd %HIVE_LIB% -for /f %%a IN ('dir /b hive-beeline-*.jar') do ( - set JAR=%HIVE_LIB%\%%a -) -popd - -if [%1]==[schematool_help] goto :schematool_help - -:schematool - call %HIVE_BIN_PATH%\ext\util\execHiveCmd.cmd %CLASS% -goto :EOF - -:schematool_help - call %HIVE_BIN_PATH%\ext\util\execHiveCmd.cmd %CLASS% --help -goto :EOF diff --git bin/ext/util/execHiveCmd.cmd bin/ext/util/execHiveCmd.cmd deleted file mode 100644 index 1f5ef49..0000000 --- bin/ext/util/execHiveCmd.cmd +++ /dev/null @@ -1,24 +0,0 @@ -@echo off -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. - -if [%1]==[] ( - echo "No class set to run. Please specify the class to run." - exit /b 1 -) -set CLASS=%1 -@rem hadoop 20 or newer - skip the aux_jars option. picked up from hiveconf -call %HADOOP% jar %JAR% %CLASS% %HIVE_OPTS% %HIVEARGS% -goto :EOF diff --git bin/hive-config.cmd bin/hive-config.cmd deleted file mode 100644 index 5f2f914..0000000 --- bin/hive-config.cmd +++ /dev/null @@ -1,61 +0,0 @@ -@echo off -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. - - -@rem processes --config and --auxpath option from command line - -if defined HIVE_HOME goto :DoneSetHiveHome -set HIVE_HOME=%~dp0 -for %%i in (%HIVE_HOME%.) do ( - set HIVE_HOME=%%~dpi -) - -if "%HIVE_HOME:~-1%" == "\" ( - set HIVE_HOME=%HIVE_HOME:~0,-1% -) - -:DoneSetHiveHome -set HIVE_CONF_DIR= -rem set HIVE_AUX_JARS_PATH= -:Loop - if [%1]==[] GOTO :FinishLoop - - if [%1]==[--config] ( - goto :SetConfig - ) - - if [%1]==[--auxpath] ( - goto :SetAux - ) - - @rem current argument does not match any aux params, finish loop here - goto :FinishLoop - SHIFT - GOTO Loop - - :SetConfig - set HIVE_CONF_DIR=%2 - shift - shift - goto :Loop - - :SetAux - set HIVE_AUX_JARS_PATH=%2 - shift - shift - goto :Loop - -:FinishLoop diff --git bin/hive.cmd bin/hive.cmd deleted file mode 100644 index a1e4806..0000000 --- bin/hive.cmd +++ /dev/null @@ -1,383 +0,0 @@ -@echo off -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -SetLocal EnableDelayedExpansion - -@rem Set the path - -if not defined HIVE_BIN_PATH ( - set HIVE_BIN_PATH=%~dp0 -) - -if "%HIVE_BIN_PATH:~-1%" == "\" ( - set HIVE_BIN_PATH=%HIVE_BIN_PATH:~0,-1% -) - -set HIVE_CONFIG_SCRIPT=%HIVE_BIN_PATH%\hive-config.cmd - -if exist %HIVE_CONFIG_SCRIPT% ( - CALL %HIVE_CONFIG_SCRIPT% %* -) - -set SERVICE= -set HELP= -set CATSERVICE= -set DEBUG= -set CURRENTARG= -set HIVEARGS= -rem set AUX_CLASSPATH= -set AUX_PARAM= - -@rem parse the command line arguments -:ProcessCmdLine - if [%1]==[] goto :FinishArgs - - set temp=%1 - set temp=%temp:~0, 7% - - if %temp%==--debug ( - set DEBUG=%* - shift - goto :ProcessCmdLine - ) - - if %1==--config ( - shift - shift - goto :ProcessCmdLine - ) - - if %1==--auxpath ( - shift - shift - goto :ProcessCmdLine - ) - - if %1==--service ( - set SERVICE=%2 - - if [%3]==[catservicexml] ( - set CATSERVICE=_catservice - shift - ) - shift - shift - goto :ProcessCmdLine - ) - - if %1==--rcfilecat ( - set SERVICE=rcfilecat - shift - goto :ProcessCmdLine - ) - - if %1==--orcfiledump ( - set SERVICE=orcfiledump - shift - goto :ProcessCmdLine - ) - - if %1==--help ( - set HELP=_help - shift - goto :ProcessCmdLine - ) - - @rem parameter at %1 does not match any option, these are optional params - goto :FinishArgs -:FinishArgs - -if defined DEBUG ( - if defined HELP ( - call %HIVE_BIN_PATH%\ext\debug.cmd HELP - goto :EOF - ) - - call %HIVE_BIN_PATH%\ext\debug.cmd %DEBUG% -) - -if defined HIVE_MAIN_CLIENT_DEBUG_OPTS ( - set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS% %HIVE_MAIN_CLIENT_DEBUG_OPTS% -) - -if not [%1]==[] ( - set CURRENTARG=%1 - call :MakeHiveArgs %* -) - -if not defined SERVICE ( - if defined HELP ( - set SERVICE=help - ) else ( - set SERVICE=cli - ) -) - -if not defined HIVE_HOME ( - echo "HIVE_HOME needs to be defined to point at the root of the hive install" - exit /b 1 -) - -if not defined HIVE_CONF_DIR ( - set HIVE_CONF_DIR=%HIVE_HOME%\conf -) - -if exist %HIVE_CONF_DIR%/hive-env.cmd CALL %HIVE_CONF_DIR%/hive-env.cmd - -@rem sort out classpath and make sure dependencies exist -set CLASSPATH=%HIVE_CONF_DIR% - -set HIVE_LIB=%HIVE_HOME%\lib - -@rem needed for execution -if not exist %HIVE_LIB%\hive-exec-*.jar ( - echo "Missing Hive Execution Jar: %HIVE_LIB%/hive-exec-*.jar" - exit /b 1 -) - -if not exist %HIVE_LIB%\hive-metastore-*.jar ( - echo "Missing Hive MetaStore Jar" - exit /b 1 -) - -@rem cli specific code -if not exist %HIVE_LIB%\hive-cli-*.jar ( - echo "Missing Hive CLI Jar" - exit /b 1 -) - -set CLASSPATH=%CLASSPATH%;%HIVE_LIB%\*;%HIVE_HOME%\hcatalog\share\hcatalog\*; - -@rem maybe we should just make users set HADOOP_HOME env variable as a prereq -@rem in the next iteration, use "where" command to find directory of hadoop install from path -if not defined HADOOP_HOME ( - echo "HADOOP_HOME needs to be defined to point at the hadoop installation" - exit /b 1 -) - -@rem supress the HADOOP_HOME warnings in 1.x.x -set HADOOP_HOME_WARN_SUPPRESS=true - -set HADOOP=%HADOOP_HOME%\bin\hadoop.cmd -if not exist %HADOOP% ( - echo "Missing hadoop installation: %HADOOP_HOME% must be set" - exit /b 1 -) - -@rem can only run against hadoop 1.0.0 as prereq for this iteration - can't figure out the regex/awk script to determine compatibility - -@rem add auxilary jars such as serdes -if not defined HIVE_AUX_JARS_PATH goto :AddAuxLibDir - -setLocal EnableDelayedExpansion -:auxJarLoop - for /f "delims=," %%a in ("!HIVE_AUX_JARS_PATH!") do ( - set auxjar=%%a - if exist %%a ( - if exist "%%a\nul" ( - @rem %%a is a dir - pushd %%a - for /f %%b IN ('dir /b *.jar') do ( - set AUX_CLASSPATH=!AUX_CLASSPATH!;%%a\%%b - call :AddToAuxParam %%a\%%b - ) - popd - ) else ( - @rem %%a is a file - set AUX_CLASSPATH=!AUX_CLASSPATH!;%%a - call :AddToAuxParam %%a - ) - ) - ) - :striploop - set stripchar=!HIVE_AUX_JARS_PATH:~0,1! - set HIVE_AUX_JARS_PATH=!HIVE_AUX_JARS_PATH:~1! - if "!HIVE_AUX_JARS_PATH!" EQU "" goto auxJarLoopEnd - if "!stripchar!" NEQ "," goto striploop - goto auxJarLoop - -:auxJarLoopEnd - -if defined HIVE_AUX_JARS_PATH ( - echo "setting aux param %HIVE_AUX_JARS_PATH%" - set AUX_CLASSPATH=%HIVE_AUX_JARS_PATH% - set AUX_PARAM=file://%HIVE_AUX_JARS_PATH% -) - - -:AddAuxLibDir -@rem adding jars from auxlib directory -if exist %HIVE_HOME%\auxlib ( - pushd %HIVE_HOME%\auxlib - for /f %%a IN ('dir /b *.jar') do ( - set AUX_CLASSPATH=%AUX_CLASSPATH%;%%a - call :AddToAuxParam %%a - ) - popd -) -set HADOOP_USER_CLASSPATH_FIRST=true - -@rem pass classpath to hadoop -set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%CLASSPATH%;%AUX_CLASSPATH% - -@rem also pass hive classpath to hadoop -if defined HIVE_CLASSPATH ( - set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%HIVE_CLASSPATH% -) - -@rem set hbase components -if defined HBASE_HOME ( - if not defined HBASE_CONF_DIR ( - if exist %HBASE_HOME%\conf ( - set HBASE_CONF_DIR=%HBASE_HOME%\conf - ) - ) - if defined HBASE_CONF_DIR ( - call :AddToHadoopClassPath %HBASE_CONF_DIR% - ) - if exist %HBASE_HOME%\lib ( - call :AddToHadoopClassPath %HBASE_HOME%\lib\* - ) -) - -if defined AUX_PARAM ( - set HIVE_OPTS=%HIVE_OPTS% -hiveconf hive.aux.jars.path="%AUX_PARAM%" - set AUX_JARS_CMD_LINE="-libjars %AUX_PARAM%" -) - -@rem Get ready to run the services -set SERVICE_COUNT=0 -set TORUN="" -call :AddServices -For /L %%i in (1,1,%SERVICE_COUNT%) do ( - if "%SERVICE%" == "!VAR%%i!" ( - set TORUN=!VAR%%i! - ) -) - -if %TORUN% == "" ( - echo "Service %SERVICE% not available" - exit /b 1 -) - - -if defined HELP ( - call %HIVE_BIN_PATH%\ext\%TORUN%.cmd %TORUN%%HELP% %* - goto :EOF -) - -@rem generate xml for the service, also append hadoop dependencies to the classpath -if defined CATSERVICE ( - if exist %HADOOP_HOME%\libexec\hadoop-config.cmd ( - call %HADOOP_HOME%\libexec\hadoop-config.cmd - ) else ( - call %HADOOP_HOME%\libexec\hadoop-config.cmd - ) - call %HIVE_BIN_PATH%\ext\%TORUN%.cmd %TORUN%%CATSERVICE% %* - goto :EOF -) - -call %HIVE_BIN_PATH%\ext\%TORUN%.cmd %TORUN% %* - - - -goto :EOF -@rem done body of script - - -@rem start utility functions here - -@rem strip off preceding arguments like --service so that subsequent args can be passed on -:MakeHiveArgs - set _count=0 - set _shift=1 - set HIVEARGS= - - if not defined CURRENTARG ( - goto :EndLoop - ) - :HiveArgsLoop - if [%1]==[] ( - goto :EndLoop - ) - if not %1==%CURRENTARG% ( - shift - goto :HiveArgsLoop - ) - - if not defined HIVEARGS ( - set HIVEARGS=%1 - ) else ( - set HIVEARGS=%HIVEARGS% %1 - ) - shift - set CURRENTARG=%1 - goto :HiveArgsLoop - :EndLoop -goto :EOF - -@rem makes list of available services -:AddServices - set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 - set VAR%SERVICE_COUNT%=cli - - set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 - set VAR%SERVICE_COUNT%=help - - set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 - set VAR%SERVICE_COUNT%=hiveserver - - set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 - set VAR%SERVICE_COUNT%=hiveserver2 - - set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 - set VAR%SERVICE_COUNT%=jar - - set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 - set VAR%SERVICE_COUNT%=lineage - - set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 - set VAR%SERVICE_COUNT%=metastore - - set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 - set VAR%SERVICE_COUNT%=rcfilecat - - set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 - set VAR%SERVICE_COUNT%=orcfiledump - - set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 - set VAR%SERVICE_COUNT%=schematool - - set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 - set VAR%SERVICE_COUNT%=cleardanglingscratchdir -goto :EOF - -:AddToAuxParam -if not defined AUX_PARAM ( - set AUX_PARAM=file:///%1 - ) else ( - set AUX_PARAM=%AUX_PARAM%,file:///%1 - ) -) -goto :EOF - -:AddToHadoopClassPath -if defined HADOOP_CLASSPATH ( - set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%1 -) else ( - set HADOOP_CLASSPATH=%1 - ) -) -goto :EOF diff --git bin/hplsql.cmd bin/hplsql.cmd deleted file mode 100644 index 6717a1c..0000000 --- bin/hplsql.cmd +++ /dev/null @@ -1,58 +0,0 @@ -@echo off -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -SetLocal EnableDelayedExpansion - -pushd %CD%\.. -if not defined HIVE_HOME ( - set HIVE_HOME=%CD% -) -popd - -if "%HADOOP_BIN_PATH:~-1%" == "\" ( - set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% -) - -if not defined JAVA_HOME ( - echo Error: JAVA_HOME is not set. - goto :eof -) - -@rem get the hadoop envrionment -if not exist %HADOOP_HOME%\libexec\hadoop-config.cmd ( - @echo +================================================================+ - @echo ^| Error: HADOOP_HOME is not set correctly ^| - @echo +----------------------------------------------------------------+ - @echo ^| Please set your HADOOP_HOME variable to the absolute path of ^| - @echo ^| the directory that contains \libexec\hadoop-config.cmd ^| - @echo +================================================================+ - exit /b 1 -) -@rem supress the HADOOP_HOME warnings in 1.x.x -set HADOOP_HOME_WARN_SUPPRESS=true - -@rem include only the HPL/SQL jar and its dependencies -pushd %HIVE_HOME%\lib -for /f %%a IN ('dir /b hive-hplsql-**.jar') do ( - set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%HIVE_HOME%\lib\%%a -) -set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%HIVE_HOME%\lib\antlr-runtime-4.5.jar -popd -set HADOOP_USER_CLASSPATH_FIRST=true -call %HADOOP_HOME%\libexec\hadoop-config.cmd - -call "%JAVA_HOME%\bin\java" %JAVA_HEAP_MAX% %HADOOP_OPTS% -classpath %HADOOP_CLASSPATH% org.apache.hive.hplsql.Hplsql %* - -endlocal diff --git cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java index bb1d865..bf23ba3 100644 --- cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java +++ cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java @@ -316,13 +316,6 @@ public void testprocessInitFiles() throws Exception { } private static void setEnv(String key, String value) throws Exception { - if (Shell.WINDOWS) - setEnvWindows(key, value); - else - setEnvLinux(key, value); - } - - private static void setEnvLinux(String key, String value) throws Exception { Class[] classes = Collections.class.getDeclaredClasses(); Map env = System.getenv(); for (Class cl : classes) { @@ -340,27 +333,6 @@ private static void setEnvLinux(String key, String value) throws Exception { } } - private static void setEnvWindows(String key, String value) throws Exception { - Class processEnvironmentClass = Class.forName("java.lang.ProcessEnvironment"); - Field theEnvironmentField = processEnvironmentClass.getDeclaredField("theEnvironment"); - theEnvironmentField.setAccessible(true); - Map env = (Map) theEnvironmentField.get(null); - if (value == null) { - env.remove(key); - } else { - env.put(key, value); - } - - Field theCaseInsensitiveEnvironmentField = processEnvironmentClass.getDeclaredField("theCaseInsensitiveEnvironment"); - theCaseInsensitiveEnvironmentField.setAccessible(true); - Map cienv = (Map) theCaseInsensitiveEnvironmentField.get(null); - if (value == null) { - cienv.remove(key); - } else { - cienv.put(key, value); - } - } - private static class FakeCliDriver extends CliDriver { @Override diff --git common/src/java/org/apache/hadoop/hive/common/FileUtils.java common/src/java/org/apache/hadoop/hive/common/FileUtils.java index ab3439c..9e07c08 100644 --- common/src/java/org/apache/hadoop/hive/common/FileUtils.java +++ common/src/java/org/apache/hadoop/hive/common/FileUtils.java @@ -240,15 +240,6 @@ public static String makeListBucketingDirName(List lbCols, List for (char c : clist) { charToEscape.set(c); } - - if(Shell.WINDOWS){ - //On windows, following chars need to be escaped as well - char [] winClist = {' ', '<','>','|'}; - for (char c : winClist) { - charToEscape.set(c); - } - } - } static boolean needsEscaping(char c) { diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index e82758f..e4f61b8 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -3466,13 +3466,13 @@ private static String findHadoopBinary() { val = (val == null ? File.separator + "usr" : val) + File.separator + "bin" + File.separator + "hadoop"; // Launch hadoop command file on windows. - return val + (Shell.WINDOWS ? ".cmd" : ""); + return val; } private static String findYarnBinary() { String val = findHadoopHome(); val = (val == null ? "yarn" : val + File.separator + "bin" + File.separator + "yarn"); - return val + (Shell.WINDOWS ? ".cmd" : ""); + return val; } private static String findHadoopHome() { diff --git common/src/java/org/apache/hive/http/HttpServer.java common/src/java/org/apache/hive/http/HttpServer.java index f99f18a..db5650d 100644 --- common/src/java/org/apache/hive/http/HttpServer.java +++ common/src/java/org/apache/hive/http/HttpServer.java @@ -365,7 +365,7 @@ Connector createChannelConnector(int queueSize, Builder b) { connector.setResolveNames(false); connector.setUseDirectBuffers(false); connector.setRequestHeaderSize(1024*64); - connector.setReuseAddress(!Shell.WINDOWS); + connector.setReuseAddress(true); return connector; } diff --git common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java index 7f6175b..fa51ef6 100644 --- common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java +++ common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java @@ -41,11 +41,6 @@ public void testHiveSitePath() throws Exception { String expectedPath = HiveTestUtils.getFileFromClasspath("hive-site.xml"); String hiveSiteLocation = HiveConf.getHiveSiteLocation().getPath(); - if (Shell.WINDOWS) { - // Do case-insensitive comparison on Windows, as drive letter can have different case. - expectedPath = expectedPath.toLowerCase(); - hiveSiteLocation = hiveSiteLocation.toLowerCase(); - } Assert.assertEquals(expectedPath, hiveSiteLocation); } diff --git contrib/src/test/org/apache/hadoop/hive/contrib/mr/TestGenericMR.java contrib/src/test/org/apache/hadoop/hive/contrib/mr/TestGenericMR.java index 5fedb3d..2d8a5e8 100644 --- contrib/src/test/org/apache/hadoop/hive/contrib/mr/TestGenericMR.java +++ contrib/src/test/org/apache/hadoop/hive/contrib/mr/TestGenericMR.java @@ -63,7 +63,7 @@ public void testIdentityMap() throws Exception { final StringWriter out = new StringWriter(); new GenericMR().map(new StringReader(in), out, identityMapper()); - assertEquals(in + "\n", getOsSpecificOutput(out.toString())); + assertEquals(in + "\n", out.toString()); } public void testKVSplitMap() throws Exception { @@ -80,7 +80,7 @@ public void map(String[] record, Output output) throws Exception { } }); - assertEquals(expected, getOsSpecificOutput(out.toString())); + assertEquals(expected, out.toString()); } public void testIdentityReduce() throws Exception { @@ -89,7 +89,7 @@ public void testIdentityReduce() throws Exception { new GenericMR().reduce(new StringReader(in), out, identityReducer()); - assertEquals(in + "\n", getOsSpecificOutput(out.toString())); + assertEquals(in + "\n", out.toString()); } public void testWordCountReduce() throws Exception { @@ -112,7 +112,7 @@ public void reduce(String key, Iterator records, Output output) final String expected = "hello\t3\nokay\t12\n"; - assertEquals(expected, getOsSpecificOutput(out.toString())); + assertEquals(expected, out.toString()); } private Mapper identityMapper() { @@ -135,9 +135,4 @@ public void reduce(String key, Iterator records, Output output) } }; } - - private static String getOsSpecificOutput(String outStr){ - assert outStr != null; - return Shell.WINDOWS ? outStr.replaceAll("\\r", "") : outStr; - } } diff --git hcatalog/bin/templeton.cmd hcatalog/bin/templeton.cmd deleted file mode 100644 index 1b1ae6f..0000000 --- hcatalog/bin/templeton.cmd +++ /dev/null @@ -1,90 +0,0 @@ -@rem Licensed to the Apache Software Foundation (ASF) under one -@rem or more contributor license agreements. See the NOTICE file -@rem distributed with this work for additional information -@rem regarding copyright ownership. The ASF licenses this file -@rem to you under the Apache License, Version 2.0 (the -@rem "License"); you may not use this file except in compliance -@rem with the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, -@rem software distributed under the License is distributed on an -@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@rem KIND, either express or implied. See the License for the -@rem specific language governing permissions and limitations -@rem under the License. -@echo off - -setlocal enabledelayedexpansion - -:main - if "%1" == "--service" ( - set service_entry=true - shift - set templeton-service-name=%1 - if not defined templeton-service-name ( - goto print_usage - ) - ) - - @rem Init hadoop env variables (CLASSPATH, HADOOP_OPTS, etc) - @rem deal with difference in the location of hadoop-config.cmd - set HADOOP_OPTS= - if exist %HADOOP_HOME%\libexec\hadoop-config.cmd ( - call %HADOOP_HOME%\libexec\hadoop-config.cmd - ) else ( - call %HADOOP_HOME%\bin\hadoop-config.cmd - ) - - @rem - @rem Compute the classpath - @rem - set WEBHCAT_CONF_DIR=%HCATALOG_HOME%\etc\webhcat - set TEMPLETON_CLASSPATH=%WEBHCAT_CONF_DIR%;%HCATALOG_HOME%;%HCATALOG_HOME%\share\webhcat\svr - - set TEMPLETON_CLASSPATH=!TEMPLETON_CLASSPATH!;%HCATALOG_HOME%\share\hcatalog\* - set TEMPLETON_CLASSPATH=!TEMPLETON_CLASSPATH!;%HCATALOG_HOME%\share\webhcat\svr\* - set TEMPLETON_CLASSPATH=!TEMPLETON_CLASSPATH!;%HCATALOG_HOME%\share\webhcat\svr\lib\* - set TEMPLETON_CLASSPATH=!TEMPLETON_CLASSPATH!;%HIVE_HOME%\conf - - @rem TODO: append hcat classpath to the templeton classpath - @rem append hadoop classpath - set CLASSPATH=%TEMPLETON_CLASSPATH%;!CLASSPATH! - - @rem compute templeton ops - if not defined TEMPLETON_LOG_DIR ( - set TEMPLETON_LOG_DIR=%HCATALOG_HOME%\logs - ) - - if not defined TEMPLETON_LOG4J ( - @rem must be prefixed with file: otherwise config is not picked up - set TEMPLETON_LOG4J=file:%WEBHCAT_CONF_DIR%\webhcat-log4j2.properties - ) - set TEMPLETON_OPTS=-Dtempleton.log.dir=%TEMPLETON_LOG_DIR% -Dlog4j.configurationFile=%TEMPLETON_LOG4J% %HADOOP_OPTS% - set arguments=%JAVA_HEAP_MAX% %TEMPLETON_OPTS% -classpath %CLASSPATH% org.apache.hive.hcatalog.templeton.Main - - if defined service_entry ( - call :makeServiceXml %arguments% - ) else ( - call %JAVA% %arguments% - ) - -goto :eof - -:makeServiceXml - set arguments=%* - @echo ^ - @echo ^%templeton-service-name%^ - @echo ^%templeton-service-name%^ - @echo ^This service runs Apache Templeton^ - @echo ^%JAVA%^ - @echo ^%arguments%^ - @echo ^ - goto :eof - - :print_usage - @echo Usage: templeton --service SERVICENAME - @echo where SERVICENAME is name of the windows service xml - -endlocal diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java index 506d3f4..6cc0a96 100644 --- hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.ql.Driver; -import org.apache.hadoop.hive.ql.WindowsPathUtil; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.util.Shell; import org.apache.hive.hcatalog.common.HCatUtil; @@ -88,9 +87,6 @@ protected void setUpHiveConf() { hiveConf .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); - if (Shell.WINDOWS) { - WindowsPathUtil.convertPathsFromWindowsToHdfs(hiveConf); - } } protected void logAndRegister(PigServer server, String query) throws IOException { diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java index 1823f2e..69874bc 100644 --- hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java @@ -160,13 +160,10 @@ public void testPartitionPublish() throws Exception { Assert.assertEquals(0, ptns.size()); Table table = msc.getTable(dbName, tableName); Assert.assertTrue(table != null); - // In Windows, we cannot remove the output directory when job fail. See - // FileOutputCommitterContainer.abortJob - if (!Shell.WINDOWS) { - Path path = new Path(table.getSd().getLocation() - + "/part1=p1value1/part0=p0value1"); - Assert.assertFalse(path.getFileSystem(conf).exists(path)); - } + + Path path = new Path(table.getSd().getLocation() + + "/part1=p1value1/part0=p0value1"); + Assert.assertFalse(path.getFileSystem(conf).exists(path)); } void runMRCreateFail( diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java index 0b5e83e..4c686fe 100644 --- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java +++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java @@ -33,7 +33,6 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.Driver; -import org.apache.hadoop.hive.ql.WindowsPathUtil; import org.apache.hadoop.hive.ql.io.IOConstants; import org.apache.hadoop.hive.ql.io.StorageFormats; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; @@ -129,10 +128,6 @@ public static void setUpBeforeClass() throws Exception { .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); - if (Shell.WINDOWS) { - WindowsPathUtil.convertPathsFromWindowsToHdfs(hiveConf); - } - driver = new Driver(hiveConf); SessionState.start(new CliSessionState(hiveConf)); //props = new Properties(); diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java index 717cbad..ad11eab 100644 --- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java +++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java @@ -29,7 +29,6 @@ import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.Driver; -import org.apache.hadoop.hive.ql.WindowsPathUtil; import org.apache.hadoop.hive.ql.io.IOConstants; import org.apache.hadoop.hive.ql.io.StorageFormats; import org.apache.hadoop.hive.ql.processors.CommandProcessor; @@ -187,9 +186,6 @@ public void setup() throws Exception { System.getProperty("test.build.data", "build/test/data") + "_" + System.currentTimeMillis() + "_" + salt.getAndIncrement() + "/dfs/"); } - if (Shell.WINDOWS) { - WindowsPathUtil.convertPathsFromWindowsToHdfs(hiveConf); - } driver = new Driver(hiveConf); @@ -201,9 +197,9 @@ public void setup() throws Exception { createTable(BASIC_TABLE, "a int, b string"); createTableInSpecifiedPath(ENCRYPTED_TABLE, "a int, b string", - WindowsPathUtil.getHdfsUriString(encryptedTablePath), driver); + encryptedTablePath, driver); - associateEncryptionZoneWithPath(WindowsPathUtil.getHdfsUriString(encryptedTablePath)); + associateEncryptionZoneWithPath(encryptedTablePath); int LOOP_SIZE = 3; String[] input = new String[LOOP_SIZE * LOOP_SIZE]; @@ -231,7 +227,6 @@ void initEncryptionShim(HiveConf conf) throws IOException { FileSystem fs; HadoopShims shims = ShimLoader.getHadoopShims(); conf.set(SECURITY_KEY_PROVIDER_URI_NAME, getKeyProviderURI()); - WindowsPathUtil.convertPathsFromWindowsToHdfs(conf); int numberOfDataNodes = 4; dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null); fs = dfs.getFileSystem(); @@ -359,7 +354,7 @@ public void testReadDataFromEncryptedHiveTableByHCatMR() throws Exception { fs.delete(path, true); } - TextOutputFormat.setOutputPath(job, new Path(WindowsPathUtil.getHdfsUriString(pathLoc))); + TextOutputFormat.setOutputPath(job, new Path(pathLoc)); job.waitForCompletion(true); diff --git hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm index b965eec..ea718c3 100644 --- hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm +++ hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm @@ -1,4 +1,4 @@ -############################################################################ +############################################################################ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,13 +15,13 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - + package TestDriverCurl; ########################################################################### # Class: TestDriver # A base class for TestDrivers. -# +# use TestDriverFactory; use TestReport; @@ -59,25 +59,25 @@ package TestDriverCurl; # None. # sub printResults - { +{ my ($testStatuses, $log, $prefix) = @_; my ($pass, $fail, $abort, $depend, $skipped) = (0, 0, 0, 0, 0); foreach (keys(%$testStatuses)) { - ($testStatuses->{$_} eq $passedStr) && $pass++; - ($testStatuses->{$_} eq $failedStr) && $fail++; - ($testStatuses->{$_} eq $abortedStr) && $abort++; - ($testStatuses->{$_} eq $dependStr) && $depend++; - ($testStatuses->{$_} eq $skippedStr) && $skipped++; + ($testStatuses->{$_} eq $passedStr) && $pass++; + ($testStatuses->{$_} eq $failedStr) && $fail++; + ($testStatuses->{$_} eq $abortedStr) && $abort++; + ($testStatuses->{$_} eq $dependStr) && $depend++; + ($testStatuses->{$_} eq $skippedStr) && $skipped++; } my $msg = "$prefix, PASSED: $pass FAILED: $fail SKIPPED: $skipped ABORTED: $abort " - . "FAILED DEPENDENCY: $depend"; + . "FAILED DEPENDENCY: $depend"; print $log "$msg\n"; print "$msg\n"; - - } + +} ############################################################################## # Sub: printGroupResultsXml @@ -93,25 +93,25 @@ sub printResults # None. # sub printGroupResultsXml - { +{ my ( $report, $groupName, $testStatuses, $totalDuration) = @_; $totalDuration=0 if ( !$totalDuration ); my ($pass, $fail, $abort, $depend) = (0, 0, 0, 0); foreach my $key (keys(%$testStatuses)) { - if ( $key =~ /^$groupName/ ) { - ($testStatuses->{$key} eq $passedStr) && $pass++; - ($testStatuses->{$key} eq $failedStr) && $fail++; - ($testStatuses->{$key} eq $abortedStr) && $abort++; - ($testStatuses->{$key} eq $dependStr) && $depend++; - } + if ( $key =~ /^$groupName/ ) { + ($testStatuses->{$key} eq $passedStr) && $pass++; + ($testStatuses->{$key} eq $failedStr) && $fail++; + ($testStatuses->{$key} eq $abortedStr) && $abort++; + ($testStatuses->{$key} eq $dependStr) && $depend++; + } } my $total= $pass + $fail + $abort; $report->totals( $groupName, $total, $fail, $abort, $totalDuration ); - } +} ############################################################################## # Sub: new @@ -123,7 +123,7 @@ sub printGroupResultsXml # Returns: # None. sub new - { +{ my $proto = shift; my $class = ref($proto) || $proto; my $self = {}; @@ -133,7 +133,7 @@ sub new $self->{'wrong_execution_mode'} = "_xyz_wrong_execution_mode_zyx_"; return $self; - } +} ############################################################################## # Sub: globalSetup @@ -150,7 +150,7 @@ sub new # None # sub globalSetup - { +{ my ($self, $globalHash, $log) = @_; my $subName = (caller(0))[3]; @@ -166,11 +166,11 @@ sub globalSetup # if "-ignore false" was provided on the command line, # it means do run tests even when marked as 'ignore' if (defined($globalHash->{'ignore'}) && $globalHash->{'ignore'} eq 'false') { - $self->{'ignore'} = 'false'; + $self->{'ignore'} = 'false'; } if (! defined $globalHash->{'localpathbase'}) { - $globalHash->{'localpathbase'} = '/tmp'; + $globalHash->{'localpathbase'} = '/tmp'; } $globalHash->{'outpath'} = $globalHash->{'outpathbase'} . "/" . $globalHash->{'runid'} . "/"; @@ -197,19 +197,19 @@ sub globalSetup # add libexec location to the path if (defined($ENV{'PATH'})) { - $ENV{'PATH'} = $globalHash->{'scriptPath'} . ":" . $ENV{'PATH'}; + $ENV{'PATH'} = $globalHash->{'scriptPath'} . ":" . $ENV{'PATH'}; } else { - $ENV{'PATH'} = $globalHash->{'scriptPath'}; + $ENV{'PATH'} = $globalHash->{'scriptPath'}; } - IPC::Run::run(['mkdir', '-p', $globalHash->{'localpath'}], \undef, $log, $log) or + IPC::Run::run(['mkdir', '-p', $globalHash->{'localpath'}], \undef, $log, $log) or die "Cannot create localpath directory " . $globalHash->{'localpath'} . - " " . "$ERRNO\n"; + " " . "$ERRNO\n"; # Create the temporary directory - IPC::Run::run(['mkdir', '-p', $globalHash->{'tmpPath'}], \undef, $log, $log) or + IPC::Run::run(['mkdir', '-p', $globalHash->{'tmpPath'}], \undef, $log, $log) or die "Cannot create temporary directory " . $globalHash->{'tmpPath'} . - " " . "$ERRNO\n"; + " " . "$ERRNO\n"; my $testCmdBasics = $self->copyTestBasicConfig($globalHash); $testCmdBasics->{'method'} = 'PUT'; @@ -221,8 +221,8 @@ sub globalSetup my $json = new JSON; $json->utf8->decode($curl_result->{'body'})->{'boolean'} or die "Cannot create hdfs directory " . $globalHash->{'outpath'} . - " " . "$ERRNO\n"; - } + " " . "$ERRNO\n"; +} ############################################################################### # Sub: globalCleanup @@ -238,14 +238,14 @@ sub globalSetup # Returns: # None sub globalCleanup - { +{ my ($self, $globalHash, $log) = @_; - IPC::Run::run(['rm', '-rf', $globalHash->{'tmpPath'}], \undef, $log, $log) or + IPC::Run::run(['rm', '-rf', $globalHash->{'tmpPath'}], \undef, $log, $log) or warn "Cannot remove temporary directory " . $globalHash->{'tmpPath'} . - " " . "$ERRNO\n"; + " " . "$ERRNO\n"; - } +} ############################################################################### # Sub: runTest @@ -260,7 +260,7 @@ sub globalCleanup # @returns reference to hash. Contents of hash are defined by the subclass. # sub runTest - { +{ my ($self, $testCmd, $log) = @_; my $subName = (caller(0))[3]; @@ -270,23 +270,23 @@ sub runTest my %result; return \%result; } - # Handle the various methods of running used in + # Handle the various methods of running used in # the original TestDrivers if ( $testCmd->{'url'} ) { - return $self->runCurlCmd( $testCmd, $log); + return $self->runCurlCmd( $testCmd, $log); } else { - die "$subName FATAL Did not find a testCmd that " . - "I know how to handle : " . $testCmd->{'Curl'}; + die "$subName FATAL Did not find a testCmd that " . + "I know how to handle : " . $testCmd->{'Curl'}; } - } +} ############################################################################### sub replaceParameters - { +{ my ($self, $testCmd, $aPfix, $log) = @_; my $url = $testCmd->{$aPfix . 'url'}; @@ -295,60 +295,49 @@ sub replaceParameters $url = $self->replaceParametersInArg($url, $testCmd, $log); $testCmd->{$aPfix . 'url'} = $url; - $testCmd->{$aPfix . 'upload_file'} = - $self->replaceParametersInArg($testCmd->{$aPfix . 'upload_file'}, $testCmd, $log); + $testCmd->{$aPfix . 'upload_file'} = + $self->replaceParametersInArg($testCmd->{$aPfix . 'upload_file'}, $testCmd, $log); - $testCmd->{$aPfix . 'user_name'} = - $self->replaceParametersInArg($testCmd->{$aPfix . 'user_name'}, $testCmd, $log); + $testCmd->{$aPfix . 'user_name'} = + $self->replaceParametersInArg($testCmd->{$aPfix . 'user_name'}, $testCmd, $log); if (defined $testCmd->{$aPfix . 'post_options'}) { - my @options = @{$testCmd->{$aPfix . 'post_options'}}; - my @new_options = (); - foreach my $option (@options) { - $option = $self->replaceParametersInArg($option, $testCmd, $log); - if (isWindows()) { - my $equal_pos = index($option, '='); - if ($equal_pos != -1) { - my $left = substr($option, 0, $equal_pos); - my $right = substr($option, $equal_pos+1); - if ($right =~ /=/) { - $right = '"'.$right.'"'; - $option = $left . "=" . $right; - } - } + my @options = @{$testCmd->{$aPfix . 'post_options'}}; + my @new_options = (); + foreach my $option (@options) { + $option = $self->replaceParametersInArg($option, $testCmd, $log); + push @new_options, ($option); } - push @new_options, ($option); - } - $testCmd->{$aPfix . 'post_options'} = \@new_options; - } + $testCmd->{$aPfix . 'post_options'} = \@new_options; + } if (defined $testCmd->{$aPfix . 'json_field_substr_match'}) { - my $json_matches = $testCmd->{$aPfix . 'json_field_substr_match'}; - my @keys = keys %{$json_matches}; + my $json_matches = $testCmd->{$aPfix . 'json_field_substr_match'}; + my @keys = keys %{$json_matches}; - foreach my $key (@keys) { - my $new_value = $self->replaceParametersInArg($json_matches->{$key}, $testCmd, $log); - $json_matches->{$key} = $new_value; - } - } + foreach my $key (@keys) { + my $new_value = $self->replaceParametersInArg($json_matches->{$key}, $testCmd, $log); + $json_matches->{$key} = $new_value; + } + } if (defined $testCmd->{$aPfix . 'json_path'}) { - my $json_path_matches = $testCmd->{$aPfix . 'json_path'}; - my @keys = keys %{$json_path_matches}; + my $json_path_matches = $testCmd->{$aPfix . 'json_path'}; + my @keys = keys %{$json_path_matches}; - foreach my $key (@keys) { - my $new_value = $self->replaceParametersInArg($json_path_matches->{$key}, $testCmd, $log); - $json_path_matches->{$key} = $new_value; - } + foreach my $key (@keys) { + my $new_value = $self->replaceParametersInArg($json_path_matches->{$key}, $testCmd, $log); + $json_path_matches->{$key} = $new_value; + } } - } +} ############################################################################### sub replaceParametersInArg - { +{ my ($self, $arg, $testCmd, $log) = @_; if(! defined $arg){ - return $arg; + return $arg; } my $outdir = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'}; $arg =~ s/:UNAME:/$testCmd->{'current_user'}/g; @@ -364,240 +353,240 @@ sub replaceParametersInArg $arg =~ s/:DB_PASSWORD:/$testCmd->{'db_password'}/g; $arg =~ s/:TNUM:/$testCmd->{'num'}/g; return $arg; - } +} ############################################################################### sub getBaseCurlCmd(){ - my ($self) = @_; - my @curl_cmd = ("curl", '--silent','--show-error', '-H','Expect:'); - if (defined $ENV{'SOCKS_PROXY'}) { - push @curl_cmd, ('--socks5-hostname', $ENV{'SOCKS_PROXY'}); - } - return @curl_cmd; + my ($self) = @_; + my @curl_cmd = ("curl", '--silent','--show-error', '-H','Expect:'); + if (defined $ENV{'SOCKS_PROXY'}) { + push @curl_cmd, ('--socks5-hostname', $ENV{'SOCKS_PROXY'}); + } + return @curl_cmd; } ############################################################################### sub runCurlCmd(){ - my ($self, $testCmd, $log) = @_; - if (defined $testCmd->{'upload_file'}) { - return $self->upload_file($testCmd,$log); - } else { - #if there are setup steps, run them first - if (defined $testCmd->{'setup'}) { - my $i = 0; - foreach my $setupCmd (@{$testCmd->{'setup'}}){ - $i++; - print $log "\nRUNNING SETUP COMMAND: $i\n"; - my $pfix = "setup_${i}_"; - my $setupTestCmd = $self->createSetupCmd($testCmd, $setupCmd, $pfix, $log); - my $setupResult = $self->execCurlCmd($setupTestCmd, $pfix, $log); - - #if status code is set in setup, check if it matches results - if(defined $setupTestCmd->{"${pfix}status_code"}){ - $self->checkResStatusCode($setupResult, $setupTestCmd->{"${pfix}status_code"}, $log); + my ($self, $testCmd, $log) = @_; + if (defined $testCmd->{'upload_file'}) { + return $self->upload_file($testCmd,$log); + } else { + #if there are setup steps, run them first + if (defined $testCmd->{'setup'}) { + my $i = 0; + foreach my $setupCmd (@{$testCmd->{'setup'}}){ + $i++; + print $log "\nRUNNING SETUP COMMAND: $i\n"; + my $pfix = "setup_${i}_"; + my $setupTestCmd = $self->createSetupCmd($testCmd, $setupCmd, $pfix, $log); + my $setupResult = $self->execCurlCmd($setupTestCmd, $pfix, $log); + + #if status code is set in setup, check if it matches results + if(defined $setupTestCmd->{"${pfix}status_code"}){ + $self->checkResStatusCode($setupResult, $setupTestCmd->{"${pfix}status_code"}, $log); + } + } } - } + return $self->execCurlCmd($testCmd, "", $log); } - return $self->execCurlCmd($testCmd, "", $log); - } } ############################################################################### sub createSetupCmd(){ - my ($self, $testCmd, $setupCmd, $pfix, $log) = @_; - my $newTestCmd = dclone ($testCmd); - for my $key (keys %$setupCmd){ - $newTestCmd->{$pfix . $key} = $setupCmd->{$key}; - } - return $newTestCmd; + my ($self, $testCmd, $setupCmd, $pfix, $log) = @_; + my $newTestCmd = dclone ($testCmd); + for my $key (keys %$setupCmd){ + $newTestCmd->{$pfix . $key} = $setupCmd->{$key}; + } + return $newTestCmd; } ############################################################################### sub upload_file(){ - my ($self, $testCmd, $log) = @_; - $testCmd->{'method'} = 'PUT'; - my $result = $self->execCurlCmd($testCmd, "", $log); - my $checkRes = $self->checkResStatusCode($result, 100, $log); - if ($checkRes == 0) { - #fail - return 0; - } - my $header = $result->{'header_fields'}; - - #final url where the file should be stored - my $location = $header->{'Location'}; - $testCmd->{'url'} = $location; - - $result = $self->execCurlCmd($testCmd, "", $log); - return $result; + my ($self, $testCmd, $log) = @_; + $testCmd->{'method'} = 'PUT'; + my $result = $self->execCurlCmd($testCmd, "", $log); + my $checkRes = $self->checkResStatusCode($result, 100, $log); + if ($checkRes == 0) { + #fail + return 0; + } + my $header = $result->{'header_fields'}; + + #final url where the file should be stored + my $location = $header->{'Location'}; + $testCmd->{'url'} = $location; + + $result = $self->execCurlCmd($testCmd, "", $log); + return $result; } ############################################################################### sub execCurlCmd(){ - my ($self, $testCmd, $argPrefix, $log) = @_; - my @curl_cmd = $self->getBaseCurlCmd(); - # Set up file locations - my $subName = (caller(0))[3]; - - my $filePrefix = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $argPrefix . $testCmd->{'num'}; - my $cmd_body = $filePrefix . ".cmd_body"; - - #results - my $res_header = $filePrefix . ".res_header"; - my $res_body = $filePrefix . ".res_body"; - - my $outdir = $filePrefix . ".out"; - my $stdoutfile = "$outdir/stdout"; - my $stderrfile = "$outdir/stderr"; - - mkpath( [ $outdir ] , 0, 0755) if ( ! -e outdir ); - if ( ! -e $outdir ) { - print $log "$0.$subName FATAL could not mkdir $outdir\n"; - die "$0.$subName FATAL could not mkdir $outdir\n"; - } - - $self->replaceParameters($testCmd, $argPrefix, $log ); - - my $method = $testCmd->{ $argPrefix . 'method'}; - - my $url = $testCmd->{ $argPrefix . 'url'}; - - #allow curl to make insecure ssl connections and transfers - if ($url =~ /^https:/) { - push @curl_cmd, '-k'; - } - - my @options = (); - if (defined $testCmd->{$argPrefix . 'post_options'}) { - @options = @{$testCmd->{$argPrefix . 'post_options'}}; - } - - #handle authentication based on secure mode - my $user_name = $testCmd->{ $argPrefix . 'user_name' }; - if (defined $testCmd->{'is_secure_mode'} && $testCmd->{'is_secure_mode'} =~ /y.*/i) { - push @curl_cmd, ('--negotiate', '-u', ':'); - - #if keytab dir is defined, look for a keytab file for user and do a kinit - if(defined $testCmd->{'keytab_dir'} && defined $user_name){ - $user_name =~ /(.*?)(\/|$)/; - my $just_uname = $1; #uname without principal - my $keytab_dir = $testCmd->{'keytab_dir'}; - print $log "regex " . "${keytab_dir}/*${just_uname}\.*keytab"; - my @files = bsd_glob( "${keytab_dir}/*${just_uname}\.*keytab" ); - if(scalar @files == 0){ - die "Could not find keytab file for user $user_name in $keytab_dir"; - } elsif(scalar @files > 1){ - die "More than one keytab file found for user $user_name in $keytab_dir"; - } - my @cmd = (); - if (defined $testCmd->{'user_realm'}){ - my $user_name_with_realm_name = $user_name.'@'.$testCmd->{'user_realm'}; - @cmd = ('kinit', '-k', '-t', $files[0], $user_name_with_realm_name); - } - else{ - @cmd = ('kinit', '-k', '-t', $files[0], $user_name); - } - print $log "Command @cmd"; - IPC::Run::run(\@cmd, \undef, $log, $log) or - die "Could not kinit as $user_name using " . $files[0] . " $ERRNO"; + my ($self, $testCmd, $argPrefix, $log) = @_; + my @curl_cmd = $self->getBaseCurlCmd(); + # Set up file locations + my $subName = (caller(0))[3]; + + my $filePrefix = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $argPrefix . $testCmd->{'num'}; + my $cmd_body = $filePrefix . ".cmd_body"; + + #results + my $res_header = $filePrefix . ".res_header"; + my $res_body = $filePrefix . ".res_body"; + + my $outdir = $filePrefix . ".out"; + my $stdoutfile = "$outdir/stdout"; + my $stderrfile = "$outdir/stderr"; + + mkpath( [ $outdir ] , 0, 0755) if ( ! -e outdir ); + if ( ! -e $outdir ) { + print $log "$0.$subName FATAL could not mkdir $outdir\n"; + die "$0.$subName FATAL could not mkdir $outdir\n"; } - } else { - #if mode is unsecure - if (defined $user_name) { - my $user_param = "user.name=${user_name}"; - if ($method eq 'POST' ) { - push @options, $user_param; - } else { - if ($url =~ /\?/) { - #has some parameters in url - $url = $url . '&' . $user_param; - } else { - $url = $url . '?' . $user_param; + $self->replaceParameters($testCmd, $argPrefix, $log ); + + my $method = $testCmd->{ $argPrefix . 'method'}; + + my $url = $testCmd->{ $argPrefix . 'url'}; + + #allow curl to make insecure ssl connections and transfers + if ($url =~ /^https:/) { + push @curl_cmd, '-k'; + } + + my @options = (); + if (defined $testCmd->{$argPrefix . 'post_options'}) { + @options = @{$testCmd->{$argPrefix . 'post_options'}}; + } + + #handle authentication based on secure mode + my $user_name = $testCmd->{ $argPrefix . 'user_name' }; + if (defined $testCmd->{'is_secure_mode'} && $testCmd->{'is_secure_mode'} =~ /y.*/i) { + push @curl_cmd, ('--negotiate', '-u', ':'); + + #if keytab dir is defined, look for a keytab file for user and do a kinit + if(defined $testCmd->{'keytab_dir'} && defined $user_name){ + $user_name =~ /(.*?)(\/|$)/; + my $just_uname = $1; #uname without principal + my $keytab_dir = $testCmd->{'keytab_dir'}; + print $log "regex " . "${keytab_dir}/*${just_uname}\.*keytab"; + my @files = bsd_glob( "${keytab_dir}/*${just_uname}\.*keytab" ); + if(scalar @files == 0){ + die "Could not find keytab file for user $user_name in $keytab_dir"; + } elsif(scalar @files > 1){ + die "More than one keytab file found for user $user_name in $keytab_dir"; + } + my @cmd = (); + if (defined $testCmd->{'user_realm'}){ + my $user_name_with_realm_name = $user_name.'@'.$testCmd->{'user_realm'}; + @cmd = ('kinit', '-k', '-t', $files[0], $user_name_with_realm_name); + } + else{ + @cmd = ('kinit', '-k', '-t', $files[0], $user_name); + } + print $log "Command @cmd"; + IPC::Run::run(\@cmd, \undef, $log, $log) or + die "Could not kinit as $user_name using " . $files[0] . " $ERRNO"; + } + + } else { + #if mode is unsecure + if (defined $user_name) { + my $user_param = "user.name=${user_name}"; + if ($method eq 'POST' ) { + push @options, $user_param; + } else { + if ($url =~ /\?/) { + #has some parameters in url + $url = $url . '&' . $user_param; + } else { + $url = $url . '?' . $user_param; + } + } } - } + } - } - - if (defined $testCmd->{'format_header'}) { - push @curl_cmd, ('-H', $testCmd->{'format_header'}); - } - - - - - if (defined $testCmd->{$argPrefix . 'format_header'}) { - push @curl_cmd, ('-H', $testCmd->{$argPrefix . 'format_header'}); - } - - if (defined $testCmd->{$argPrefix . 'upload_file'}) { - push @curl_cmd, ('-T', $testCmd->{$argPrefix . 'upload_file'}); - } - - # if(!defined $testCmd->{'post_options'}){ - # $testCmd->{'post_options'} = \(); - # } - - if (defined $testCmd->{$argPrefix . 'check_call_back'}) { - my $d = HTTP::Daemon->new || die; - $testCmd->{'http_daemon'} = $d; - $testCmd->{'callback_url'} = $d->url . 'templeton/$jobId'; - push @curl_cmd, ('-d', 'callback=' . $testCmd->{'callback_url'}); - push @{$testCmd->{$argPrefix . 'post_options'}}, ('callback=' . $testCmd->{'callback_url'}); - # #my @options = @{$testCmd->{'post_options'}}; - # print $log "post options @options\n"; - } - - foreach my $option (@options) { - push @curl_cmd, ('-d', $option); - } - - push @curl_cmd, ("-X", $method, "-o", $res_body, "-D", $res_header); - push @curl_cmd, ($url); - - print $log "$0:$subName Going to run command : " . join (' , ', @curl_cmd); - print $log "\n"; - - - my %result; - my $out; - my $err; - IPC::Run::run(\@curl_cmd, \undef, $out, $err) - or die "Failed running curl cmd " . join ' ', @curl_cmd; - - $result{'rc'} = $? >> 8; - $result{'stderr'} = $err; - $result{'stdout'} = $out; - $result{'body'} = `cat $res_body`; - - my @full_header = `cat $res_header`; - $result{'full_header'} = join '\n', @full_header; - - #find the final http status code - for my $line ( @full_header){ - if($line =~ /.*(HTTP\/1.1)\s+(\S+)/){ - $result{'status_code'} = $2; + if (defined $testCmd->{'format_header'}) { + push @curl_cmd, ('-H', $testCmd->{'format_header'}); } - } - - my %header_field; - foreach my $line (@full_header) { - chomp $line; - $line =~ /(.*?)\s*:\s*(.*)/; - if (defined $1 && defined $2 ) { - $header_field{$1} = $2; + + + + + if (defined $testCmd->{$argPrefix . 'format_header'}) { + push @curl_cmd, ('-H', $testCmd->{$argPrefix . 'format_header'}); } - } - $result{'header_fields'} = \%header_field; - print $log "result : " . dump(%result); - #dump(%result); - - return \%result; + if (defined $testCmd->{$argPrefix . 'upload_file'}) { + push @curl_cmd, ('-T', $testCmd->{$argPrefix . 'upload_file'}); + } + + # if(!defined $testCmd->{'post_options'}){ + # $testCmd->{'post_options'} = \(); + # } + + if (defined $testCmd->{$argPrefix . 'check_call_back'}) { + my $d = HTTP::Daemon->new || die; + $testCmd->{'http_daemon'} = $d; + $testCmd->{'callback_url'} = $d->url . 'templeton/$jobId'; + push @curl_cmd, ('-d', 'callback=' . $testCmd->{'callback_url'}); + push @{$testCmd->{$argPrefix . 'post_options'}}, ('callback=' . $testCmd->{'callback_url'}); + # #my @options = @{$testCmd->{'post_options'}}; + # print $log "post options @options\n"; + } + + foreach my $option (@options) { + push @curl_cmd, ('-d', $option); + } + + push @curl_cmd, ("-X", $method, "-o", $res_body, "-D", $res_header); + push @curl_cmd, ($url); + + print $log "$0:$subName Going to run command : " . join (' , ', @curl_cmd); + print $log "\n"; + + + my %result; + my $out; + my $err; + IPC::Run::run(\@curl_cmd, \undef, $out, $err) + or die "Failed running curl cmd " . join ' ', @curl_cmd; + + $result{'rc'} = $? >> 8; + $result{'stderr'} = $err; + $result{'stdout'} = $out; + $result{'body'} = `cat $res_body`; + + my @full_header = `cat $res_header`; + $result{'full_header'} = join '\n', @full_header; + + #find the final http status code + for my $line ( @full_header){ + if($line =~ /.*(HTTP\/1.1)\s+(\S+)/){ + $result{'status_code'} = $2; + } + } + + my %header_field; + foreach my $line (@full_header) { + chomp $line; + $line =~ /(.*?)\s*:\s*(.*)/; + if (defined $1 && defined $2 ) { + $header_field{$1} = $2; + } + } + $result{'header_fields'} = \%header_field; + + print $log "result : " . dump(%result); + #dump(%result); + + return \%result; } @@ -616,14 +605,14 @@ () # @returns reference to hash. Contents of hash are defined by the subclass. # sub generateBenchmark - { +{ my %result; return \%result; - } +} ############################################################################### # Sub: compare -# Compare the results of the test run with the generated benchmark results. +# Compare the results of the test run with the generated benchmark results. # This is a pure virtual function. # # Parameters: @@ -638,7 +627,7 @@ sub generateBenchmark # same" is defined by the subclass. # sub compare - { +{ my ($self, $testResult, $benchmarkResult, $log, $testCmd) = @_; my $subName = (caller(0))[3]; @@ -652,374 +641,374 @@ sub compare my $result = 1; # until proven wrong... if (defined $testCmd->{'status_code'}) { - my $res = $self->checkResStatusCode($testResult, $testCmd->{'status_code'}, $log); - if ($res == 0) { - $result = 0; - } + my $res = $self->checkResStatusCode($testResult, $testCmd->{'status_code'}, $log); + if ($res == 0) { + $result = 0; + } } my $json_hash; my %json_info; # for information on JSONPath, check http://goessner.net/articles/JsonPath/ if (defined $testCmd->{'json_path'}) { - my $json_matches = $testCmd->{'json_path'}; - foreach my $key (keys %$json_matches) { - my $regex_expected_value = $json_matches->{$key}; - my $path = JSON::Path->new($key); - - # decode $testResult->{'body'} to an array of hash - my $body = decode_json $testResult->{'body'}; - my @filtered_body; - if (defined $testCmd->{'filter_job_names'}) { - foreach my $filter (@{$testCmd->{'filter_job_names'}}) { - my @filtered_body_tmp = grep { $_->{detail}{profile}{jobName} eq $filter } @$body; - @filtered_body = (@filtered_body, @filtered_body_tmp); - } - } else { - @filtered_body = @$body; - } - my @sorted_filtered_body; - if (ref @$body[0] eq 'HASH') { - @sorted_filtered_body = sort { $a->{id} cmp $b->{id} } @filtered_body; - } else { - @sorted_filtered_body = sort { $a cmp $b } @filtered_body; - } - my $value = $path->value(\@sorted_filtered_body); - if (JSON::is_bool($value)) { - $value = $value ? 'true' : 'false'; - } - - if ($value !~ /$regex_expected_value/s) { - print $log "$0::$subName INFO check failed:" - . " json pattern check failed. For field " - . "$key, regex <" . $regex_expected_value - . "> did not match the result <" . $value - . ">\n"; - $result = 0; - last; + my $json_matches = $testCmd->{'json_path'}; + foreach my $key (keys %$json_matches) { + my $regex_expected_value = $json_matches->{$key}; + my $path = JSON::Path->new($key); + + # decode $testResult->{'body'} to an array of hash + my $body = decode_json $testResult->{'body'}; + my @filtered_body; + if (defined $testCmd->{'filter_job_names'}) { + foreach my $filter (@{$testCmd->{'filter_job_names'}}) { + my @filtered_body_tmp = grep { $_->{detail}{profile}{jobName} eq $filter } @$body; + @filtered_body = (@filtered_body, @filtered_body_tmp); + } + } else { + @filtered_body = @$body; + } + my @sorted_filtered_body; + if (ref @$body[0] eq 'HASH') { + @sorted_filtered_body = sort { $a->{id} cmp $b->{id} } @filtered_body; + } else { + @sorted_filtered_body = sort { $a cmp $b } @filtered_body; + } + my $value = $path->value(\@sorted_filtered_body); + if (JSON::is_bool($value)) { + $value = $value ? 'true' : 'false'; + } + + if ($value !~ /$regex_expected_value/s) { + print $log "$0::$subName INFO check failed:" + . " json pattern check failed. For field " + . "$key, regex <" . $regex_expected_value + . "> did not match the result <" . $value + . ">\n"; + $result = 0; + last; + } } - } - } + } if (defined $testCmd->{'json_field_substr_match'} || $testCmd->{'json_field_match_object'}) { - my $json = new JSON; - $json_hash = $json->utf8->decode($testResult->{'body'}); - my $json_matches = $testCmd->{'json_field_substr_match'}; - my $json_matches_object = $testCmd->{'json_field_match_object'}; - - %json_info = %$json_hash; - if (defined $json_info{'info'}) { - %json_info = %{$json_info{'info'}}; - - } - print $log "\n\n json_info"; - print $log dump(%json_info); - print $log "\n\n"; - - if (defined $json_hash->{'id'}) { - print STDERR "jobid " . $json_hash->{'id'} . "\n"; - $json_info{'id'} = $json_hash->{'id'}; - } - - if(defined $json_matches->{'location_perms'} || defined $json_matches->{'location_group'}){ - $self->setLocationPermGroup(\%json_info, $testCmd, $log); - } - - foreach my $key (keys %$json_matches) { - my $json_field_val = $json_info{$key}; - if( (ref($json_field_val) && ! UNIVERSAL::isa($json_field_val,'SCALAR')) || - (!ref($json_field_val) && ! UNIVERSAL::isa(\$json_field_val,'SCALAR')) ){ - #flatten the object into a string - $json_field_val = dump($json_field_val); + my $json = new JSON; + $json_hash = $json->utf8->decode($testResult->{'body'}); + my $json_matches = $testCmd->{'json_field_substr_match'}; + my $json_matches_object = $testCmd->{'json_field_match_object'}; + + %json_info = %$json_hash; + if (defined $json_info{'info'}) { + %json_info = %{$json_info{'info'}}; + } - if (JSON::is_bool($json_field_val)) { - $json_field_val = $json_field_val ? 'true' : 'false'; + print $log "\n\n json_info"; + print $log dump(%json_info); + print $log "\n\n"; + + if (defined $json_hash->{'id'}) { + print STDERR "jobid " . $json_hash->{'id'} . "\n"; + $json_info{'id'} = $json_hash->{'id'}; + } + + if(defined $json_matches->{'location_perms'} || defined $json_matches->{'location_group'}){ + $self->setLocationPermGroup(\%json_info, $testCmd, $log); } - my $regex_expected_value = $json_matches->{$key}; - print $log "Comparing $key: $json_field_val with regex /$regex_expected_value/\n"; - - if ($json_field_val !~ /$regex_expected_value/s) { - print $log "$0::$subName WARN check failed:" - . " json pattern check failed. For field " - . "$key, regex <" . $regex_expected_value - . "> did not match the result <" . $json_field_val - . ">\n"; - $result = 0; + + foreach my $key (keys %$json_matches) { + my $json_field_val = $json_info{$key}; + if( (ref($json_field_val) && ! UNIVERSAL::isa($json_field_val,'SCALAR')) || + (!ref($json_field_val) && ! UNIVERSAL::isa(\$json_field_val,'SCALAR')) ){ + #flatten the object into a string + $json_field_val = dump($json_field_val); + } + if (JSON::is_bool($json_field_val)) { + $json_field_val = $json_field_val ? 'true' : 'false'; + } + my $regex_expected_value = $json_matches->{$key}; + print $log "Comparing $key: $json_field_val with regex /$regex_expected_value/\n"; + + if ($json_field_val !~ /$regex_expected_value/s) { + print $log "$0::$subName WARN check failed:" + . " json pattern check failed. For field " + . "$key, regex <" . $regex_expected_value + . "> did not match the result <" . $json_field_val + . ">\n"; + $result = 0; + } } - } - - foreach my $key (keys %$json_matches_object) { - my $json_field_val = $json_info{$key}; - my $regex_expected_obj = $json->utf8->decode($json_matches_object->{$key}); - print $log "Comparing $key: " . dump($json_field_val) . ",expected value: " . dump($regex_expected_obj); - - if (!Compare($json_field_val, $regex_expected_obj)) { - print $log "$0::$subName WARN check failed:" - . " json compare failed. For field " - . "$key, regex <" . dump($regex_expected_obj) - . "> did not match the result <" . dump($json_field_val) - . ">\n"; - $result = 0; + + foreach my $key (keys %$json_matches_object) { + my $json_field_val = $json_info{$key}; + my $regex_expected_obj = $json->utf8->decode($json_matches_object->{$key}); + print $log "Comparing $key: " . dump($json_field_val) . ",expected value: " . dump($regex_expected_obj); + + if (!Compare($json_field_val, $regex_expected_obj)) { + print $log "$0::$subName WARN check failed:" + . " json compare failed. For field " + . "$key, regex <" . dump($regex_expected_obj) + . "> did not match the result <" . dump($json_field_val) + . ">\n"; + $result = 0; + } } - } } - + #kill it if there is a request to kill if($testCmd->{'kill_job_timeout'}){ - sleep $testCmd->{'kill_job_timeout'}; - my $jobid = $json_hash->{'id'}; - if (!defined $jobid) { - print $log "$0::$subName WARN check failed: " - . "no jobid (id field)found in result"; - $result = 0; - } else { - $self->killJob($testCmd, $jobid, $log); - } + sleep $testCmd->{'kill_job_timeout'}; + my $jobid = $json_hash->{'id'}; + if (!defined $jobid) { + print $log "$0::$subName WARN check failed: " + . "no jobid (id field)found in result"; + $result = 0; + } else { + $self->killJob($testCmd, $jobid, $log); + } } #try to get the call back url request until timeout if ($result == 1 && defined $testCmd->{'check_call_back'}) { - my $timeout = 300; #wait for 5 mins for callback - if(defined $testCmd->{'timeout'}){ - $timeout = $testCmd->{'timeout'}; - } - - my $d = $testCmd->{'http_daemon'}; - $d->timeout($timeout); - my $url_requested; - $testCmd->{'callback_url'} =~ s/\$jobId/$json_hash->{'id'}/g; - print $log "Expanded callback url : <" . $testCmd->{'callback_url'} . ">\n"; - do{ - print $log "Waiting for call back url request\n"; - if (my $c = $d->accept) { - if (my $r = $c->get_request) { - my $durl = $d->url; - chop $durl; - $url_requested = $durl . $r->uri->path ; - print $log "Got request at url <" . $url_requested . ">\n"; - $c->send_status_line(200); - $c->close; - } - undef($c); - } else { - print $log "Timeout on wait on call back url" . "\n"; - $result = 0; + my $timeout = 300; #wait for 5 mins for callback + if(defined $testCmd->{'timeout'}){ + $timeout = $testCmd->{'timeout'}; + } + + my $d = $testCmd->{'http_daemon'}; + $d->timeout($timeout); + my $url_requested; + $testCmd->{'callback_url'} =~ s/\$jobId/$json_hash->{'id'}/g; + print $log "Expanded callback url : <" . $testCmd->{'callback_url'} . ">\n"; + do{ + print $log "Waiting for call back url request\n"; + if (my $c = $d->accept) { + if (my $r = $c->get_request) { + my $durl = $d->url; + chop $durl; + $url_requested = $durl . $r->uri->path ; + print $log "Got request at url <" . $url_requested . ">\n"; + $c->send_status_line(200); + $c->close; + } + undef($c); + } else { + print $log "Timeout on wait on call back url" . "\n"; + $result = 0; + } + }while (defined $url_requested && $url_requested ne $testCmd->{'callback_url'}); + $d->close; + if (!defined $url_requested || $url_requested ne $testCmd->{'callback_url'}) { + print $log "failed to recieve request on call back url"; + $result = 0; } - }while (defined $url_requested && $url_requested ne $testCmd->{'callback_url'}); - $d->close; - if (!defined $url_requested || $url_requested ne $testCmd->{'callback_url'}) { - print $log "failed to recieve request on call back url"; - $result = 0; - } } if ( (defined $testCmd->{'check_job_created'}) || (defined $testCmd->{'check_job_complete'}) || (defined $testCmd->{'check_job_exit_value'}) - || (defined $testCmd->{'check_job_percent_complete'}) ) { - my $jobid = $json_hash->{'id'}; - if (!defined $jobid) { - print $log "$0::$subName WARN check failed: " - . "no jobid (id field)found in result"; - $result = 0; - } else { - my $jobResult = $self->getJobResult($testCmd, $jobid, $log); - my $json = new JSON; - my $res_hash = $json->utf8->decode($jobResult->{'body'}); - if (! defined $res_hash->{'status'}) { - print $log "$0::$subName WARN check failed: " - . "jobresult not defined "; - $result = 0; - } - if (defined($testCmd->{'check_job_complete'}) || defined($testCmd->{'check_job_exit_value'}) - || defined($testCmd->{'check_job_percent_complete'})) { - my $jobComplete; - my $NUM_RETRIES = 60; - my $SLEEP_BETWEEN_RETRIES = 5; - - #first wait for job completion - while ($NUM_RETRIES-- > 0) { - $jobComplete = $res_hash->{'status'}->{'jobComplete'}; - if (defined $jobComplete && (lc($jobComplete) eq "true" || lc($jobComplete) eq "1")) { - last; - } - sleep $SLEEP_BETWEEN_RETRIES; - $jobResult = $self->getJobResult($testCmd, $jobid, $log); - $json = new JSON; - $res_hash = $json->utf8->decode($jobResult->{'body'}); - } - if ( (!defined $jobComplete) || (lc($jobComplete) ne "true" && lc($jobComplete) ne "1")) { - print $log "$0::$subName WARN check failed: " - . " timeout on wait for job completion "; + || (defined $testCmd->{'check_job_percent_complete'}) ) { + my $jobid = $json_hash->{'id'}; + if (!defined $jobid) { + print $log "$0::$subName WARN check failed: " + . "no jobid (id field)found in result"; $result = 0; - } else { - # job has completed, check the runState value - if (defined($testCmd->{'check_job_complete'})) { - my $runState = $res_hash->{'status'}->{'runState'}; - my $runStateVal = $self->getRunStateNum($testCmd->{'check_job_complete'}); - if ( (!defined $runState) || $runState ne $runStateVal) { - print $log "check_job_complete failed. got runState $runState, expected $runStateVal"; - $result = 0; - } - } - if (defined($testCmd->{'check_job_exit_value'})) { - my $exitValue = $res_hash->{'exitValue'}; - my $expectedExitValue = $testCmd->{'check_job_exit_value'}; - if ( (!defined $exitValue) || $exitValue % 128 ne $expectedExitValue) { - print $log "check_job_exit_value failed. got exitValue $exitValue, expected $expectedExitValue"; - $result = 0; - } - } - # check the percentComplete value - if (defined($testCmd->{'check_job_percent_complete'})) { - my $pcValue = $res_hash->{'percentComplete'}; - my $expectedPercentComplete = $testCmd->{'check_job_percent_complete'}; - if ( (!defined $pcValue) || $pcValue !~ m/$expectedPercentComplete/ ) { - print $log "check_job_percent_complete failed. got percentComplete $pcValue, expected $expectedPercentComplete"; + } else { + my $jobResult = $self->getJobResult($testCmd, $jobid, $log); + my $json = new JSON; + my $res_hash = $json->utf8->decode($jobResult->{'body'}); + if (! defined $res_hash->{'status'}) { + print $log "$0::$subName WARN check failed: " + . "jobresult not defined "; $result = 0; - } - } - } - - #Check userargs - print $log "$0::$subName INFO Checking userargs"; - my @options = @{$testCmd->{'post_options'}}; - if( !defined $res_hash->{'userargs'}){ - print $log "$0::$subName INFO expected userargs" - . " but userargs not defined\n"; - $result = 0; - } - - #create exp_userargs hash from @options - my %exp_userargs = (); - foreach my $opt ( @options ){ - print $log "opt $opt"; - my ($key, $val) = split q:=:, $opt, 2; - if(defined $exp_userargs{$key}){ - - #if we have already seen this value - #then make the value an array and push new value in - if(ref($exp_userargs{$key}) eq ""){ - my @ar = ($exp_userargs{$key}); - $exp_userargs{$key} = \@ar; - } - my $ar = $exp_userargs{$key}; - push @$ar, ($val); - } - else{ - $exp_userargs{$key} = $val; - } - } - - my %r_userargs = %{$res_hash->{'userargs'}}; - foreach my $key( keys %exp_userargs){ - if($key eq 'inputreader'){ - next; - } - if( !defined $r_userargs{$key}){ - print $log "$0::$subName INFO $key not found in userargs \n"; - $result = 0; - next; - } - - print $log "$0::$subName DEBUG comparing expected " - . " $key ->" . dump($exp_userargs{$key}) - . " With result $key ->" . dump($r_userargs{$key}) . "\n"; - - if (!Compare($exp_userargs{$key}, $r_userargs{$key})) { - print $log "$0::$subName WARN check failed:" - . " json compare failed. For field " - . "$key, regex <" . dump($r_userargs{$key}) - . "> did not match the result <" . dump($exp_userargs{$key}) - . ">\n"; - $result = 0; - } - } - if ($result != 0 && $testCmd->{'check_logs'}) { - my $testCmdBasics = $self->copyTestBasicConfig($testCmd); - $testCmdBasics->{'method'} = 'GET'; - $testCmdBasics->{'url'} = ':WEBHDFS_URL:/webhdfs/v1:OUTDIR:' . '/status/logs?op=LISTSTATUS'; - my $curl_result = $self->execCurlCmd($testCmdBasics, "", $log); - my $path = JSON::Path->new("FileStatuses.FileStatus[*].pathSuffix"); - my @value = $path->values($curl_result->{'body'}); - if ($testCmd->{'check_logs'}->{'job_num'} && $testCmd->{'check_logs'}->{'job_num'} ne (scalar @value)-1) { - print $log "$0::$subName INFO check failed: " - . " Expect " . $testCmd->{'check_logs'}->{'job_num'} . " jobs in logs, but get " . scalar @value; - $result = 0; - return $result; } - foreach my $jobid (@value) { - if ($jobid eq 'list.txt') { - next; - } - my $testCmdBasics = $self->copyTestBasicConfig($testCmd); - $testCmdBasics->{'method'} = 'GET'; - $testCmdBasics->{'url'} = ':WEBHDFS_URL:/webhdfs/v1:OUTDIR:' . '/status/logs/' . $jobid . '?op=LISTSTATUS'; - my $curl_result = $self->execCurlCmd($testCmdBasics, "", $log); - - my $path = JSON::Path->new("FileStatuses.FileStatus[*]"); - my @value = $path->values($curl_result->{'body'}); - - my $foundjobconf = 0; - foreach my $elem (@value) { - if ($elem->{'pathSuffix'} eq "job.xml.html") { - $foundjobconf = 1; - if ($elem->{'length'} eq "0") { - print $log "$0::$subName INFO check failed: " - . " job.xml.html for " . $jobid . " is empty"; - $result = 0; - return $result; - } - next; + if (defined($testCmd->{'check_job_complete'}) || defined($testCmd->{'check_job_exit_value'}) + || defined($testCmd->{'check_job_percent_complete'})) { + my $jobComplete; + my $NUM_RETRIES = 60; + my $SLEEP_BETWEEN_RETRIES = 5; + + #first wait for job completion + while ($NUM_RETRIES-- > 0) { + $jobComplete = $res_hash->{'status'}->{'jobComplete'}; + if (defined $jobComplete && (lc($jobComplete) eq "true" || lc($jobComplete) eq "1")) { + last; + } + sleep $SLEEP_BETWEEN_RETRIES; + $jobResult = $self->getJobResult($testCmd, $jobid, $log); + $json = new JSON; + $res_hash = $json->utf8->decode($jobResult->{'body'}); + } + if ( (!defined $jobComplete) || (lc($jobComplete) ne "true" && lc($jobComplete) ne "1")) { + print $log "$0::$subName WARN check failed: " + . " timeout on wait for job completion "; + $result = 0; + } else { + # job has completed, check the runState value + if (defined($testCmd->{'check_job_complete'})) { + my $runState = $res_hash->{'status'}->{'runState'}; + my $runStateVal = $self->getRunStateNum($testCmd->{'check_job_complete'}); + if ( (!defined $runState) || $runState ne $runStateVal) { + print $log "check_job_complete failed. got runState $runState, expected $runStateVal"; + $result = 0; + } + } + if (defined($testCmd->{'check_job_exit_value'})) { + my $exitValue = $res_hash->{'exitValue'}; + my $expectedExitValue = $testCmd->{'check_job_exit_value'}; + if ( (!defined $exitValue) || $exitValue % 128 ne $expectedExitValue) { + print $log "check_job_exit_value failed. got exitValue $exitValue, expected $expectedExitValue"; + $result = 0; + } + } + # check the percentComplete value + if (defined($testCmd->{'check_job_percent_complete'})) { + my $pcValue = $res_hash->{'percentComplete'}; + my $expectedPercentComplete = $testCmd->{'check_job_percent_complete'}; + if ( (!defined $pcValue) || $pcValue !~ m/$expectedPercentComplete/ ) { + print $log "check_job_percent_complete failed. got percentComplete $pcValue, expected $expectedPercentComplete"; + $result = 0; + } + } } - my $attempt = $elem->{'pathSuffix'}; - my $testCmdBasics = $self->copyTestBasicConfig($testCmd); - $testCmdBasics->{'method'} = 'GET'; - $testCmdBasics->{'url'} = ':WEBHDFS_URL:/webhdfs/v1:OUTDIR:' . '/status/logs/' . $jobid . '/' . $attempt . '?op=LISTSTATUS'; - my $curl_result = $self->execCurlCmd($testCmdBasics, "", $log); - my $path = JSON::Path->new("FileStatuses.FileStatus[*].pathSuffix"); - my @value = $path->values($curl_result->{'body'}); - my @files = ('stderr', 'stdout', 'syslog'); - foreach my $file (@files) { - if ( !grep( /$file/, @value ) ) { - print $log "$0::$subName INFO check failed: " - . " Cannot find " . $file . " in logs/" . $attempt; + + #Check userargs + print $log "$0::$subName INFO Checking userargs"; + my @options = @{$testCmd->{'post_options'}}; + if( !defined $res_hash->{'userargs'}){ + print $log "$0::$subName INFO expected userargs" + . " but userargs not defined\n"; $result = 0; - return $result; - } } - $path = JSON::Path->new("FileStatuses.FileStatus[*].length"); - @value = $path->values($curl_result->{'body'}); - my $foundnonzerofile = 0; - foreach my $length (@value) { - if ($length ne "0") { - $foundnonzerofile = 1; - } + + #create exp_userargs hash from @options + my %exp_userargs = (); + foreach my $opt ( @options ){ + print $log "opt $opt"; + my ($key, $val) = split q:=:, $opt, 2; + if(defined $exp_userargs{$key}){ + + #if we have already seen this value + #then make the value an array and push new value in + if(ref($exp_userargs{$key}) eq ""){ + my @ar = ($exp_userargs{$key}); + $exp_userargs{$key} = \@ar; + } + my $ar = $exp_userargs{$key}; + push @$ar, ($val); + } + else{ + $exp_userargs{$key} = $val; + } + } + + my %r_userargs = %{$res_hash->{'userargs'}}; + foreach my $key( keys %exp_userargs){ + if($key eq 'inputreader'){ + next; + } + if( !defined $r_userargs{$key}){ + print $log "$0::$subName INFO $key not found in userargs \n"; + $result = 0; + next; + } + + print $log "$0::$subName DEBUG comparing expected " + . " $key ->" . dump($exp_userargs{$key}) + . " With result $key ->" . dump($r_userargs{$key}) . "\n"; + + if (!Compare($exp_userargs{$key}, $r_userargs{$key})) { + print $log "$0::$subName WARN check failed:" + . " json compare failed. For field " + . "$key, regex <" . dump($r_userargs{$key}) + . "> did not match the result <" . dump($exp_userargs{$key}) + . ">\n"; + $result = 0; + } } - if (!$foundnonzerofile) { - print $log "$0::$subName INFO check failed: " - . " All files in logs/" . $attempt . " are empty"; - $result = 0; - return $result; + if ($result != 0 && $testCmd->{'check_logs'}) { + my $testCmdBasics = $self->copyTestBasicConfig($testCmd); + $testCmdBasics->{'method'} = 'GET'; + $testCmdBasics->{'url'} = ':WEBHDFS_URL:/webhdfs/v1:OUTDIR:' . '/status/logs?op=LISTSTATUS'; + my $curl_result = $self->execCurlCmd($testCmdBasics, "", $log); + my $path = JSON::Path->new("FileStatuses.FileStatus[*].pathSuffix"); + my @value = $path->values($curl_result->{'body'}); + if ($testCmd->{'check_logs'}->{'job_num'} && $testCmd->{'check_logs'}->{'job_num'} ne (scalar @value)-1) { + print $log "$0::$subName INFO check failed: " + . " Expect " . $testCmd->{'check_logs'}->{'job_num'} . " jobs in logs, but get " . scalar @value; + $result = 0; + return $result; + } + foreach my $jobid (@value) { + if ($jobid eq 'list.txt') { + next; + } + my $testCmdBasics = $self->copyTestBasicConfig($testCmd); + $testCmdBasics->{'method'} = 'GET'; + $testCmdBasics->{'url'} = ':WEBHDFS_URL:/webhdfs/v1:OUTDIR:' . '/status/logs/' . $jobid . '?op=LISTSTATUS'; + my $curl_result = $self->execCurlCmd($testCmdBasics, "", $log); + + my $path = JSON::Path->new("FileStatuses.FileStatus[*]"); + my @value = $path->values($curl_result->{'body'}); + + my $foundjobconf = 0; + foreach my $elem (@value) { + if ($elem->{'pathSuffix'} eq "job.xml.html") { + $foundjobconf = 1; + if ($elem->{'length'} eq "0") { + print $log "$0::$subName INFO check failed: " + . " job.xml.html for " . $jobid . " is empty"; + $result = 0; + return $result; + } + next; + } + my $attempt = $elem->{'pathSuffix'}; + my $testCmdBasics = $self->copyTestBasicConfig($testCmd); + $testCmdBasics->{'method'} = 'GET'; + $testCmdBasics->{'url'} = ':WEBHDFS_URL:/webhdfs/v1:OUTDIR:' . '/status/logs/' . $jobid . '/' . $attempt . '?op=LISTSTATUS'; + my $curl_result = $self->execCurlCmd($testCmdBasics, "", $log); + my $path = JSON::Path->new("FileStatuses.FileStatus[*].pathSuffix"); + my @value = $path->values($curl_result->{'body'}); + my @files = ('stderr', 'stdout', 'syslog'); + foreach my $file (@files) { + if ( !grep( /$file/, @value ) ) { + print $log "$0::$subName INFO check failed: " + . " Cannot find " . $file . " in logs/" . $attempt; + $result = 0; + return $result; + } + } + $path = JSON::Path->new("FileStatuses.FileStatus[*].length"); + @value = $path->values($curl_result->{'body'}); + my $foundnonzerofile = 0; + foreach my $length (@value) { + if ($length ne "0") { + $foundnonzerofile = 1; + } + } + if (!$foundnonzerofile) { + print $log "$0::$subName INFO check failed: " + . " All files in logs/" . $attempt . " are empty"; + $result = 0; + return $result; + } + } + if (!$foundjobconf) { + print $log "$0::$subName INFO check failed: " + . " Cannot find job.xml.html for " . $jobid; + $result = 0; + return $result; + } + } } - } - if (!$foundjobconf) { - print $log "$0::$subName INFO check failed: " - . " Cannot find job.xml.html for " . $jobid; - $result = 0; - return $result; - } } - } } - } } return $result; - } +} ############################################################################## # Check whether we should be running this test or not. @@ -1044,33 +1033,33 @@ ($$) ############################################################################### sub setLocationPermGroup{ - my ($self, $job_info, $testCmd, $log) = @_; - my $location = $job_info->{'location'}; - $location =~ /hdfs.*:\d+(\/.*)\/(.*)/; - my $dir = $1; - my $file = $2; - - my $testCmdBasics = $self->copyTestBasicConfig($testCmd); - $testCmdBasics->{'method'} = 'GET'; - $testCmdBasics->{'num'} = $testCmdBasics->{'num'} . "_checkFile"; - $testCmdBasics->{'url'} = ':WEBHDFS_URL:/webhdfs/v1' - . $dir . '?op=LISTSTATUS'; - - - my $result = $self->execCurlCmd($testCmdBasics, "", $log); - - my $json = new JSON; - my $json_hash = $json->utf8->decode($result->{'body'}); - my @filestatuses = @{$json_hash->{'FileStatuses'}->{'FileStatus'}}; - foreach my $filestatus (@filestatuses){ - if($filestatus->{'pathSuffix'} eq $file){ - $job_info->{'location_perms'} = numPermToStringPerm($filestatus->{'permission'}); - $job_info->{'location_group'} = $filestatus->{'group'}; - $job_info->{'location_owner'} = $filestatus->{'owner'}; - last; - } + my ($self, $job_info, $testCmd, $log) = @_; + my $location = $job_info->{'location'}; + $location =~ /hdfs.*:\d+(\/.*)\/(.*)/; + my $dir = $1; + my $file = $2; + + my $testCmdBasics = $self->copyTestBasicConfig($testCmd); + $testCmdBasics->{'method'} = 'GET'; + $testCmdBasics->{'num'} = $testCmdBasics->{'num'} . "_checkFile"; + $testCmdBasics->{'url'} = ':WEBHDFS_URL:/webhdfs/v1' + . $dir . '?op=LISTSTATUS'; + + + my $result = $self->execCurlCmd($testCmdBasics, "", $log); + + my $json = new JSON; + my $json_hash = $json->utf8->decode($result->{'body'}); + my @filestatuses = @{$json_hash->{'FileStatuses'}->{'FileStatus'}}; + foreach my $filestatus (@filestatuses){ + if($filestatus->{'pathSuffix'} eq $file){ + $job_info->{'location_perms'} = numPermToStringPerm($filestatus->{'permission'}); + $job_info->{'location_group'} = $filestatus->{'group'}; + $job_info->{'location_owner'} = $filestatus->{'owner'}; + last; + } - } + } } @@ -1079,7 +1068,7 @@ ($$) sub dec2bin { my $decimal = shift; my $binary = unpack("B32", pack("N", $decimal)); - $binary =~ s/^0+(?=\d)//; # remove leading zeros + $binary =~ s/^0+(?=\d)//; # remove leading zeros return $binary; } @@ -1091,19 +1080,19 @@ sub digitPermToStringPerm{ my $stringPerm = ""; if($binaryPerm =~ /1\d\d$/){ $stringPerm .= "r"; - }else{ + }else{ $stringPerm .= "-"; } if($binaryPerm =~ /\d1\d$/){ $stringPerm .= "w"; - }else{ + }else{ $stringPerm .= "-"; } if($binaryPerm =~ /\d\d1$/){ $stringPerm .= "x"; - }else{ + }else{ $stringPerm .= "-"; } @@ -1122,72 +1111,72 @@ sub numPermToStringPerm{ ############################################################################### sub getRunStateNum{ - my ($self, $job_complete_state) = @_; - if (lc($job_complete_state) eq 'success') { - return 2; - } elsif (lc($job_complete_state) eq 'failure') { - return 3; - } elsif (lc($job_complete_state) eq 'killed') { - return 5; - } + my ($self, $job_complete_state) = @_; + if (lc($job_complete_state) eq 'success') { + return 2; + } elsif (lc($job_complete_state) eq 'failure') { + return 3; + } elsif (lc($job_complete_state) eq 'killed') { + return 5; + } } ############################################################################### sub getJobResult{ - my ($self, $testCmd, $jobid, $log) = @_; - my $testCmdBasics = $self->copyTestBasicConfig($testCmd); - $testCmdBasics->{'method'} = 'GET'; - $testCmdBasics->{'num'} = $testCmdBasics->{'num'} . "_jobStatusCheck"; - $testCmdBasics->{'url'} = ':TEMPLETON_URL:/templeton/v1/jobs/' - . $jobid . '?' . "user.name=:UNAME:" ; - return $self->execCurlCmd($testCmdBasics, "", $log); + my ($self, $testCmd, $jobid, $log) = @_; + my $testCmdBasics = $self->copyTestBasicConfig($testCmd); + $testCmdBasics->{'method'} = 'GET'; + $testCmdBasics->{'num'} = $testCmdBasics->{'num'} . "_jobStatusCheck"; + $testCmdBasics->{'url'} = ':TEMPLETON_URL:/templeton/v1/jobs/' + . $jobid . '?' . "user.name=:UNAME:" ; + return $self->execCurlCmd($testCmdBasics, "", $log); } ############################################################################### sub killJob{ - my ($self, $testCmd, $jobid, $log) = @_; - my $testCmdBasics = $self->copyTestBasicConfig($testCmd); - $testCmdBasics->{'method'} = 'DELETE'; - $testCmdBasics->{'num'} = $testCmdBasics->{'num'} . "_killJob"; - $testCmdBasics->{'url'} = ':TEMPLETON_URL:/templeton/v1/jobs/' - . $jobid . '?' . "user.name=:UNAME:" ; - return $self->execCurlCmd($testCmdBasics, "", $log); + my ($self, $testCmd, $jobid, $log) = @_; + my $testCmdBasics = $self->copyTestBasicConfig($testCmd); + $testCmdBasics->{'method'} = 'DELETE'; + $testCmdBasics->{'num'} = $testCmdBasics->{'num'} . "_killJob"; + $testCmdBasics->{'url'} = ':TEMPLETON_URL:/templeton/v1/jobs/' + . $jobid . '?' . "user.name=:UNAME:" ; + return $self->execCurlCmd($testCmdBasics, "", $log); } ############################################################################### #Copy test config essential for running a sub command sub copyTestBasicConfig{ - my ($self, $testCmd) = @_; - my %testCmdBasics; - foreach my $key (keys %$testCmd) { - if ($key ne 'method' - && $key ne 'url' - && $key ne 'upload_file' - && $key ne 'post_options' - ) { - $testCmdBasics{$key} = $testCmd->{$key}; + my ($self, $testCmd) = @_; + my %testCmdBasics; + foreach my $key (keys %$testCmd) { + if ($key ne 'method' + && $key ne 'url' + && $key ne 'upload_file' + && $key ne 'post_options' + ) { + $testCmdBasics{$key} = $testCmd->{$key}; + } } - } - # $testCmdBasics{'localpath'} = $testCmd->{'localpath'}; - # $testCmdBasics{'group'} = $testCmd->{'group'}; - # $testCmdBasics{'num'} = $testCmd->{'num'}; - return \%testCmdBasics; + # $testCmdBasics{'localpath'} = $testCmd->{'localpath'}; + # $testCmdBasics{'group'} = $testCmd->{'group'}; + # $testCmdBasics{'num'} = $testCmd->{'num'}; + return \%testCmdBasics; } ############################################################################### sub checkResStatusCode{ - my ($self, $testResult, $e_status_code, $log) = @_; - my $subName = (caller(0))[3]; + my ($self, $testResult, $e_status_code, $log) = @_; + my $subName = (caller(0))[3]; - # print STDERR "expected " . $e_status_code . " was " . $testResult->{'status_code'}; + # print STDERR "expected " . $e_status_code . " was " . $testResult->{'status_code'}; - if (!defined $testResult->{'status_code'} || - $testResult->{'status_code'} != $e_status_code) { - print $log "$0::$subName INFO Check failed: status_code " . - "$e_status_code expected, test returned " . - "<$testResult->{'status_code'}>\n"; - return 0; - } - return 1; + if (!defined $testResult->{'status_code'} || + $testResult->{'status_code'} != $e_status_code) { + print $log "$0::$subName INFO Check failed: status_code " . + "$e_status_code expected, test returned " . + "<$testResult->{'status_code'}>\n"; + return 0; + } + return 1; } @@ -1210,8 +1199,8 @@ sub checkResStatusCode{ # None # sub recordResults - { - } +{ +} ############################################################################### # Sub: cleanup @@ -1225,13 +1214,13 @@ sub recordResults # testResult - reference to hash returned by runTest. # benchmarkResult - reference to hash returned by generateBenchmark. # log - reference to a stream pointer for the logs -# +# # Returns: # None # sub cleanup - { - } +{ +} ############################################################################### # Sub: run @@ -1254,7 +1243,7 @@ sub cleanup # failed. # sub run - { +{ my ($self, $testsToRun, $testsToMatch, $cfg, $log, $dbh, $testStatuses, $confFile, $startat, $logname ) = @_; @@ -1272,8 +1261,8 @@ sub run # Read the global keys foreach (keys(%$cfg)) { - next if $_ eq 'groups'; - $globalHash{$_} = $cfg->{$_}; + next if $_ eq 'groups'; + $globalHash{$_} = $cfg->{$_}; } $globalHash{$_} = $cfg->{$_}; @@ -1285,344 +1274,335 @@ sub run my %groupExecuted; foreach my $group (@{$cfg->{'groups'}}) { - - print $log "INFO $subName at ".__LINE__.": Running TEST GROUP(".$group->{'name'}.")\n"; - - my %groupHash = %globalHash; - $groupHash{'group'} = $group->{'name'}; - - # Read the group keys - foreach (keys(%$group)) { - next if $_ eq 'tests'; - $groupHash{$_} = $group->{$_}; - } - - - # Run each test - foreach my $test (@{$group->{'tests'}}) { - # Check if we're supposed to run this one or not. - if (!$runAll) { - # check if we are supposed to run this test or not. - my $foundIt = 0; - foreach (@$testsToRun) { - if (/^$groupHash{'group'}(_[0-9]+)?$/) { - if (not defined $1) { - # In this case it's just the group name, so we'll - # run every test in the group - $foundIt = 1; - last; - } else { - # maybe, it at least matches the group - my $num = "_" . $test->{'num'}; - if ($num eq $1) { - $foundIt = 1; - last; + + print $log "INFO $subName at ".__LINE__.": Running TEST GROUP(".$group->{'name'}.")\n"; + + my %groupHash = %globalHash; + $groupHash{'group'} = $group->{'name'}; + + # Read the group keys + foreach (keys(%$group)) { + next if $_ eq 'tests'; + $groupHash{$_} = $group->{$_}; + } + + + # Run each test + foreach my $test (@{$group->{'tests'}}) { + # Check if we're supposed to run this one or not. + if (!$runAll) { + # check if we are supposed to run this test or not. + my $foundIt = 0; + foreach (@$testsToRun) { + if (/^$groupHash{'group'}(_[0-9]+)?$/) { + if (not defined $1) { + # In this case it's just the group name, so we'll + # run every test in the group + $foundIt = 1; + last; + } else { + # maybe, it at least matches the group + my $num = "_" . $test->{'num'}; + if ($num eq $1) { + $foundIt = 1; + last; + } + } + } } - } - } - } - foreach (@$testsToMatch) { - my $protoName = $groupHash{'group'} . "_" . $test->{'num'}; - if ($protoName =~ /$_/) { - if (not defined $1) { - # In this case it's just the group name, so we'll - # run every test in the group - $foundIt = 1; - last; - } else { - # maybe, it at least matches the group - my $num = "_" . $test->{'num'}; - if ($num eq $1) { - $foundIt = 1; - last; + foreach (@$testsToMatch) { + my $protoName = $groupHash{'group'} . "_" . $test->{'num'}; + if ($protoName =~ /$_/) { + if (not defined $1) { + # In this case it's just the group name, so we'll + # run every test in the group + $foundIt = 1; + last; + } else { + # maybe, it at least matches the group + my $num = "_" . $test->{'num'}; + if ($num eq $1) { + $foundIt = 1; + last; + } + } + } } - } + + next unless $foundIt; } - } - next unless $foundIt; - } + # This is a test, so run it. + my %testHash = %groupHash; + foreach (keys(%$test)) { + $testHash{$_} = $test->{$_}; + } - # This is a test, so run it. - my %testHash = %groupHash; - foreach (keys(%$test)) { - $testHash{$_} = $test->{$_}; - } + my $testName = $testHash{'group'} . "_" . $testHash{'num'}; + + # if ( $groupExecuted{ $group->{'name'} }== 0 ){ + # $groupExecuted{ $group->{'name'} }=1; + # + # my $xmlDir= $globalHash{'localxmlpathbase'}."/run".$globalHash->{'UID'}; + # mkpath( [ $xmlDir ] , 1, 0777) if ( ! -e $xmlDir ); + # + # my $filename = $group->{'name'}.".xml"; + # $report = new TestReport ( $properties, "$xmlDir/$filename" ); + # $report->purge(); + # } + + # Check that ignore isn't set for this file, group, or test + if (defined $testHash{'ignore'}) { + print $log "Ignoring test $testName, ignore message: " . + $testHash{'ignore'} . "\n"; + next; + } - my $testName = $testHash{'group'} . "_" . $testHash{'num'}; - - # if ( $groupExecuted{ $group->{'name'} }== 0 ){ - # $groupExecuted{ $group->{'name'} }=1; - # - # my $xmlDir= $globalHash{'localxmlpathbase'}."/run".$globalHash->{'UID'}; - # mkpath( [ $xmlDir ] , 1, 0777) if ( ! -e $xmlDir ); - # - # my $filename = $group->{'name'}.".xml"; - # $report = new TestReport ( $properties, "$xmlDir/$filename" ); - # $report->purge(); - # } - - # Check that ignore isn't set for this file, group, or test - if (defined $testHash{'ignore'}) { - print $log "Ignoring test $testName, ignore message: " . - $testHash{'ignore'} . "\n"; - next; - } + # Have we not reached the starting point yet? + if (!$sawstart) { + if ($testName eq $startat) { + $sawstart = 1; + } else { + next; + } + } - # Have we not reached the starting point yet? - if (!$sawstart) { - if ($testName eq $startat) { - $sawstart = 1; - } else { - next; - } - } + # Check that this test doesn't depend on an earlier test or tests + # that failed. Don't abort if that test wasn't run, just assume the + # user knew what they were doing and set it up right. + my $skipThisOne = 0; + foreach (keys(%testHash)) { + if (/^depends_on/ && defined($testStatuses->{$testHash{$_}}) && + $testStatuses->{$testHash{$_}} ne $passedStr) { + print $log "Skipping test $testName, it depended on " . + "$testHash{$_} which returned a status of " . + "$testStatuses->{$testHash{$_}}\n"; + $testStatuses->{$testName} = $dependStr; + $skipThisOne = 1; + last; + } + } + if ($skipThisOne) { + printResults($testStatuses, $log, "Results so far"); + next; + } - # Check that this test doesn't depend on an earlier test or tests - # that failed. Don't abort if that test wasn't run, just assume the - # user knew what they were doing and set it up right. - my $skipThisOne = 0; - foreach (keys(%testHash)) { - if (/^depends_on/ && defined($testStatuses->{$testHash{$_}}) && - $testStatuses->{$testHash{$_}} ne $passedStr) { - print $log "Skipping test $testName, it depended on " . - "$testHash{$_} which returned a status of " . - "$testStatuses->{$testHash{$_}}\n"; - $testStatuses->{$testName} = $dependStr; - $skipThisOne = 1; - last; - } - } - if ($skipThisOne) { - printResults($testStatuses, $log, "Results so far"); - next; - } + print $log "\n******************************************************\n"; + print $log "\nTEST: $confFile::$testName\n"; + print $log "******************************************************\n"; + print $log "Beginning test $testName at " . time . "\n"; + my %dbinfo = ( + 'testrun_id' => $testHash{'trid'}, + 'test_type' => $testHash{'driver'}, + #'test_file' => $testHash{'file'}, + 'test_file' => $confFile, + 'test_group' => $testHash{'group'}, + 'test_num' => $testHash{'num'}, + ); + my $beginTime = time; + my $endTime = 0; + my ($testResult, $benchmarkResult); + eval { + $testResult = $self->runTest(\%testHash, $log); + $benchmarkResult = $self->generateBenchmark(\%testHash, $log); + my $result = + $self->compare($testResult, $benchmarkResult, $log, \%testHash); + $msg = "INFO: $subName() at ".__LINE__.":Test $testName"; + + if ($result eq $self->{'wrong_execution_mode'}) { + $msg .= " SKIPPED"; + $testStatuses->{$testName} = $skippedStr; + } elsif ($result) { + $msg .= " SUCCEEDED"; + $testStatuses->{$testName} = $passedStr; + + } else { + $msg .= " FAILED"; + $testStatuses->{$testName} = $failedStr; - print $log "\n******************************************************\n"; - print $log "\nTEST: $confFile::$testName\n"; - print $log "******************************************************\n"; - print $log "Beginning test $testName at " . time . "\n"; - my %dbinfo = ( - 'testrun_id' => $testHash{'trid'}, - 'test_type' => $testHash{'driver'}, - #'test_file' => $testHash{'file'}, - 'test_file' => $confFile, - 'test_group' => $testHash{'group'}, - 'test_num' => $testHash{'num'}, - ); - my $beginTime = time; - my $endTime = 0; - my ($testResult, $benchmarkResult); - eval { - $testResult = $self->runTest(\%testHash, $log); - $benchmarkResult = $self->generateBenchmark(\%testHash, $log); - my $result = - $self->compare($testResult, $benchmarkResult, $log, \%testHash); - $msg = "INFO: $subName() at ".__LINE__.":Test $testName"; - - if ($result eq $self->{'wrong_execution_mode'}) { - $msg .= " SKIPPED"; - $testStatuses->{$testName} = $skippedStr; - } elsif ($result) { - $msg .= " SUCCEEDED"; - $testStatuses->{$testName} = $passedStr; - - } else { - $msg .= " FAILED"; - $testStatuses->{$testName} = $failedStr; - - } - $endTime = time; - $msg .= "\nEnding test $testName at " . $endTime ."\n"; - #print $msg; - print $log $msg; - $duration = $endTime - $beginTime; - $dbinfo{'duration'} = $duration; - $self->recordResults($result, $testResult - , $benchmarkResult, \%dbinfo, $log); - - }; - - - if ($@) { - $msg= "ERROR $subName at : ".__LINE__." Failed to run test $testName <$@>\n"; - $msg .= "Ending test $testName at " . time ."\n"; - #print $msg; - print $log $msg; - $testStatuses->{$testName} = $abortedStr; - $dbinfo{'duration'} = $duration; - } + } + $endTime = time; + $msg .= "\nEnding test $testName at " . $endTime ."\n"; + #print $msg; + print $log $msg; + $duration = $endTime - $beginTime; + $dbinfo{'duration'} = $duration; + $self->recordResults($result, $testResult + , $benchmarkResult, \%dbinfo, $log); + + }; + + + if ($@) { + $msg= "ERROR $subName at : ".__LINE__." Failed to run test $testName <$@>\n"; + $msg .= "Ending test $testName at " . time ."\n"; + #print $msg; + print $log $msg; + $testStatuses->{$testName} = $abortedStr; + $dbinfo{'duration'} = $duration; + } - eval { - $dbinfo{'status'} = $testStatuses->{$testName}; - if ($dbh) { - $dbh->insertTestCase(\%dbinfo); - } - }; - if ($@) { - chomp $@; - warn "Failed to insert test case info, error <$@>\n"; - } + eval { + $dbinfo{'status'} = $testStatuses->{$testName}; + if ($dbh) { + $dbh->insertTestCase(\%dbinfo); + } + }; + if ($@) { + chomp $@; + warn "Failed to insert test case info, error <$@>\n"; + } - $self->cleanup($testStatuses->{$testName}, \%testHash, $testResult, - $benchmarkResult, $log); - #$report->testcase( $group->{'name'}, $testName, $duration, $msg, $testStatuses->{$testName}, $testResult ) if ( $report ); - $report->testcase( $group->{'name'}, $testName, $duration, $msg, $testStatuses->{$testName} ) if ( $report ); - $groupDuration = $groupDuration + $duration; - $totalDuration = $totalDuration + $duration; - printResults( $testStatuses, $log, "Results so far" ); - } + $self->cleanup($testStatuses->{$testName}, \%testHash, $testResult, + $benchmarkResult, $log); + #$report->testcase( $group->{'name'}, $testName, $duration, $msg, $testStatuses->{$testName}, $testResult ) if ( $report ); + $report->testcase( $group->{'name'}, $testName, $duration, $msg, $testStatuses->{$testName} ) if ( $report ); + $groupDuration = $groupDuration + $duration; + $totalDuration = $totalDuration + $duration; + printResults( $testStatuses, $log, "Results so far" ); + } - if ( $report ) { - $report->systemOut( $logname, $group->{'name'}); - printGroupResultsXml( $report, $group->{'name'}, $testStatuses, $groupDuration ); - } - $report = 0; - $groupDuration=0; + if ( $report ) { + $report->systemOut( $logname, $group->{'name'}); + printGroupResultsXml( $report, $group->{'name'}, $testStatuses, $groupDuration ); + } + $report = 0; + $groupDuration=0; } # Do the global cleanup $self->globalCleanup(\%globalHash, $log); - } +} # TODO These should be removed sub tmpIPCRun(){ - my $self = shift; - my $subName = (caller(0))[3]; - my $runningSubName= shift; - my $refCmd = shift; - my @cmd = @$refCmd; - my $log = shift; - my $msg = shift; - - print $log "$0::$subName INFO Running ( @cmd )\n"; - - my $result= `@cmd`; - if ( $@ ) { - my $msg= "$0::$subName FATAL Failed to run from $runningSubName $msg < $@ >\n$result\n"; - print $log $msg; - die "$msg"; - } + my $self = shift; + my $subName = (caller(0))[3]; + my $runningSubName= shift; + my $refCmd = shift; + my @cmd = @$refCmd; + my $log = shift; + my $msg = shift; + + print $log "$0::$subName INFO Running ( @cmd )\n"; + + my $result= `@cmd`; + if ( $@ ) { + my $msg= "$0::$subName FATAL Failed to run from $runningSubName $msg < $@ >\n$result\n"; + print $log $msg; + die "$msg"; + } - return $?; + return $?; } sub tmpIPCRunSplitStdoe { - my $self = shift; - my $subName = (caller(0))[3]; - my $runningSubName= shift; - my $refCmd = shift; - my @cmd = @$refCmd; - my $dir = shift; - my $log = shift; - my $msg = shift; - my $die = shift; - - - my $failed = 0; - - my $outfilename = $dir."out.tmp"; - my $errfilename = $dir."err.tmp"; - print $log "$0::$subName INFO Running from $runningSubName ( @cmd 1>$outfilename 2>$errfilename )\n"; - #make sure files are writeable - open( TMP, ">$outfilename" ) || die "$0::$subName FATAL: Cannot open $outfilename for writing\n"; - close( TMP ); - open( TMP, ">$errfilename" ) || die "$0::$subName FATAL: Cannot open $errfilename for writing\n"; - close( TMP ); - - #RUN CMD - my $msg; - print $log `@cmd 1>$outfilename 2>$errfilename`; - - my $failed=0; - if ( $@ ) { - $msg= "$0::$subName FATAL < $@ >\n"; - $failed++; - } - - #READ FILES - my $stdout=""; - my $stderr="";; - open( TMP, "$outfilename" ) || die "$0::$subName FATAL: Cannot open $outfilename for reading\n"; - while ( ) { - $stdout .= $_; - } - close( TMP ); - - open( TMP, "$errfilename" ) || die "$0::$subName FATAL: Cannot open $errfilename for reading\n"; - while ( ) { - $stderr .= $_; - } - close( TMP ); - - #DIE IF Test Failed, otherwise return stdout and stderr - if ( $failed ) { - - $msg = "$0::$subName FATAL: Faied from $runningSubName \nSTDOUT:" . $stdout . "\nSTDERR:" . $stderr . "\n" if ( $failed ); - print $log "$msg"; - die $msg if ( $die != "1" ); #die by defaultast - return ( -1, $stdout, $stderr ); - - } - - return ( $?, $stdout, $stderr); + my $self = shift; + my $subName = (caller(0))[3]; + my $runningSubName= shift; + my $refCmd = shift; + my @cmd = @$refCmd; + my $dir = shift; + my $log = shift; + my $msg = shift; + my $die = shift; + + + my $failed = 0; + + my $outfilename = $dir."out.tmp"; + my $errfilename = $dir."err.tmp"; + print $log "$0::$subName INFO Running from $runningSubName ( @cmd 1>$outfilename 2>$errfilename )\n"; + #make sure files are writeable + open( TMP, ">$outfilename" ) || die "$0::$subName FATAL: Cannot open $outfilename for writing\n"; + close( TMP ); + open( TMP, ">$errfilename" ) || die "$0::$subName FATAL: Cannot open $errfilename for writing\n"; + close( TMP ); + + #RUN CMD + my $msg; + print $log `@cmd 1>$outfilename 2>$errfilename`; + + my $failed=0; + if ( $@ ) { + $msg= "$0::$subName FATAL < $@ >\n"; + $failed++; + } + + #READ FILES + my $stdout=""; + my $stderr="";; + open( TMP, "$outfilename" ) || die "$0::$subName FATAL: Cannot open $outfilename for reading\n"; + while ( ) { + $stdout .= $_; + } + close( TMP ); + + open( TMP, "$errfilename" ) || die "$0::$subName FATAL: Cannot open $errfilename for reading\n"; + while ( ) { + $stderr .= $_; + } + close( TMP ); + + #DIE IF Test Failed, otherwise return stdout and stderr + if ( $failed ) { + + $msg = "$0::$subName FATAL: Faied from $runningSubName \nSTDOUT:" . $stdout . "\nSTDERR:" . $stderr . "\n" if ( $failed ); + print $log "$msg"; + die $msg if ( $die != "1" ); #die by defaultast + return ( -1, $stdout, $stderr ); + + } + + return ( $?, $stdout, $stderr); } sub tmpIPCRunJoinStdoe { - my $self = shift; - my $subName = (caller(0))[3]; - my $runningSubName= shift; - my $refCmd = shift; - my @cmd = @$refCmd; - my $outfilename= shift; - my $log = shift; - my $msg = shift; - my $die = shift; - - #make sure files are writeable - open( TMP, ">$outfilename" ) || die "$0::$subName FATAL: Cannot open $outfilename for writing\n"; - close( TMP ); - - #RUN CMD - my $msg; - my $failed=0; - print $log "$0::$subName INFO Running ( @cmd 2>&1$outfilename 2>/dev/null )\n"; - print $log `@cmd 2>&1 > $outfilename 2>/dev/null`; - if ( $@ ) { - $failed++; - $msg= "$0::$subName FATAL < $@ >\n"; - } - - #READ FILES - my $stdoe=""; - open( TMP, "$outfilename" ) || die "$0::$subName FATAL: Cannot open $outfilename for reading\n"; - while ( ) { - $stdoe .= $_; - } - close( TMP ); - - if ( $failed ) { - print $log "$msg"; - die $msg if ( $die != "1" ); #die by default - return ( -1 ); - } - return ( $? ); -} + my $self = shift; + my $subName = (caller(0))[3]; + my $runningSubName= shift; + my $refCmd = shift; + my @cmd = @$refCmd; + my $outfilename= shift; + my $log = shift; + my $msg = shift; + my $die = shift; + + #make sure files are writeable + open( TMP, ">$outfilename" ) || die "$0::$subName FATAL: Cannot open $outfilename for writing\n"; + close( TMP ); + + #RUN CMD + my $msg; + my $failed=0; + print $log "$0::$subName INFO Running ( @cmd 2>&1$outfilename 2>/dev/null )\n"; + print $log `@cmd 2>&1 > $outfilename 2>/dev/null`; + if ( $@ ) { + $failed++; + $msg= "$0::$subName FATAL < $@ >\n"; + } -sub isWindows -{ - if($^O =~ /mswin/i) { - return 1; + #READ FILES + my $stdoe=""; + open( TMP, "$outfilename" ) || die "$0::$subName FATAL: Cannot open $outfilename for reading\n"; + while ( ) { + $stdoe .= $_; } - else { - return 0; + close( TMP ); + + if ( $failed ) { + print $log "$msg"; + die $msg if ( $die != "1" ); #die by default + return ( -1 ); } + return ( $? ); } + 1; diff --git hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java index 48ee7cf..b9cb067 100644 --- hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java +++ hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java @@ -40,7 +40,6 @@ import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NotificationEvent; import org.apache.hadoop.hive.metastore.api.PartitionEventType; -import org.apache.hadoop.hive.ql.WindowsPathUtil; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.io.RCFileInputFormat; import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; @@ -109,9 +108,6 @@ public static void startMetaStoreServer() throws Exception { useExternalMS = true; return; } - if (Shell.WINDOWS) { - WindowsPathUtil.convertPathsFromWindowsToHdfs(hcatConf); - } System.setProperty(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname, DbNotificationListener.class.getName()); // turn on db notification listener on metastore @@ -136,9 +132,6 @@ public static HiveConf getConf(){ } public static String fixPath(String path) { - if(!Shell.WINDOWS) { - return path; - } String expectedDir = path.replaceAll("\\\\", "/"); if (!expectedDir.startsWith("/")) { expectedDir = "/" + expectedDir; diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java index e868102..54b8419 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ExecServiceImpl.java @@ -175,53 +175,7 @@ private ExecBean auxRun(String program, List args, Map e LOG.info("Running: " + cmd); ExecBean res = new ExecBean(); - if(Shell.WINDOWS){ - //The default executor is sometimes causing failure on windows. hcat - // command sometimes returns non zero exit status with it. It seems - // to hit some race conditions on windows. - env = execEnv(env); - String[] envVals = new String[env.size()]; - int i=0; - for( Entry kv : env.entrySet()){ - envVals[i++] = kv.getKey() + "=" + kv.getValue(); - LOG.info("Setting " + kv.getKey() + "=" + kv.getValue()); - } - - Process proc; - synchronized (WindowsProcessLaunchLock) { - // To workaround the race condition issue with child processes - // inheriting unintended handles during process launch that can - // lead to hangs on reading output and error streams, we - // serialize process creation. More info available at: - // http://support.microsoft.com/kb/315939 - proc = Runtime.getRuntime().exec(cmd.toStrings(), envVals); - } - - //consume stderr - StreamOutputWriter errorGobbler = new - StreamOutputWriter(proc.getErrorStream(), "ERROR", errStream); - - //consume stdout - StreamOutputWriter outputGobbler = new - StreamOutputWriter(proc.getInputStream(), "OUTPUT", outStream); - - //start collecting input streams - errorGobbler.start(); - outputGobbler.start(); - //execute - try{ - res.exitcode = proc.waitFor(); - } catch (InterruptedException e) { - throw new IOException(e); - } finally { - //flush - errorGobbler.out.flush(); - outputGobbler.out.flush(); - } - } - else { - res.exitcode = executor.execute(cmd, execEnv(env)); - } + res.exitcode = executor.execute(cmd, execEnv(env)); String enc = appConf.get(AppConfig.EXEC_ENCODING_NAME); res.stdout = outStream.toString(enc); diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java index 0ea964f..f0296cb 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java @@ -69,7 +69,6 @@ public EnqueueBean run(String user, Map userArgs, args.addAll(makeBasicArgs(execute, srcFile, otherFiles, statusdir, completedUrl, enablelog, enableJobReconnect)); args.add("--"); - TempletonUtils.addCmdForWindows(args); addHiveMetaStoreTokenArg(); args.add(appConf.hivePath()); @@ -92,18 +91,18 @@ public EnqueueBean run(String user, Map userArgs, for (String prop : appConf.hiveProps()) { args.add("--hiveconf"); - args.add(TempletonUtils.quoteForWindows(prop)); + args.add(prop); } for (String prop : defines) { args.add("--hiveconf"); - args.add(TempletonUtils.quoteForWindows(prop)); + args.add(prop); } for (String hiveArg : hiveArgs) { - args.add(TempletonUtils.quoteForWindows(hiveArg)); + args.add(hiveArg); } if (TempletonUtils.isset(execute)) { args.add("-e"); - args.add(TempletonUtils.quoteForWindows(execute)); + args.add(execute); } else if (TempletonUtils.isset(srcFile)) { args.add("-f"); args.add(TempletonUtils.hadoopFsPath(srcFile, appConf, runAs) diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java index 10ff2c0..84cd5b9 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java @@ -69,7 +69,6 @@ public EnqueueBean run(String user, Map userArgs, String jar, St args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles, enablelog, enableJobReconnect, jobType)); args.add("--"); - TempletonUtils.addCmdForWindows(args); //check if the rest command specified explicitly to use hcatalog if(usesHcatalog){ @@ -89,13 +88,13 @@ public EnqueueBean run(String user, Map userArgs, String jar, St // (which is not very useful since users might not have access to that file system). //This is likely the HIVE-5188 issue args.add("-libjars"); - args.add(TempletonUtils.quoteForWindows(libjarsListAsString)); + args.add(libjarsListAsString); } if (TempletonUtils.isset(files)) { String filesListAsString = TempletonUtils.hadoopFsListAsString(files, appConf, runAs); args.add("-files"); - args.add(TempletonUtils.quoteForWindows(filesListAsString)); + args.add(filesListAsString); } //the token file location comes after mainClass, as a -D prop=val args.add("-D"); @@ -107,10 +106,10 @@ public EnqueueBean run(String user, Map userArgs, String jar, St for (String d : defines) { args.add("-D"); - args.add(TempletonUtils.quoteForWindows(d)); + args.add(d); } for (String arg : jarArgs) { - args.add(TempletonUtils.quoteForWindows(arg)); + args.add(arg); } } catch (FileNotFoundException e) { throw new BadParam(e.getMessage()); diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java index b589917..aeb89df 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java @@ -126,7 +126,6 @@ public EnqueueBean run(String user, Map userArgs, appConf.get(AppConfig.HIVE_PROPS_NAME)); } args.add("--"); - TempletonUtils.addCmdForWindows(args); args.add(appConf.pigPath()); //the token file location should be first argument of pig args.add("-D" + TempletonControllerJob.TOKEN_FILE_ARG_PLACEHOLDER); @@ -135,7 +134,7 @@ public EnqueueBean run(String user, Map userArgs, args.add("-D" + TempletonControllerJob.MAPREDUCE_JOB_TAGS_ARG_PLACEHOLDER); for (String pigArg : pigArgs) { - args.add(TempletonUtils.quoteForWindows(pigArg)); + args.add(pigArg); } if(needsMetastoreAccess) { addHiveMetaStoreTokenArg(); @@ -143,7 +142,7 @@ public EnqueueBean run(String user, Map userArgs, if (TempletonUtils.isset(execute)) { args.add("-execute"); - args.add(TempletonUtils.quoteForWindows(execute)); + args.add(execute); } else if (TempletonUtils.isset(srcFile)) { args.add("-file"); args.add(TempletonUtils.hadoopFsPath(srcFile, appConf, runAs) diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java index 3f2a797..fde5f60 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java @@ -74,12 +74,11 @@ public EnqueueBean run(String user, args.addAll(makeBasicArgs(optionsFile, otherFiles, statusdir, completedUrl, enablelog, enableJobReconnect, libdir)); args.add("--"); - TempletonUtils.addCmdForWindows(args); args.add(appConf.sqoopPath()); if (TempletonUtils.isset(command)) { String[] temArgs = command.split(" "); for (int i = 0; i < temArgs.length; i++) { - args.add(TempletonUtils.quoteForWindows(temArgs[i])); + args.add(temArgs[i]); // The token file location and mapreduce job tag should be right after the tool argument if (i == 0 && !temArgs[i].startsWith("--")) { @@ -97,8 +96,8 @@ public EnqueueBean run(String user, } //the jars in libJars will be localized to CWD of the launcher task; then -libjars will //cause them to be localized for the Sqoop MR job tasks - args.add(TempletonUtils.quoteForWindows("-libjars")); - args.add(TempletonUtils.quoteForWindows(libJars)); + args.add("-libjars"); + args.add(libJars); } } } else if (TempletonUtils.isset(optionsFile)) { diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java index f487d51..839b56a 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java @@ -102,11 +102,11 @@ public EnqueueBean run(String user, Map userArgs, for (String e : cmdenvs) { args.add("-cmdenv"); - args.add(TempletonUtils.quoteForWindows(e)); + args.add(e); } for (String arg : jarArgs) { - args.add(TempletonUtils.quoteForWindows(arg)); + args.add(arg); } return args; diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java index 41ddb9c..71b2941 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java @@ -227,15 +227,6 @@ private static void handleTokenFile(List jarArgsList, String tokenPlaceH //Token is available, so replace the placeholder tokenFile = tokenFile.replaceAll("\"", ""); String tokenArg = tokenProperty + "=" + tokenFile; - if (Shell.WINDOWS) { - try { - tokenArg = TempletonUtils.quoteForWindows(tokenArg); - } catch (BadParam e) { - String msg = "cannot pass " + tokenFile + " to " + tokenProperty; - LOG.error(msg, e); - throw new IOException(msg, e); - } - } for(int i=0; i 0) { - String nonQuotedPart = param; - boolean addQuotes = true; - if (param.charAt(0) == '\"' && param.charAt(param.length() - 1) == '\"') { - if (param.length() < 2) - throw new BadParam("Passed in parameter is incorrectly quoted: " + param); - - addQuotes = false; - nonQuotedPart = param.substring(1, param.length() - 1); - } - - // If we have any quotes other then the outside quotes, throw - if (nonQuotedPart.contains("\"")) { - throw new BadParam("Passed in parameter is incorrectly quoted: " + param); - } - - if (addQuotes) { - param = '\"' + param + '\"'; - } - } - } - return param; - } - - public static void addCmdForWindows(ArrayList args) { - if(Shell.WINDOWS){ - args.add("cmd"); - args.add("/c"); - args.add("call"); - } - } - /** * replaces all occurrences of "\," with ","; returns {@code s} if no modifications needed */ diff --git itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index 257dc28..e641253 100644 --- itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hive.llap.LlapItUtils; import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster; import org.apache.hadoop.hive.metastore.MetaStoreUtils; -import org.apache.hadoop.hive.ql.WindowsPathUtil; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.util.ZooKeeperHiveHelper; import org.apache.hadoop.hive.shims.HadoopShims.MiniDFSShim; @@ -218,7 +217,7 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM // Initialize dfs dfs = ShimLoader.getHadoopShims().getMiniDfs(hiveConf, 4, true, null, isHA); fs = dfs.getFileSystem(); - String uriString = WindowsPathUtil.getHdfsUriString(fs.getUri().toString()); + String uriString = fs.getUri().toString(); // Initialize the execution engine based on cluster type switch (miniClusterType) { diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestReplicationScenarios.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestReplicationScenarios.java index 7836c47..39834cd 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestReplicationScenarios.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestReplicationScenarios.java @@ -86,9 +86,6 @@ public static void setUpBeforeClass() throws Exception { useExternalMS = true; return; } - if (Shell.WINDOWS) { - WindowsPathUtil.convertPathsFromWindowsToHdfs(hconf); - } hconf.setVar(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS, DBNOTIF_LISTENER_CLASSNAME); // turn on db notification listener on metastore diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java index 0d88e09..36f31f4 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java @@ -34,7 +34,6 @@ import org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener; import org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider; import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.ql.WindowsPathUtil; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.security.UserGroupInformation; @@ -59,9 +58,6 @@ protected String getAuthorizationProvider(){ protected HiveConf createHiveConf() throws Exception { HiveConf conf = new HiveConf(this.getClass()); - if (Shell.WINDOWS) { - WindowsPathUtil.convertPathsFromWindowsToHdfs(conf); - } return conf; } diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java index 185dbd5..8fabdb8 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java @@ -28,7 +28,6 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.WindowsPathUtil; import org.apache.hadoop.util.Shell; import org.junit.AfterClass; import org.junit.Assert; @@ -49,9 +48,6 @@ static public void oneTimeSetup() throws Exception { m_dfs = new MiniDFSCluster.Builder(new Configuration()).numDataNodes(1).format(true).build(); conf = new HiveConf(); - if (Shell.WINDOWS) { - WindowsPathUtil.convertPathsFromWindowsToHdfs(conf); - } conf.set(HiveConf.ConfVars.HIVE_SCRATCH_DIR_LOCK.toString(), "true"); conf.set(HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL.toString(), "true"); LoggerFactory.getLogger("SessionState"); diff --git itests/hive-unit/src/test/java/org/apache/hive/service/server/TestHS2ClearDanglingScratchDir.java itests/hive-unit/src/test/java/org/apache/hive/service/server/TestHS2ClearDanglingScratchDir.java index 081ac96..a7aceae 100644 --- itests/hive-unit/src/test/java/org/apache/hive/service/server/TestHS2ClearDanglingScratchDir.java +++ itests/hive-unit/src/test/java/org/apache/hive/service/server/TestHS2ClearDanglingScratchDir.java @@ -25,7 +25,6 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.WindowsPathUtil; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.util.Shell; @@ -38,9 +37,6 @@ public void testScratchDirCleared() throws Exception { MiniDFSCluster m_dfs = new MiniDFSCluster.Builder(new Configuration()).numDataNodes(1).format(true).build(); HiveConf conf = new HiveConf(); conf.addResource(m_dfs.getConfiguration(0)); - if (Shell.WINDOWS) { - WindowsPathUtil.convertPathsFromWindowsToHdfs(conf); - } conf.set(HiveConf.ConfVars.HIVE_SCRATCH_DIR_LOCK.toString(), "true"); conf.set(HiveConf.ConfVars.HIVE_SERVER2_CLEAR_DANGLING_SCRATCH_DIR.toString(), "true"); diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 1eaf7af..0b223f4 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -361,16 +361,10 @@ public void initConf() throws Exception { conf.set(confEntry.getKey(), clusterSpecificConf.get(confEntry.getKey())); } } - - // Windows paths should be converted after MiniMrShim.setupConfiguration() - // since setupConfiguration may overwrite configuration values. - if (Shell.WINDOWS) { - WindowsPathUtil.convertPathsFromWindowsToHdfs(conf); - } } private void setFsRelatedProperties(HiveConf conf, boolean isLocalFs, FileSystem fs) { - String fsUriString = WindowsPathUtil.getHdfsUriString(fs.getUri().toString()); + String fsUriString = fs.getUri().toString(); // Different paths if running locally vs a remote fileSystem. Ideally this difference should not exist. Path warehousePath; @@ -650,7 +644,7 @@ private void setupFileSystem(HadoopShims shims) throws IOException { private void setupMiniCluster(HadoopShims shims, String confDir) throws IOException { - String uriString = WindowsPathUtil.getHdfsUriString(fs.getUri().toString()); + String uriString = fs.getUri().toString(); if (clusterType.getCoreClusterType() == CoreClusterType.TEZ) { if (confDir != null && !confDir.isEmpty()) { @@ -740,8 +734,7 @@ public void addFile(File qf, boolean partial) throws IOException { qMap.put(qf.getName(), query); if (partial) return; - if(checkHadoopVersionExclude(qf.getName(), query) - || checkOSExclude(qf.getName(), query)) { + if(checkHadoopVersionExclude(qf.getName(), query)) { qSkipSet.add(qf.getName()); } @@ -836,35 +829,6 @@ private boolean checkHadoopVersionExclude(String fileName, String query){ return false; } - private boolean checkOSExclude(String fileName, String query){ - // Look for a hint to not run a test on some Hadoop versions - Pattern pattern = Pattern.compile("-- (EX|IN)CLUDE_OS_WINDOWS"); - - // detect whether this query wants to be excluded or included - // on windows - Matcher matcher = pattern.matcher(query); - if (matcher.find()) { - String prefix = matcher.group(1); - if ("EX".equals(prefix)) { - //windows is to be exluded - if(Shell.WINDOWS){ - System.out.println("Due to the OS being windows " + - "adding the query " + fileName + - " to the set of tests to skip"); - return true; - } - } - else if(!Shell.WINDOWS){ - //non windows to be exluded - System.out.println("Due to the OS not being windows " + - "adding the query " + fileName + - " to the set of tests to skip"); - return true; - } - } - return false; - } - private boolean checkNeedJavaSpecificOutput(String fileName, String query) { Pattern pattern = Pattern.compile("-- JAVA_VERSION_SPECIFIC_OUTPUT"); Matcher matcher = pattern.matcher(query); @@ -1827,19 +1791,10 @@ private static int executeDiffCommand(String inFileName, diffCommandArgs.add("-a"); // Ignore changes in the amount of white space - if (ignoreWhiteSpace || Shell.WINDOWS) { + if (ignoreWhiteSpace) { diffCommandArgs.add("-b"); } - // Files created on Windows machines have different line endings - // than files created on Unix/Linux. Windows uses carriage return and line feed - // ("\r\n") as a line ending, whereas Unix uses just line feed ("\n"). - // Also StringBuilder.toString(), Stream to String conversions adds extra - // spaces at the end of the line. - if (Shell.WINDOWS) { - diffCommandArgs.add("--strip-trailing-cr"); // Strip trailing carriage return on input - diffCommandArgs.add("-B"); // Ignore changes whose lines are all blank - } // Add files to compare to the arguments list diffCommandArgs.add(getQuotedString(inFileName)); diffCommandArgs.add(getQuotedString(outFileName)); @@ -1909,7 +1864,7 @@ private static int executeCmd(String[] args, String outFile, String errFile) thr } private static String getQuotedString(String str){ - return Shell.WINDOWS ? String.format("\"%s\"", str) : str; + return str; } public ASTNode parseQuery(String tname) throws Exception { diff --git llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java index 02f7911..dce0c56 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java @@ -197,8 +197,6 @@ public static final String SHUFFLE_TRANSFERTO_ALLOWED = "llap.shuffle.transferTo.allowed"; public static final boolean DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED = true; - public static final boolean WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED = - false; static final String DATA_FILE_NAME = "file.out"; static final String INDEX_FILE_NAME = "file.out.index"; @@ -266,8 +264,7 @@ private ShuffleHandler(Configuration conf) { DEFAULT_SHUFFLE_BUFFER_SIZE); shuffleTransferToAllowed = conf.getBoolean(SHUFFLE_TRANSFERTO_ALLOWED, - (Shell.WINDOWS)?WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED: - DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED); + DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED); ThreadFactory bossFactory = new ThreadFactoryBuilder() .setNameFormat("ShuffleHandler Netty Boss #%d") diff --git llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java index 41ce035..a9b23b6 100644 --- llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java +++ llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java @@ -100,41 +100,9 @@ private MiniLlapCluster(String clusterName, @Nullable MiniZooKeeperCluster miniZ throw new RuntimeException("Could not cleanup test workDir: " + targetWorkDir, e); } - if (Shell.WINDOWS) { - // The test working directory can exceed the maximum path length supported - // by some Windows APIs and cmd.exe (260 characters). To work around this, - // create a symlink in temporary storage with a much shorter path, - // targeting the full path to the test working directory. Then, use the - // symlink as the test working directory. - String targetPath = targetWorkDir.getAbsolutePath(); - File link = new File(System.getProperty("java.io.tmpdir"), - String.valueOf(System.currentTimeMillis())); - String linkPath = link.getAbsolutePath(); - - try { - FileContext.getLocalFSFileContext().delete(new Path(linkPath), true); - } catch (IOException e) { - throw new YarnRuntimeException("could not cleanup symlink: " + linkPath, e); - } - - // Guarantee target exists before creating symlink. - targetWorkDir.mkdirs(); - - Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor( - Shell.getSymlinkCommand(targetPath, linkPath)); - try { - shexec.execute(); - } catch (IOException e) { - throw new YarnRuntimeException(String.format( - "failed to create symlink from %s to %s, shell output: %s", linkPath, - targetPath, shexec.getOutput()), e); - } + targetWorkDir.mkdir(); + this.testWorkDir = targetWorkDir; - this.testWorkDir = link; - } else { - targetWorkDir.mkdir(); - this.testWorkDir = targetWorkDir; - } if (miniZkCluster == null) { ownZkCluster = true; this.zkWorkDir = new File(testWorkDir, "mini-zk-cluster"); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java index 00884cd..4767af1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java @@ -94,8 +94,6 @@ static final String IO_EXCEPTION_BROKEN_PIPE_STRING = "Broken pipe"; static final String IO_EXCEPTION_STREAM_CLOSED = "Stream closed"; - static final String IO_EXCEPTION_PIPE_ENDED_WIN = "The pipe has been ended"; - static final String IO_EXCEPTION_PIPE_CLOSED_WIN = "The pipe is being closed"; /** * sends periodic reports back to the tracker. @@ -247,16 +245,6 @@ public File getAbsolutePath(String filename) { if (f.isFile() && f.canRead()) { return f; } - if (Shell.WINDOWS) { - // Try filename with executable extentions - String[] exts = new String[] {".exe", ".bat"}; - for (String ext : exts) { - File fileWithExt = new File(f.toString() + ext); - if (fileWithExt.isFile() && fileWithExt.canRead()) { - return fileWithExt; - } - } - } } catch (Exception exp) { } classvalue = classvalue.substring(val + 1).trim(); @@ -303,11 +291,6 @@ protected void initializeOp(Configuration hconf) throws HiveException { } boolean isBrokenPipeException(IOException e) { - if (Shell.WINDOWS) { - String errMsg = e.getMessage(); - return errMsg.equalsIgnoreCase(IO_EXCEPTION_PIPE_CLOSED_WIN) || - errMsg.equalsIgnoreCase(IO_EXCEPTION_PIPE_ENDED_WIN); - } return (e.getMessage().equalsIgnoreCase(IO_EXCEPTION_BROKEN_PIPE_STRING) || e.getMessage().equalsIgnoreCase(IO_EXCEPTION_STREAM_CLOSED)); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 12a03d0..e81cbce 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -867,22 +867,6 @@ public static StreamStatus readColumn(DataInput in, OutputStream out) throws IOE return StreamStatus.EOF; } - // Default new line characters on windows are "CRLF" so detect if there are any windows - // native newline characters and handle them. - if (Shell.WINDOWS) { - // if the CR is not followed by the LF on windows then add it back to the stream and - // proceed with next characters in the input stream. - if (foundCrChar && b != Utilities.newLineCode) { - out.write(Utilities.carriageReturnCode); - foundCrChar = false; - } - - if (b == Utilities.carriageReturnCode) { - foundCrChar = true; - continue; - } - } - if (b == Utilities.newLineCode) { return StreamStatus.TERMINATED; } diff --git ql/src/java/org/apache/hadoop/hive/ql/util/DosToUnix.java ql/src/java/org/apache/hadoop/hive/ql/util/DosToUnix.java deleted file mode 100644 index 6aecd49..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/util/DosToUnix.java +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.util; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStreamReader; - -public class DosToUnix { - - public static String convertWindowsScriptToUnix(File windowsScriptFile) throws IOException { - String windowsScriptFilename = windowsScriptFile.getName(); - String unixScriptFilename = getUnixScriptNameFor(windowsScriptFilename); - File unixScriptFile = null; - if (windowsScriptFile.getParent() != null) { - unixScriptFile = new File(windowsScriptFile.getParent() + "/" + unixScriptFilename); - } else { - unixScriptFile = new File(unixScriptFilename); - } - BufferedWriter writer = new BufferedWriter(new FileWriter(unixScriptFile)); - try { - BufferedReader reader = new BufferedReader(new FileReader(windowsScriptFile)); - try { - int prev = reader.read(); - int next = reader.read(); - while( prev != -1 ) { - if ( prev != -1 && ( prev != '\r' || next != '\n' ) ) { - writer.write(prev); - } - prev = next; - next = reader.read(); - } - } - finally { - reader.close(); - } - } - finally { - writer.close(); - } - unixScriptFile.setExecutable(true); - return unixScriptFile.getAbsolutePath(); - } - - public static String getUnixScriptNameFor(String windowsScriptFilename) { - int pos = windowsScriptFilename.indexOf("."); - String unixScriptFilename; - if ( pos >= 0 ) { - unixScriptFilename = windowsScriptFilename.substring(0, pos) + "_unix" + windowsScriptFilename.substring(pos); - } - else { - unixScriptFilename = windowsScriptFilename + "_unix"; - } - return unixScriptFilename; - } - - public static boolean isWindowsScript(File file) { - BufferedReader reader = null; - try { - reader = new BufferedReader(new InputStreamReader(new FileInputStream(file))); - char[] buffer = new char[4096]; - int readLength = reader.read(buffer); - if (readLength >= 2 && buffer[0] == '#' && buffer[1] == '!') { - for(int i=2; i(); diff --git ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java index 2de1b25..dafbe16 100644 --- ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java +++ ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java @@ -183,13 +183,7 @@ public void testUnion() throws URISyntaxException, IOException { * @return URI corresponding to the path. */ private static URI createURI(String path) throws URISyntaxException { - if (!Shell.WINDOWS) { - // If this is not windows shell, path better follow unix convention. - // Else, the below call will throw an URISyntaxException - return new URI(path); - } else { - return new Path(path).toUri(); - } + return new URI(path); } // Test when two jars are added with shared dependencies and one jar is deleted, the shared dependencies should not be deleted diff --git ql/src/test/org/apache/hadoop/hive/ql/util/TestDosToUnix.java ql/src/test/org/apache/hadoop/hive/ql/util/TestDosToUnix.java deleted file mode 100644 index e9a4675..0000000 --- ql/src/test/org/apache/hadoop/hive/ql/util/TestDosToUnix.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.util; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.Writer; - -import junit.framework.TestCase; - -public class TestDosToUnix extends TestCase { - - private static final String dataFile = System.getProperty("test.tmp.dir", ".") + "data_TestDosToUnix"; - @Override - protected void setUp() throws Exception { - super.setUp(); - Writer output = null; - String text = "#!/usr/bin/env ruby \r\n Test date \r\n More test data.\r\n"; - File file = new File(dataFile); - output = new BufferedWriter(new FileWriter(file)); - output.write(text); - output.close(); - } - - @Override - protected void tearDown() throws Exception { - super.tearDown(); - File f = new File(dataFile); - if(!f.delete()) { - throw new RuntimeException("Could not delete the data file"); - } - } - - public void testIsWindowsScript() { - File file = new File(dataFile); - assertEquals(true, DosToUnix.isWindowsScript(file)); - } - - public void testGetUnixScriptNameFor() { - assertEquals("test_unix", DosToUnix.getUnixScriptNameFor("test")); - assertEquals("test_unix.rb", DosToUnix.getUnixScriptNameFor("test.rb")); - } - - public void testConvertWindowsScriptToUnix() { - File file = new File(dataFile); - try { - assertEquals(true, DosToUnix.isWindowsScript(file)); - String convertedFile = DosToUnix.convertWindowsScriptToUnix(file); - File cFile = new File(convertedFile); - assertEquals(false, DosToUnix.isWindowsScript(cFile)); - if(!cFile.delete()) { - throw new RuntimeException("Could not delete the converted data file"); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - - } -} diff --git ql/src/test/queries/clientpositive/avro_timestamp.q ql/src/test/queries/clientpositive/avro_timestamp.q index 847f250..5a19414 100644 --- ql/src/test/queries/clientpositive/avro_timestamp.q +++ ql/src/test/queries/clientpositive/avro_timestamp.q @@ -1,6 +1,4 @@ set hive.mapred.mode=nonstrict; --- Exclude test on Windows due to space character being escaped in Hive paths on Windows. --- EXCLUDE_OS_WINDOWS DROP TABLE avro_timestamp_staging; DROP TABLE avro_timestamp; diff --git ql/src/test/queries/clientpositive/avro_timestamp_win.q ql/src/test/queries/clientpositive/avro_timestamp_win.q deleted file mode 100644 index 9abfd44..0000000 --- ql/src/test/queries/clientpositive/avro_timestamp_win.q +++ /dev/null @@ -1,28 +0,0 @@ --- Windows-specific test due to space character being escaped in Hive paths on Windows. --- INCLUDE_OS_WINDOWS --- JAVA_VERSION_SPECIFIC_OUTPUT - -DROP TABLE avro_timestamp_staging; -DROP TABLE avro_timestamp; -DROP TABLE avro_timestamp_casts; - -CREATE TABLE avro_timestamp_staging (d timestamp, m1 map, l1 array) - ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' - COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' - STORED AS TEXTFILE; - -LOAD DATA LOCAL INPATH '../../data/files/avro_timestamp.txt' OVERWRITE INTO TABLE avro_timestamp_staging; - -CREATE TABLE avro_timestamp (d timestamp, m1 map, l1 array) - PARTITIONED BY (p1 int, p2 timestamp) - ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' - COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' - STORED AS AVRO; - -INSERT OVERWRITE TABLE avro_timestamp PARTITION(p1=2, p2='2014-09-26 07:08:09.123') SELECT * FROM avro_timestamp_staging; - -SELECT * FROM avro_timestamp; -SELECT d, COUNT(d) FROM avro_timestamp GROUP BY d; -SELECT * FROM avro_timestamp WHERE d!='1947-02-11 07:08:09.123'; -SELECT * FROM avro_timestamp WHERE d<'2014-12-21 07:08:09.123'; -SELECT * FROM avro_timestamp WHERE d>'8000-12-01 07:08:09.123'; diff --git ql/src/test/queries/clientpositive/combine2.q ql/src/test/queries/clientpositive/combine2.q index 851b7e1..ba2b8f3 100644 --- ql/src/test/queries/clientpositive/combine2.q +++ ql/src/test/queries/clientpositive/combine2.q @@ -11,9 +11,6 @@ set hive.exec.dynamic.partition.mode=nonstrict; set mapred.cache.shared.enabled=false; set hive.merge.smallfiles.avgsize=0; --- EXCLUDE_OS_WINDOWS --- excluded on windows because of difference in file name encoding logic - -- SORT_QUERY_RESULTS create table combine2(key string) partitioned by (value string); diff --git ql/src/test/queries/clientpositive/combine2_hadoop20.q ql/src/test/queries/clientpositive/combine2_hadoop20.q index d4b0d72..8095cf6 100644 --- ql/src/test/queries/clientpositive/combine2_hadoop20.q +++ ql/src/test/queries/clientpositive/combine2_hadoop20.q @@ -10,9 +10,6 @@ set hive.exec.dynamic.partition.mode=nonstrict; set mapred.cache.shared.enabled=false; set hive.merge.smallfiles.avgsize=0; --- EXCLUDE_OS_WINDOWS --- excluded on windows because of difference in file name encoding logic - -- SORT_QUERY_RESULTS create table combine2(key string) partitioned by (value string); diff --git ql/src/test/queries/clientpositive/combine2_win.q ql/src/test/queries/clientpositive/combine2_win.q deleted file mode 100644 index c6b8827..0000000 --- ql/src/test/queries/clientpositive/combine2_win.q +++ /dev/null @@ -1,41 +0,0 @@ -set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; -set mapred.min.split.size=256; -set mapred.min.split.size.per.node=256; -set mapred.min.split.size.per.rack=256; -set mapred.max.split.size=256; -set hive.exec.dynamic.partition=true; -set hive.exec.dynamic.partition.mode=nonstrict; -set mapred.cache.shared.enabled=false; -set hive.merge.smallfiles.avgsize=0; - --- INCLUDE_OS_WINDOWS --- included only on windows because of difference in file name encoding logic - --- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20S) - -create table combine2(key string) partitioned by (value string); - -insert overwrite table combine2 partition(value) -select * from ( - select key, value from src where key < 10 - union all - select key, '|' as value from src where key = 11 - union all - select key, '2010-04-21 09:45:00' value from src where key = 19) s; - -show partitions combine2; - -explain -select key, value from combine2 where value is not null order by key; - -select key, value from combine2 where value is not null order by key; - -explain extended -select count(1) from combine2 where value is not null; - -select count(1) from combine2 where value is not null; - -explain -select ds, count(1) from srcpart where ds is not null group by ds; - -select ds, count(1) from srcpart where ds is not null group by ds; diff --git ql/src/test/queries/clientpositive/escape1.q ql/src/test/queries/clientpositive/escape1.q index a28dba8..ac499e3 100644 --- ql/src/test/queries/clientpositive/escape1.q +++ ql/src/test/queries/clientpositive/escape1.q @@ -3,9 +3,6 @@ set hive.mapred.mode=nonstrict; set hive.exec.dynamic.partition=true; set hive.exec.max.dynamic.partitions.pernode=200; --- EXCLUDE_OS_WINDOWS --- excluded on windows because of difference in file name encoding logic - DROP TABLE escape1; DROP TABLE escape_raw; diff --git ql/src/test/queries/clientpositive/escape2.q ql/src/test/queries/clientpositive/escape2.q index 9acc501..840b920 100644 --- ql/src/test/queries/clientpositive/escape2.q +++ ql/src/test/queries/clientpositive/escape2.q @@ -5,9 +5,6 @@ set hive.exec.max.dynamic.partitions.pernode=200; set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; set hive.default.fileformat=RCFILE; --- EXCLUDE_OS_WINDOWS --- excluded on windows because of difference in file name encoding logic - DROP TABLE IF EXISTS escape2; DROP TABLE IF EXISTS escape_raw; diff --git ql/src/test/queries/clientpositive/input_part10.q ql/src/test/queries/clientpositive/input_part10.q index b0698b9..e6d6c22 100644 --- ql/src/test/queries/clientpositive/input_part10.q +++ ql/src/test/queries/clientpositive/input_part10.q @@ -1,6 +1,3 @@ --- EXCLUDE_OS_WINDOWS --- excluded on windows because of difference in file name encoding logic - CREATE TABLE part_special ( a STRING, b STRING diff --git ql/src/test/queries/clientpositive/input_part10_win.q ql/src/test/queries/clientpositive/input_part10_win.q deleted file mode 100644 index 0f179ec..0000000 --- ql/src/test/queries/clientpositive/input_part10_win.q +++ /dev/null @@ -1,23 +0,0 @@ --- INCLUDE_OS_WINDOWS --- included only on windows because of difference in file name encoding logic - -CREATE TABLE part_special ( - a STRING, - b STRING -) PARTITIONED BY ( - ds STRING, - ts STRING -); - -EXPLAIN -INSERT OVERWRITE TABLE part_special PARTITION(ds='2008 04 08', ts = '10:11:12=455') -SELECT 1, 2 FROM src LIMIT 1; - -INSERT OVERWRITE TABLE part_special PARTITION(ds='2008 04 08', ts = '10:11:12=455') -SELECT 1, 2 FROM src LIMIT 1; - -DESCRIBE EXTENDED part_special PARTITION(ds='2008 04 08', ts = '10:11:12=455'); - -SELECT * FROM part_special WHERE ds='2008 04 08' AND ts = '10:11:12=455'; - - diff --git ql/src/test/queries/clientpositive/load_dyn_part14.q ql/src/test/queries/clientpositive/load_dyn_part14.q index 6c19338..eec0db8 100644 --- ql/src/test/queries/clientpositive/load_dyn_part14.q +++ ql/src/test/queries/clientpositive/load_dyn_part14.q @@ -1,6 +1,3 @@ --- EXCLUDE_OS_WINDOWS --- excluded on windows because of difference in file name encoding logic - -- SORT_QUERY_RESULTS create table if not exists nzhang_part14 (key string) diff --git ql/src/test/queries/clientpositive/load_dyn_part14_win.q ql/src/test/queries/clientpositive/load_dyn_part14_win.q deleted file mode 100755 index 8ff4772..0000000 --- ql/src/test/queries/clientpositive/load_dyn_part14_win.q +++ /dev/null @@ -1,38 +0,0 @@ --- INCLUDE_OS_WINDOWS --- included only on windows because of difference in file name encoding logic - --- SORT_QUERY_RESULTS - -create table if not exists nzhang_part14 (key string) - partitioned by (value string); - -describe extended nzhang_part14; - -set hive.exec.dynamic.partition=true; -set hive.exec.dynamic.partition.mode=nonstrict; - -explain -insert overwrite table nzhang_part14 partition(value) -select key, value from ( - select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a - union all - select * from (select 'k2' as key, '' as value from src limit 2)b - union all - select * from (select 'k3' as key, ' ' as value from src limit 2)c -) T; - -insert overwrite table nzhang_part14 partition(value) -select key, value from ( - select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a - union all - select * from (select 'k2' as key, '' as value from src limit 2)b - union all - select * from (select 'k3' as key, ' ' as value from src limit 2)c -) T; - - -show partitions nzhang_part14; - -select * from nzhang_part14 where value <> 'a'; - - diff --git ql/src/test/queries/clientpositive/partition_timestamp.q ql/src/test/queries/clientpositive/partition_timestamp.q index c2a153d..45bbba1 100644 --- ql/src/test/queries/clientpositive/partition_timestamp.q +++ ql/src/test/queries/clientpositive/partition_timestamp.q @@ -1,6 +1,4 @@ set hive.mapred.mode=nonstrict; --- Exclude test on Windows due to space character being escaped in Hive paths on Windows. --- EXCLUDE_OS_WINDOWS drop table partition_timestamp_1; create table partition_timestamp_1 (key string, value string) partitioned by (dt timestamp, region string); diff --git ql/src/test/queries/clientpositive/partition_timestamp2.q ql/src/test/queries/clientpositive/partition_timestamp2.q index 8460a9f..ba4ec17 100644 --- ql/src/test/queries/clientpositive/partition_timestamp2.q +++ ql/src/test/queries/clientpositive/partition_timestamp2.q @@ -1,7 +1,5 @@ set hive.mapred.mode=nonstrict; set hive.optimize.metadataonly=true; --- Exclude test on Windows due to space character being escaped in Hive paths on Windows. --- EXCLUDE_OS_WINDOWS drop table partition_timestamp2_1; create table partition_timestamp2_1 (key string, value string) partitioned by (dt timestamp, region int); diff --git ql/src/test/queries/clientpositive/partition_timestamp2_win.q ql/src/test/queries/clientpositive/partition_timestamp2_win.q deleted file mode 100644 index 67b8f1e..0000000 --- ql/src/test/queries/clientpositive/partition_timestamp2_win.q +++ /dev/null @@ -1,58 +0,0 @@ --- Windows-specific due to space character being escaped in Hive paths on Windows. --- INCLUDE_OS_WINDOWS -drop table partition_timestamp2_1; - -create table partition_timestamp2_1 (key string, value string) partitioned by (dt timestamp, region int); - --- test timestamp literal syntax -from (select * from src tablesample (1 rows)) x -insert overwrite table partition_timestamp2_1 partition(dt=timestamp '2000-01-01 00:00:00', region=1) select * -insert overwrite table partition_timestamp2_1 partition(dt=timestamp '2000-01-01 01:00:00', region=1) select * -insert overwrite table partition_timestamp2_1 partition(dt=timestamp '1999-01-01 00:00:00', region=2) select * -insert overwrite table partition_timestamp2_1 partition(dt=timestamp '1999-01-01 01:00:00', region=2) select *; - -select distinct dt from partition_timestamp2_1; -select * from partition_timestamp2_1; - --- insert overwrite -insert overwrite table partition_timestamp2_1 partition(dt=timestamp '2000-01-01 00:00:00', region=1) - select 'changed_key', 'changed_value' from src tablesample (2 rows); -select * from partition_timestamp2_1; - --- truncate -truncate table partition_timestamp2_1 partition(dt=timestamp '2000-01-01 00:00:00', region=1); -select distinct dt from partition_timestamp2_1; -select * from partition_timestamp2_1; - --- alter table add partition -alter table partition_timestamp2_1 add partition (dt=timestamp '1980-01-02 00:00:00', region=3); -select distinct dt from partition_timestamp2_1; -select * from partition_timestamp2_1; - --- alter table drop -alter table partition_timestamp2_1 drop partition (dt=timestamp '1999-01-01 01:00:00', region=2); -select distinct dt from partition_timestamp2_1; -select * from partition_timestamp2_1; - --- alter table set serde -alter table partition_timestamp2_1 partition(dt=timestamp '1980-01-02 00:00:00', region=3) - set serde 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe'; - --- alter table set fileformat -alter table partition_timestamp2_1 partition(dt=timestamp '1980-01-02 00:00:00', region=3) - set fileformat rcfile; -describe extended partition_timestamp2_1 partition(dt=timestamp '1980-01-02 00:00:00', region=3); - -insert overwrite table partition_timestamp2_1 partition(dt=timestamp '1980-01-02 00:00:00', region=3) - select * from src tablesample (2 rows); -select * from partition_timestamp2_1 order by key,value,dt,region; - --- alter table set location -alter table partition_timestamp2_1 partition(dt=timestamp '1980-01-02 00:00:00', region=3) - set location "file:///tmp/partition_timestamp2_1"; -describe extended partition_timestamp2_1 partition(dt=timestamp '1980-01-02 00:00:00', region=3); - --- alter table touch -alter table partition_timestamp2_1 touch partition(dt=timestamp '1980-01-02 00:00:00', region=3); - -drop table partition_timestamp2_1; diff --git ql/src/test/queries/clientpositive/partition_timestamp_win.q ql/src/test/queries/clientpositive/partition_timestamp_win.q deleted file mode 100644 index bcc87df..0000000 --- ql/src/test/queries/clientpositive/partition_timestamp_win.q +++ /dev/null @@ -1,59 +0,0 @@ --- Windows-specific test due to space character being escaped in Hive paths on Windows. --- INCLUDE_OS_WINDOWS -drop table partition_timestamp_1; - -create table partition_timestamp_1 (key string, value string) partitioned by (dt timestamp, region string); - -insert overwrite table partition_timestamp_1 partition(dt='2000-01-01 01:00:00', region= '1') - select * from src tablesample (10 rows); -insert overwrite table partition_timestamp_1 partition(dt='2000-01-01 02:00:00', region= '2') - select * from src tablesample (5 rows); -insert overwrite table partition_timestamp_1 partition(dt='2001-01-01 01:00:00', region= '2020-20-20') - select * from src tablesample (5 rows); -insert overwrite table partition_timestamp_1 partition(dt='2001-01-01 02:00:00', region= '1') - select * from src tablesample (20 rows); -insert overwrite table partition_timestamp_1 partition(dt='2001-01-01 03:00:00', region= '10') - select * from src tablesample (11 rows); - -select distinct dt from partition_timestamp_1; -select * from partition_timestamp_1 where dt = '2000-01-01 01:00:00' and region = '2' order by key,value; - --- 10 -select count(*) from partition_timestamp_1 where dt = timestamp '2000-01-01 01:00:00'; --- 10. Also try with string value in predicate -select count(*) from partition_timestamp_1 where dt = '2000-01-01 01:00:00'; --- 5 -select count(*) from partition_timestamp_1 where dt = timestamp '2000-01-01 02:00:00' and region = '2'; --- 11 -select count(*) from partition_timestamp_1 where dt = timestamp '2001-01-01 03:00:00' and region = '10'; --- 30 -select count(*) from partition_timestamp_1 where region = '1'; --- 0 -select count(*) from partition_timestamp_1 where dt = timestamp '2000-01-01 01:00:00' and region = '3'; --- 0 -select count(*) from partition_timestamp_1 where dt = timestamp '1999-01-01 01:00:00'; - --- Try other comparison operations - --- 20 -select count(*) from partition_timestamp_1 where dt > timestamp '2000-01-01 01:00:00' and region = '1'; --- 10 -select count(*) from partition_timestamp_1 where dt < timestamp '2000-01-02 01:00:00' and region = '1'; --- 20 -select count(*) from partition_timestamp_1 where dt >= timestamp '2000-01-02 01:00:00' and region = '1'; --- 10 -select count(*) from partition_timestamp_1 where dt <= timestamp '2000-01-01 01:00:00' and region = '1'; --- 20 -select count(*) from partition_timestamp_1 where dt <> timestamp '2000-01-01 01:00:00' and region = '1'; --- 10 -select count(*) from partition_timestamp_1 where dt between timestamp '1999-12-30 12:00:00' and timestamp '2000-01-03 12:00:00' and region = '1'; - - --- Try a string key with timestamp-like strings - --- 5 -select count(*) from partition_timestamp_1 where region = '2020-20-20'; --- 5 -select count(*) from partition_timestamp_1 where region > '2010-01-01'; - -drop table partition_timestamp_1; diff --git ql/src/test/queries/clientpositive/scriptfile1.q ql/src/test/queries/clientpositive/scriptfile1.q index a075177..988cdca 100644 --- ql/src/test/queries/clientpositive/scriptfile1.q +++ ql/src/test/queries/clientpositive/scriptfile1.q @@ -2,8 +2,6 @@ set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; -- SORT_QUERY_RESULTS --- EXCLUDE_OS_WINDOWS - -- NO_SESSION_REUSE CREATE TABLE dest1(key INT, value STRING); diff --git ql/src/test/queries/clientpositive/scriptfile1_win.q ql/src/test/queries/clientpositive/scriptfile1_win.q deleted file mode 100644 index fd19d84..0000000 --- ql/src/test/queries/clientpositive/scriptfile1_win.q +++ /dev/null @@ -1,16 +0,0 @@ -set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; --- INCLUDE_OS_WINDOWS - -CREATE TABLE dest1(key INT, value STRING); - -ADD FILE ../../ql/src/test/scripts/testgrep_win.bat; - -FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value) - USING 'testgrep_win.bat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -INSERT OVERWRITE TABLE dest1 SELECT tmap.tkey, tmap.tvalue; - -SELECT dest1.* FROM dest1; diff --git ql/src/test/queries/clientpositive/transform_acid.q ql/src/test/queries/clientpositive/transform_acid.q index cf8bd24..c890ef8 100644 --- ql/src/test/queries/clientpositive/transform_acid.q +++ ql/src/test/queries/clientpositive/transform_acid.q @@ -3,8 +3,6 @@ set hive.support.concurrency=true; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; --- EXCLUDE_OS_WINDOWS - create table transform_acid(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true'); insert into table transform_acid select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 1; diff --git ql/src/test/queries/clientpositive/vector_partitioned_date_time.q ql/src/test/queries/clientpositive/vector_partitioned_date_time.q index ee22c01..bf4c461 100644 --- ql/src/test/queries/clientpositive/vector_partitioned_date_time.q +++ ql/src/test/queries/clientpositive/vector_partitioned_date_time.q @@ -2,9 +2,6 @@ set hive.mapred.mode=nonstrict; set hive.explain.user=false; set hive.fetch.task.conversion=none; --- Exclude test on Windows due to space character being escaped in Hive paths on Windows. --- EXCLUDE_OS_WINDOWS - -- Check if vectorization code is handling partitioning on DATE and the other data types. diff --git ql/src/test/queries/clientpositive/vector_partitioned_date_time_win.q ql/src/test/queries/clientpositive/vector_partitioned_date_time_win.q deleted file mode 100644 index bd4931c..0000000 --- ql/src/test/queries/clientpositive/vector_partitioned_date_time_win.q +++ /dev/null @@ -1,129 +0,0 @@ -set hive.fetch.task.conversion=minimal; - --- Windows-specific test due to space character being escaped in Hive paths on Windows. --- INCLUDE_OS_WINDOWS - --- Check if vectorization code is handling partitioning on DATE and the other data types. - - -CREATE TABLE flights_tiny ( - origin_city_name STRING, - dest_city_name STRING, - fl_date DATE, - arr_delay FLOAT, - fl_num INT -); - -LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt.1' OVERWRITE INTO TABLE flights_tiny; - -CREATE TABLE flights_tiny_orc STORED AS ORC AS -SELECT origin_city_name, dest_city_name, fl_date, to_utc_timestamp(fl_date, 'America/Los_Angeles') as fl_time, arr_delay, fl_num -FROM flights_tiny; - -SELECT * FROM flights_tiny_orc; - -SET hive.vectorized.execution.enabled=false; - -select * from flights_tiny_orc sort by fl_num, fl_date limit 25; - -select fl_date, count(*) from flights_tiny_orc group by fl_date; - -SET hive.vectorized.execution.enabled=true; - -explain vectorization expression -select * from flights_tiny_orc sort by fl_num, fl_date limit 25; - -select * from flights_tiny_orc sort by fl_num, fl_date limit 25; - -explain vectorization expression -select fl_date, count(*) from flights_tiny_orc group by fl_date; - -select fl_date, count(*) from flights_tiny_orc group by fl_date; - - -SET hive.vectorized.execution.enabled=false; - -CREATE TABLE flights_tiny_orc_partitioned_date ( - origin_city_name STRING, - dest_city_name STRING, - fl_time TIMESTAMP, - arr_delay FLOAT, - fl_num INT -) -PARTITIONED BY (fl_date DATE) -STORED AS ORC; - -set hive.exec.dynamic.partition.mode=nonstrict; - -INSERT INTO TABLE flights_tiny_orc_partitioned_date -PARTITION (fl_date) -SELECT origin_city_name, dest_city_name, fl_time, arr_delay, fl_num, fl_date -FROM flights_tiny_orc; - - -select * from flights_tiny_orc_partitioned_date; - -select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25; - -select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date; - -SET hive.vectorized.execution.enabled=true; - -explain vectorization expression -select * from flights_tiny_orc_partitioned_date; - -select * from flights_tiny_orc_partitioned_date; - -explain vectorization expression -select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25; - -select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25; - -explain vectorization expression -select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date; - -select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date; - - -SET hive.vectorized.execution.enabled=false; - -CREATE TABLE flights_tiny_orc_partitioned_timestamp ( - origin_city_name STRING, - dest_city_name STRING, - fl_date DATE, - arr_delay FLOAT, - fl_num INT -) -PARTITIONED BY (fl_time TIMESTAMP) -STORED AS ORC; - -set hive.exec.dynamic.partition.mode=nonstrict; - -INSERT INTO TABLE flights_tiny_orc_partitioned_timestamp -PARTITION (fl_time) -SELECT origin_city_name, dest_city_name, fl_date, arr_delay, fl_num, fl_time -FROM flights_tiny_orc; - - -select * from flights_tiny_orc_partitioned_timestamp; - -select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25; - -select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time; - -SET hive.vectorized.execution.enabled=true; - -explain vectorization expression -select * from flights_tiny_orc_partitioned_timestamp; - -select * from flights_tiny_orc_partitioned_timestamp; - -explain vectorization expression -select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25; - -select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25; - -explain vectorization expression -select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time; - -select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time; diff --git ql/src/test/results/clientpositive/avro_timestamp_win.q.java1.7.out ql/src/test/results/clientpositive/avro_timestamp_win.q.java1.7.out deleted file mode 100755 index 6046474..0000000 --- ql/src/test/results/clientpositive/avro_timestamp_win.q.java1.7.out +++ /dev/null @@ -1,134 +0,0 @@ -PREHOOK: query: -- Windows-specific test due to space character being escaped in Hive paths on Windows. --- INCLUDE_OS_WINDOWS --- JAVA_VERSION_SPECIFIC_OUTPUT - -DROP TABLE avro_timestamp_staging -PREHOOK: type: DROPTABLE -POSTHOOK: query: -- Windows-specific test due to space character being escaped in Hive paths on Windows. --- INCLUDE_OS_WINDOWS --- JAVA_VERSION_SPECIFIC_OUTPUT - -DROP TABLE avro_timestamp_staging -POSTHOOK: type: DROPTABLE -PREHOOK: query: DROP TABLE avro_timestamp -PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE avro_timestamp -POSTHOOK: type: DROPTABLE -PREHOOK: query: DROP TABLE avro_timestamp_casts -PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE avro_timestamp_casts -POSTHOOK: type: DROPTABLE -PREHOOK: query: CREATE TABLE avro_timestamp_staging (d timestamp, m1 map, l1 array) - ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' - COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' - STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@avro_timestamp_staging -POSTHOOK: query: CREATE TABLE avro_timestamp_staging (d timestamp, m1 map, l1 array) - ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' - COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' - STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@avro_timestamp_staging -PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/avro_timestamp.txt' OVERWRITE INTO TABLE avro_timestamp_staging -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@avro_timestamp_staging -POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/avro_timestamp.txt' OVERWRITE INTO TABLE avro_timestamp_staging -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@avro_timestamp_staging -PREHOOK: query: CREATE TABLE avro_timestamp (d timestamp, m1 map, l1 array) - PARTITIONED BY (p1 int, p2 timestamp) - ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' - COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' - STORED AS AVRO -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@avro_timestamp -POSTHOOK: query: CREATE TABLE avro_timestamp (d timestamp, m1 map, l1 array) - PARTITIONED BY (p1 int, p2 timestamp) - ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' - COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' - STORED AS AVRO -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@avro_timestamp -PREHOOK: query: INSERT OVERWRITE TABLE avro_timestamp PARTITION(p1=2, p2='2014-09-26 07:08:09.123') SELECT * FROM avro_timestamp_staging -PREHOOK: type: QUERY -PREHOOK: Input: default@avro_timestamp_staging -PREHOOK: Output: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -POSTHOOK: query: INSERT OVERWRITE TABLE avro_timestamp PARTITION(p1=2, p2='2014-09-26 07:08:09.123') SELECT * FROM avro_timestamp_staging -POSTHOOK: type: QUERY -POSTHOOK: Input: default@avro_timestamp_staging -POSTHOOK: Output: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -POSTHOOK: Lineage: avro_timestamp PARTITION(p1=2,p2=2014-09-26 07:08:09.123).d SIMPLE [(avro_timestamp_staging)avro_timestamp_staging.FieldSchema(name:d, type:timestamp, comment:null), ] -POSTHOOK: Lineage: avro_timestamp PARTITION(p1=2,p2=2014-09-26 07:08:09.123).l1 SIMPLE [(avro_timestamp_staging)avro_timestamp_staging.FieldSchema(name:l1, type:array, comment:null), ] -POSTHOOK: Lineage: avro_timestamp PARTITION(p1=2,p2=2014-09-26 07:08:09.123).m1 SIMPLE [(avro_timestamp_staging)avro_timestamp_staging.FieldSchema(name:m1, type:map, comment:null), ] -PREHOOK: query: SELECT * FROM avro_timestamp -PREHOOK: type: QUERY -PREHOOK: Input: default@avro_timestamp -PREHOOK: Input: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM avro_timestamp -POSTHOOK: type: QUERY -POSTHOOK: Input: default@avro_timestamp -POSTHOOK: Input: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -#### A masked pattern was here #### -2012-02-21 07:08:09.123 {"foo":"1980-12-16 07:08:09.123","bar":"1998-05-07 07:08:09.123"} ["2011-09-04 07:08:09.123","2011-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 -2014-02-11 07:08:09.123 {"baz":"1981-12-16 07:08:09.123"} ["2011-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 -1947-02-11 07:08:09.123 {"baz":"1921-12-16 07:08:09.123"} ["2011-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 -8200-02-11 07:08:09.123 {"baz":"6981-12-16 07:08:09.123"} ["1039-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 -PREHOOK: query: SELECT d, COUNT(d) FROM avro_timestamp GROUP BY d -PREHOOK: type: QUERY -PREHOOK: Input: default@avro_timestamp -PREHOOK: Input: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -#### A masked pattern was here #### -POSTHOOK: query: SELECT d, COUNT(d) FROM avro_timestamp GROUP BY d -POSTHOOK: type: QUERY -POSTHOOK: Input: default@avro_timestamp -POSTHOOK: Input: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -#### A masked pattern was here #### -1947-02-11 07:08:09.123 1 -2012-02-21 07:08:09.123 1 -2014-02-11 07:08:09.123 1 -8200-02-11 07:08:09.123 1 -PREHOOK: query: SELECT * FROM avro_timestamp WHERE d!='1947-02-11 07:08:09.123' -PREHOOK: type: QUERY -PREHOOK: Input: default@avro_timestamp -PREHOOK: Input: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM avro_timestamp WHERE d!='1947-02-11 07:08:09.123' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@avro_timestamp -POSTHOOK: Input: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -#### A masked pattern was here #### -2012-02-21 07:08:09.123 {"foo":"1980-12-16 07:08:09.123","bar":"1998-05-07 07:08:09.123"} ["2011-09-04 07:08:09.123","2011-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 -2014-02-11 07:08:09.123 {"baz":"1981-12-16 07:08:09.123"} ["2011-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 -8200-02-11 07:08:09.123 {"baz":"6981-12-16 07:08:09.123"} ["1039-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 -PREHOOK: query: SELECT * FROM avro_timestamp WHERE d<'2014-12-21 07:08:09.123' -PREHOOK: type: QUERY -PREHOOK: Input: default@avro_timestamp -PREHOOK: Input: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM avro_timestamp WHERE d<'2014-12-21 07:08:09.123' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@avro_timestamp -POSTHOOK: Input: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -#### A masked pattern was here #### -2012-02-21 07:08:09.123 {"foo":"1980-12-16 07:08:09.123","bar":"1998-05-07 07:08:09.123"} ["2011-09-04 07:08:09.123","2011-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 -2014-02-11 07:08:09.123 {"baz":"1981-12-16 07:08:09.123"} ["2011-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 -1947-02-11 07:08:09.123 {"baz":"1921-12-16 07:08:09.123"} ["2011-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 -PREHOOK: query: SELECT * FROM avro_timestamp WHERE d>'8000-12-01 07:08:09.123' -PREHOOK: type: QUERY -PREHOOK: Input: default@avro_timestamp -PREHOOK: Input: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM avro_timestamp WHERE d>'8000-12-01 07:08:09.123' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@avro_timestamp -POSTHOOK: Input: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -#### A masked pattern was here #### -8200-02-11 07:08:09.123 {"baz":"6981-12-16 07:08:09.123"} ["1039-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 diff --git ql/src/test/results/clientpositive/avro_timestamp_win.q.java1.8.out ql/src/test/results/clientpositive/avro_timestamp_win.q.java1.8.out deleted file mode 100755 index 087d571..0000000 --- ql/src/test/results/clientpositive/avro_timestamp_win.q.java1.8.out +++ /dev/null @@ -1,134 +0,0 @@ -PREHOOK: query: -- Windows-specific test due to space character being escaped in Hive paths on Windows. --- INCLUDE_OS_WINDOWS --- JAVA_VERSION_SPECIFIC_OUTPUT - -DROP TABLE avro_timestamp_staging -PREHOOK: type: DROPTABLE -POSTHOOK: query: -- Windows-specific test due to space character being escaped in Hive paths on Windows. --- INCLUDE_OS_WINDOWS --- JAVA_VERSION_SPECIFIC_OUTPUT - -DROP TABLE avro_timestamp_staging -POSTHOOK: type: DROPTABLE -PREHOOK: query: DROP TABLE avro_timestamp -PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE avro_timestamp -POSTHOOK: type: DROPTABLE -PREHOOK: query: DROP TABLE avro_timestamp_casts -PREHOOK: type: DROPTABLE -POSTHOOK: query: DROP TABLE avro_timestamp_casts -POSTHOOK: type: DROPTABLE -PREHOOK: query: CREATE TABLE avro_timestamp_staging (d timestamp, m1 map, l1 array) - ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' - COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' - STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@avro_timestamp_staging -POSTHOOK: query: CREATE TABLE avro_timestamp_staging (d timestamp, m1 map, l1 array) - ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' - COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' - STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@avro_timestamp_staging -PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/avro_timestamp.txt' OVERWRITE INTO TABLE avro_timestamp_staging -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@avro_timestamp_staging -POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/avro_timestamp.txt' OVERWRITE INTO TABLE avro_timestamp_staging -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@avro_timestamp_staging -PREHOOK: query: CREATE TABLE avro_timestamp (d timestamp, m1 map, l1 array) - PARTITIONED BY (p1 int, p2 timestamp) - ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' - COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' - STORED AS AVRO -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@avro_timestamp -POSTHOOK: query: CREATE TABLE avro_timestamp (d timestamp, m1 map, l1 array) - PARTITIONED BY (p1 int, p2 timestamp) - ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' - COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' - STORED AS AVRO -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@avro_timestamp -PREHOOK: query: INSERT OVERWRITE TABLE avro_timestamp PARTITION(p1=2, p2='2014-09-26 07:08:09.123') SELECT * FROM avro_timestamp_staging -PREHOOK: type: QUERY -PREHOOK: Input: default@avro_timestamp_staging -PREHOOK: Output: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -POSTHOOK: query: INSERT OVERWRITE TABLE avro_timestamp PARTITION(p1=2, p2='2014-09-26 07:08:09.123') SELECT * FROM avro_timestamp_staging -POSTHOOK: type: QUERY -POSTHOOK: Input: default@avro_timestamp_staging -POSTHOOK: Output: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -POSTHOOK: Lineage: avro_timestamp PARTITION(p1=2,p2=2014-09-26 07:08:09.123).d SIMPLE [(avro_timestamp_staging)avro_timestamp_staging.FieldSchema(name:d, type:timestamp, comment:null), ] -POSTHOOK: Lineage: avro_timestamp PARTITION(p1=2,p2=2014-09-26 07:08:09.123).l1 SIMPLE [(avro_timestamp_staging)avro_timestamp_staging.FieldSchema(name:l1, type:array, comment:null), ] -POSTHOOK: Lineage: avro_timestamp PARTITION(p1=2,p2=2014-09-26 07:08:09.123).m1 SIMPLE [(avro_timestamp_staging)avro_timestamp_staging.FieldSchema(name:m1, type:map, comment:null), ] -PREHOOK: query: SELECT * FROM avro_timestamp -PREHOOK: type: QUERY -PREHOOK: Input: default@avro_timestamp -PREHOOK: Input: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM avro_timestamp -POSTHOOK: type: QUERY -POSTHOOK: Input: default@avro_timestamp -POSTHOOK: Input: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -#### A masked pattern was here #### -2012-02-21 07:08:09.123 {"bar":"1998-05-07 07:08:09.123","foo":"1980-12-16 07:08:09.123"} ["2011-09-04 07:08:09.123","2011-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 -2014-02-11 07:08:09.123 {"baz":"1981-12-16 07:08:09.123"} ["2011-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 -1947-02-11 07:08:09.123 {"baz":"1921-12-16 07:08:09.123"} ["2011-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 -8200-02-11 07:08:09.123 {"baz":"6981-12-16 07:08:09.123"} ["1039-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 -PREHOOK: query: SELECT d, COUNT(d) FROM avro_timestamp GROUP BY d -PREHOOK: type: QUERY -PREHOOK: Input: default@avro_timestamp -PREHOOK: Input: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -#### A masked pattern was here #### -POSTHOOK: query: SELECT d, COUNT(d) FROM avro_timestamp GROUP BY d -POSTHOOK: type: QUERY -POSTHOOK: Input: default@avro_timestamp -POSTHOOK: Input: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -#### A masked pattern was here #### -1947-02-11 07:08:09.123 1 -2012-02-21 07:08:09.123 1 -2014-02-11 07:08:09.123 1 -8200-02-11 07:08:09.123 1 -PREHOOK: query: SELECT * FROM avro_timestamp WHERE d!='1947-02-11 07:08:09.123' -PREHOOK: type: QUERY -PREHOOK: Input: default@avro_timestamp -PREHOOK: Input: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM avro_timestamp WHERE d!='1947-02-11 07:08:09.123' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@avro_timestamp -POSTHOOK: Input: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -#### A masked pattern was here #### -2012-02-21 07:08:09.123 {"bar":"1998-05-07 07:08:09.123","foo":"1980-12-16 07:08:09.123"} ["2011-09-04 07:08:09.123","2011-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 -2014-02-11 07:08:09.123 {"baz":"1981-12-16 07:08:09.123"} ["2011-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 -8200-02-11 07:08:09.123 {"baz":"6981-12-16 07:08:09.123"} ["1039-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 -PREHOOK: query: SELECT * FROM avro_timestamp WHERE d<'2014-12-21 07:08:09.123' -PREHOOK: type: QUERY -PREHOOK: Input: default@avro_timestamp -PREHOOK: Input: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM avro_timestamp WHERE d<'2014-12-21 07:08:09.123' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@avro_timestamp -POSTHOOK: Input: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -#### A masked pattern was here #### -2012-02-21 07:08:09.123 {"bar":"1998-05-07 07:08:09.123","foo":"1980-12-16 07:08:09.123"} ["2011-09-04 07:08:09.123","2011-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 -2014-02-11 07:08:09.123 {"baz":"1981-12-16 07:08:09.123"} ["2011-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 -1947-02-11 07:08:09.123 {"baz":"1921-12-16 07:08:09.123"} ["2011-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 -PREHOOK: query: SELECT * FROM avro_timestamp WHERE d>'8000-12-01 07:08:09.123' -PREHOOK: type: QUERY -PREHOOK: Input: default@avro_timestamp -PREHOOK: Input: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM avro_timestamp WHERE d>'8000-12-01 07:08:09.123' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@avro_timestamp -POSTHOOK: Input: default@avro_timestamp@p1=2/p2=2014-09-26%2007%3A08%3A09.123 -#### A masked pattern was here #### -8200-02-11 07:08:09.123 {"baz":"6981-12-16 07:08:09.123"} ["1039-09-05 07:08:09.123"] 2 2014-09-26 07:08:09.123 diff --git ql/src/test/results/clientpositive/combine2_win.q.out ql/src/test/results/clientpositive/combine2_win.q.out deleted file mode 100644 index b33f31b..0000000 --- ql/src/test/results/clientpositive/combine2_win.q.out +++ /dev/null @@ -1,767 +0,0 @@ -PREHOOK: query: -- INCLUDE_OS_WINDOWS --- included only on windows because of difference in file name encoding logic - --- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) - -create table combine2(key string) partitioned by (value string) -PREHOOK: type: CREATETABLE -POSTHOOK: query: -- INCLUDE_OS_WINDOWS --- included only on windows because of difference in file name encoding logic - --- INCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) - -create table combine2(key string) partitioned by (value string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: default@combine2 -PREHOOK: query: insert overwrite table combine2 partition(value) -select * from ( - select key, value from src where key < 10 - union all - select key, '|' as value from src where key = 11 - union all - select key, '2010-04-21 09:45:00' value from src where key = 19) s -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@combine2 -POSTHOOK: query: insert overwrite table combine2 partition(value) -select * from ( - select key, value from src where key < 10 - union all - select key, '|' as value from src where key = 11 - union all - select key, '2010-04-21 09:45:00' value from src where key = 19) s -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@combine2@value=%7C -POSTHOOK: Output: default@combine2@value=2010-04-21%2009%3A45%3A00 -POSTHOOK: Output: default@combine2@value=val_0 -POSTHOOK: Output: default@combine2@value=val_2 -POSTHOOK: Output: default@combine2@value=val_4 -POSTHOOK: Output: default@combine2@value=val_5 -POSTHOOK: Output: default@combine2@value=val_8 -POSTHOOK: Output: default@combine2@value=val_9 -POSTHOOK: Lineage: combine2 PARTITION(value=2010-04-21 09:45:00).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_4).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_5).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_8).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_9).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=|).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -PREHOOK: query: show partitions combine2 -PREHOOK: type: SHOWPARTITIONS -POSTHOOK: query: show partitions combine2 -POSTHOOK: type: SHOWPARTITIONS -POSTHOOK: Lineage: combine2 PARTITION(value=2010-04-21 09:45:00).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_4).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_5).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_8).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_9).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=|).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -value=%7C -value=2010-04-21%2009%3A45%3A00 -value=val_0 -value=val_2 -value=val_4 -value=val_5 -value=val_8 -value=val_9 -PREHOOK: query: explain -select key, value from combine2 where value is not null order by key -PREHOOK: type: QUERY -POSTHOOK: query: explain -select key, value from combine2 where value is not null order by key -POSTHOOK: type: QUERY -POSTHOOK: Lineage: combine2 PARTITION(value=2010-04-21 09:45:00).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_4).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_5).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_8).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_9).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=|).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME combine2))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_WHERE (TOK_FUNCTION TOK_ISNOTNULL (TOK_TABLE_OR_COL value))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key))))) - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Alias -> Map Operator Tree: - combine2 - TableScan - alias: combine2 - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: - expr: _col0 - type: string - sort order: + - tag: -1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string - Reduce Operator Tree: - Extract - File Output Operator - compressed: false - GlobalTableId: 0 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - - Stage: Stage-0 - Fetch Operator - limit: -1 - - -PREHOOK: query: select key, value from combine2 where value is not null order by key -PREHOOK: type: QUERY -PREHOOK: Input: default@combine2 -PREHOOK: Input: default@combine2@value=%7C -PREHOOK: Input: default@combine2@value=2010-04-21%2009%3A45%3A00 -PREHOOK: Input: default@combine2@value=val_0 -PREHOOK: Input: default@combine2@value=val_2 -PREHOOK: Input: default@combine2@value=val_4 -PREHOOK: Input: default@combine2@value=val_5 -PREHOOK: Input: default@combine2@value=val_8 -PREHOOK: Input: default@combine2@value=val_9 -#### A masked pattern was here #### -POSTHOOK: query: select key, value from combine2 where value is not null order by key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@combine2 -POSTHOOK: Input: default@combine2@value=%7C -POSTHOOK: Input: default@combine2@value=2010-04-21%2009%3A45%3A00 -POSTHOOK: Input: default@combine2@value=val_0 -POSTHOOK: Input: default@combine2@value=val_2 -POSTHOOK: Input: default@combine2@value=val_4 -POSTHOOK: Input: default@combine2@value=val_5 -POSTHOOK: Input: default@combine2@value=val_8 -POSTHOOK: Input: default@combine2@value=val_9 -#### A masked pattern was here #### -POSTHOOK: Lineage: combine2 PARTITION(value=2010-04-21 09:45:00).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_4).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_5).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_8).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_9).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=|).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -0 val_0 -0 val_0 -0 val_0 -11 | -19 2010-04-21 09:45:00 -2 val_2 -4 val_4 -5 val_5 -5 val_5 -5 val_5 -8 val_8 -9 val_9 -PREHOOK: query: explain extended -select count(1) from combine2 where value is not null -PREHOOK: type: QUERY -POSTHOOK: query: explain extended -select count(1) from combine2 where value is not null -POSTHOOK: type: QUERY -POSTHOOK: Lineage: combine2 PARTITION(value=2010-04-21 09:45:00).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_4).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_5).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_8).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_9).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=|).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME combine2))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION count 1))) (TOK_WHERE (TOK_FUNCTION TOK_ISNOTNULL (TOK_TABLE_OR_COL value))))) - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Alias -> Map Operator Tree: - combine2 - TableScan - alias: combine2 - GatherStats: false - Select Operator - Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - mode: hash - outputColumnNames: _col0 - Reduce Output Operator - sort order: - tag: -1 - value expressions: - expr: _col0 - type: bigint - Needs Tagging: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: value=%7C - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - value | - properties: - bucket_count -1 - columns key - columns.types string -#### A masked pattern was here #### - name default.combine2 - numFiles 1 - numRows 1 - partition_columns value - rawDataSize 2 - serialization.ddl struct combine2 { string key} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 3 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key - columns.types string -#### A masked pattern was here #### - name default.combine2 - numFiles 8 - numPartitions 8 - numRows 12 - partition_columns value - rawDataSize 14 - serialization.ddl struct combine2 { string key} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 26 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.combine2 - name: default.combine2 -#### A masked pattern was here #### - Partition - base file name: value=2010-04-21%2009%3A45%3A00 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - value 2010-04-21 09:45:00 - properties: - bucket_count -1 - columns key - columns.types string -#### A masked pattern was here #### - name default.combine2 - numFiles 1 - numRows 1 - partition_columns value - rawDataSize 2 - serialization.ddl struct combine2 { string key} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 3 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key - columns.types string -#### A masked pattern was here #### - name default.combine2 - numFiles 8 - numPartitions 8 - numRows 12 - partition_columns value - rawDataSize 14 - serialization.ddl struct combine2 { string key} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 26 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.combine2 - name: default.combine2 -#### A masked pattern was here #### - Partition - base file name: value=val_0 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - value val_0 - properties: - bucket_count -1 - columns key - columns.types string -#### A masked pattern was here #### - name default.combine2 - numFiles 1 - numRows 3 - partition_columns value - rawDataSize 3 - serialization.ddl struct combine2 { string key} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 6 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key - columns.types string -#### A masked pattern was here #### - name default.combine2 - numFiles 8 - numPartitions 8 - numRows 12 - partition_columns value - rawDataSize 14 - serialization.ddl struct combine2 { string key} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 26 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.combine2 - name: default.combine2 -#### A masked pattern was here #### - Partition - base file name: value=val_2 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - value val_2 - properties: - bucket_count -1 - columns key - columns.types string -#### A masked pattern was here #### - name default.combine2 - numFiles 1 - numRows 1 - partition_columns value - rawDataSize 1 - serialization.ddl struct combine2 { string key} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key - columns.types string -#### A masked pattern was here #### - name default.combine2 - numFiles 8 - numPartitions 8 - numRows 12 - partition_columns value - rawDataSize 14 - serialization.ddl struct combine2 { string key} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 26 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.combine2 - name: default.combine2 -#### A masked pattern was here #### - Partition - base file name: value=val_4 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - value val_4 - properties: - bucket_count -1 - columns key - columns.types string -#### A masked pattern was here #### - name default.combine2 - numFiles 1 - numRows 1 - partition_columns value - rawDataSize 1 - serialization.ddl struct combine2 { string key} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key - columns.types string -#### A masked pattern was here #### - name default.combine2 - numFiles 8 - numPartitions 8 - numRows 12 - partition_columns value - rawDataSize 14 - serialization.ddl struct combine2 { string key} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 26 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.combine2 - name: default.combine2 -#### A masked pattern was here #### - Partition - base file name: value=val_5 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - value val_5 - properties: - bucket_count -1 - columns key - columns.types string -#### A masked pattern was here #### - name default.combine2 - numFiles 1 - numRows 3 - partition_columns value - rawDataSize 3 - serialization.ddl struct combine2 { string key} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 6 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key - columns.types string -#### A masked pattern was here #### - name default.combine2 - numFiles 8 - numPartitions 8 - numRows 12 - partition_columns value - rawDataSize 14 - serialization.ddl struct combine2 { string key} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 26 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.combine2 - name: default.combine2 -#### A masked pattern was here #### - Partition - base file name: value=val_8 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - value val_8 - properties: - bucket_count -1 - columns key - columns.types string -#### A masked pattern was here #### - name default.combine2 - numFiles 1 - numRows 1 - partition_columns value - rawDataSize 1 - serialization.ddl struct combine2 { string key} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key - columns.types string -#### A masked pattern was here #### - name default.combine2 - numFiles 8 - numPartitions 8 - numRows 12 - partition_columns value - rawDataSize 14 - serialization.ddl struct combine2 { string key} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 26 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.combine2 - name: default.combine2 -#### A masked pattern was here #### - Partition - base file name: value=val_9 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - value val_9 - properties: - bucket_count -1 - columns key - columns.types string -#### A masked pattern was here #### - name default.combine2 - numFiles 1 - numRows 1 - partition_columns value - rawDataSize 1 - serialization.ddl struct combine2 { string key} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key - columns.types string -#### A masked pattern was here #### - name default.combine2 - numFiles 8 - numPartitions 8 - numRows 12 - partition_columns value - rawDataSize 14 - serialization.ddl struct combine2 { string key} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 26 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.combine2 - name: default.combine2 - Reduce Operator Tree: - Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - mode: mergepartial - outputColumnNames: _col0 - Select Operator - expressions: - expr: _col0 - type: bigint - outputColumnNames: _col0 - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0 - columns.types bigint - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.format 1 - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Truncated Path -> Alias: - /combine2/value=%7C [combine2] - /combine2/value=2010-04-21%2009%3A45%3A00 [combine2] - /combine2/value=val_0 [combine2] - /combine2/value=val_2 [combine2] - /combine2/value=val_4 [combine2] - /combine2/value=val_5 [combine2] - /combine2/value=val_8 [combine2] - /combine2/value=val_9 [combine2] - - Stage: Stage-0 - Fetch Operator - limit: -1 - - -PREHOOK: query: select count(1) from combine2 where value is not null -PREHOOK: type: QUERY -PREHOOK: Input: default@combine2 -PREHOOK: Input: default@combine2@value=%7C -PREHOOK: Input: default@combine2@value=2010-04-21%2009%3A45%3A00 -PREHOOK: Input: default@combine2@value=val_0 -PREHOOK: Input: default@combine2@value=val_2 -PREHOOK: Input: default@combine2@value=val_4 -PREHOOK: Input: default@combine2@value=val_5 -PREHOOK: Input: default@combine2@value=val_8 -PREHOOK: Input: default@combine2@value=val_9 -#### A masked pattern was here #### -POSTHOOK: query: select count(1) from combine2 where value is not null -POSTHOOK: type: QUERY -POSTHOOK: Input: default@combine2 -POSTHOOK: Input: default@combine2@value=%7C -POSTHOOK: Input: default@combine2@value=2010-04-21%2009%3A45%3A00 -POSTHOOK: Input: default@combine2@value=val_0 -POSTHOOK: Input: default@combine2@value=val_2 -POSTHOOK: Input: default@combine2@value=val_4 -POSTHOOK: Input: default@combine2@value=val_5 -POSTHOOK: Input: default@combine2@value=val_8 -POSTHOOK: Input: default@combine2@value=val_9 -#### A masked pattern was here #### -POSTHOOK: Lineage: combine2 PARTITION(value=2010-04-21 09:45:00).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_4).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_5).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_8).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_9).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=|).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -12 -PREHOOK: query: explain -select ds, count(1) from srcpart where ds is not null group by ds -PREHOOK: type: QUERY -POSTHOOK: query: explain -select ds, count(1) from srcpart where ds is not null group by ds -POSTHOOK: type: QUERY -POSTHOOK: Lineage: combine2 PARTITION(value=2010-04-21 09:45:00).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_4).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_5).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_8).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_9).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=|).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL ds)) (TOK_SELEXPR (TOK_FUNCTION count 1))) (TOK_WHERE (TOK_FUNCTION TOK_ISNOTNULL (TOK_TABLE_OR_COL ds))) (TOK_GROUPBY (TOK_TABLE_OR_COL ds)))) - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Alias -> Map Operator Tree: - srcpart - TableScan - alias: srcpart - Select Operator - expressions: - expr: ds - type: string - outputColumnNames: ds - Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: ds - type: string - mode: hash - outputColumnNames: _col0, _col1 - Reduce Output Operator - key expressions: - expr: _col0 - type: string - sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint - Reduce Operator Tree: - Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string - mode: mergepartial - outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 0 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - - Stage: Stage-0 - Fetch Operator - limit: -1 - - -PREHOOK: query: select ds, count(1) from srcpart where ds is not null group by ds -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -#### A masked pattern was here #### -POSTHOOK: query: select ds, count(1) from srcpart where ds is not null group by ds -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -#### A masked pattern was here #### -POSTHOOK: Lineage: combine2 PARTITION(value=2010-04-21 09:45:00).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_0).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_4).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_5).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_8).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=val_9).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: combine2 PARTITION(value=|).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -2008-04-08 1000 -2008-04-09 1000 diff --git ql/src/test/results/clientpositive/input_part10_win.q.out ql/src/test/results/clientpositive/input_part10_win.q.out deleted file mode 100644 index 80da57c..0000000 --- ql/src/test/results/clientpositive/input_part10_win.q.out +++ /dev/null @@ -1,131 +0,0 @@ -PREHOOK: query: -- INCLUDE_OS_WINDOWS --- included only on windows because of difference in file name encoding logic - -CREATE TABLE part_special ( - a STRING, - b STRING -) PARTITIONED BY ( - ds STRING, - ts STRING -) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@part_special -POSTHOOK: query: -- INCLUDE_OS_WINDOWS --- included only on windows because of difference in file name encoding logic - -CREATE TABLE part_special ( - a STRING, - b STRING -) PARTITIONED BY ( - ds STRING, - ts STRING -) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@part_special -PREHOOK: query: EXPLAIN -INSERT OVERWRITE TABLE part_special PARTITION(ds='2008 04 08', ts = '10:11:12=455') -SELECT 1, 2 FROM src LIMIT 1 -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -INSERT OVERWRITE TABLE part_special PARTITION(ds='2008 04 08', ts = '10:11:12=455') -SELECT 1, 2 FROM src LIMIT 1 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 1 (type: int), 2 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: int) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.part_special - - Stage: Stage-0 - Move Operator - tables: - partition: - ds 2008 04 08 - ts 10:11:12=455 - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.part_special - - Stage: Stage-2 - Stats-Aggr Operator - -PREHOOK: query: INSERT OVERWRITE TABLE part_special PARTITION(ds='2008 04 08', ts = '10:11:12=455') -SELECT 1, 2 FROM src LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@part_special@ds=2008%2004%2008/ts=10%3A11%3A12%3D455 -POSTHOOK: query: INSERT OVERWRITE TABLE part_special PARTITION(ds='2008 04 08', ts = '10:11:12=455') -SELECT 1, 2 FROM src LIMIT 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@part_special@ds=2008%2004%2008/ts=10%3A11%3A12%3D455 -POSTHOOK: Lineage: part_special PARTITION(ds=2008 04 08,ts=10:11:12=455).a SIMPLE [] -POSTHOOK: Lineage: part_special PARTITION(ds=2008 04 08,ts=10:11:12=455).b SIMPLE [] -PREHOOK: query: DESCRIBE EXTENDED part_special PARTITION(ds='2008 04 08', ts = '10:11:12=455') -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@part_special -POSTHOOK: query: DESCRIBE EXTENDED part_special PARTITION(ds='2008 04 08', ts = '10:11:12=455') -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@part_special -a string -b string -ds string -ts string - -# Partition Information -# col_name data_type comment - -ds string -ts string - -#### A masked pattern was here #### -PREHOOK: query: SELECT * FROM part_special WHERE ds='2008 04 08' AND ts = '10:11:12=455' -PREHOOK: type: QUERY -PREHOOK: Input: default@part_special -PREHOOK: Input: default@part_special@ds=2008%2004%2008/ts=10%3A11%3A12%3D455 -#### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM part_special WHERE ds='2008 04 08' AND ts = '10:11:12=455' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@part_special -POSTHOOK: Input: default@part_special@ds=2008%2004%2008/ts=10%3A11%3A12%3D455 -#### A masked pattern was here #### -1 2 2008 04 08 10:11:12=455 diff --git ql/src/test/results/clientpositive/load_dyn_part14_win.q.out ql/src/test/results/clientpositive/load_dyn_part14_win.q.out deleted file mode 100644 index d793e39..0000000 --- ql/src/test/results/clientpositive/load_dyn_part14_win.q.out +++ /dev/null @@ -1,298 +0,0 @@ -PREHOOK: query: -- INCLUDE_OS_WINDOWS --- included only on windows because of difference in file name encoding logic - --- SORT_QUERY_RESULTS - -create table if not exists nzhang_part14 (key string) - partitioned by (value string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@nzhang_part14 -POSTHOOK: query: -- INCLUDE_OS_WINDOWS --- included only on windows because of difference in file name encoding logic - --- SORT_QUERY_RESULTS - -create table if not exists nzhang_part14 (key string) - partitioned by (value string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@nzhang_part14 -PREHOOK: query: describe extended nzhang_part14 -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@nzhang_part14 -POSTHOOK: query: describe extended nzhang_part14 -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@nzhang_part14 -key string -value string - -# Partition Information -# col_name data_type comment - -value string - -#### A masked pattern was here #### -PREHOOK: query: explain -insert overwrite table nzhang_part14 partition(value) -select key, value from ( - select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a - union all - select * from (select 'k2' as key, '' as value from src limit 2)b - union all - select * from (select 'k3' as key, ' ' as value from src limit 2)c -) T -PREHOOK: type: QUERY -POSTHOOK: query: explain -insert overwrite table nzhang_part14 partition(value) -select key, value from ( - select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a - union all - select * from (select 'k2' as key, '' as value from src limit 2)b - union all - select * from (select 'k3' as key, ' ' as value from src limit 2)c -) T -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-9, Stage-10 - Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 - Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 - Stage-3 depends on stages: Stage-0 - Stage-4 - Stage-6 - Stage-7 depends on stages: Stage-6 - Stage-9 is a root stage - Stage-10 is a root stage - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Limit - Number of rows: 2 - Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Operator Tree: - Limit - Number of rows: 2 - Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - expressions: 'k1' (type: string), null (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 6 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part14 - TableScan - Union - Statistics: Num rows: 6 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part14 - TableScan - Union - Statistics: Num rows: 6 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part14 - - Stage: Stage-8 - Conditional Operator - - Stage: Stage-5 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-0 - Move Operator - tables: - partition: - value - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part14 - - Stage: Stage-3 - Stats-Aggr Operator - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part14 - - Stage: Stage-6 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part14 - - Stage: Stage-7 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-9 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Limit - Number of rows: 2 - Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Operator Tree: - Limit - Number of rows: 2 - Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - expressions: 'k2' (type: string), '' (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-10 - Map Reduce - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Limit - Number of rows: 2 - Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Operator Tree: - Limit - Number of rows: 2 - Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - expressions: 'k3' (type: string), ' ' (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - -PREHOOK: query: insert overwrite table nzhang_part14 partition(value) -select key, value from ( - select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a - union all - select * from (select 'k2' as key, '' as value from src limit 2)b - union all - select * from (select 'k3' as key, ' ' as value from src limit 2)c -) T -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@nzhang_part14 -POSTHOOK: query: insert overwrite table nzhang_part14 partition(value) -select key, value from ( - select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a - union all - select * from (select 'k2' as key, '' as value from src limit 2)b - union all - select * from (select 'k3' as key, ' ' as value from src limit 2)c -) T -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@nzhang_part14@value=%2520 -POSTHOOK: Output: default@nzhang_part14@value=__HIVE_DEFAULT_PARTITION__ -POSTHOOK: Lineage: nzhang_part14 PARTITION(value=%20).key EXPRESSION [] -POSTHOOK: Lineage: nzhang_part14 PARTITION(value=__HIVE_DEFAULT_PARTITION__).key EXPRESSION [] -PREHOOK: query: show partitions nzhang_part14 -PREHOOK: type: SHOWPARTITIONS -PREHOOK: Input: default@nzhang_part14 -POSTHOOK: query: show partitions nzhang_part14 -POSTHOOK: type: SHOWPARTITIONS -POSTHOOK: Input: default@nzhang_part14 -value=%2520 -value=__HIVE_DEFAULT_PARTITION__ -PREHOOK: query: select * from nzhang_part14 where value <> 'a' -PREHOOK: type: QUERY -PREHOOK: Input: default@nzhang_part14 -PREHOOK: Input: default@nzhang_part14@value=%2520 -PREHOOK: Input: default@nzhang_part14@value=__HIVE_DEFAULT_PARTITION__ -#### A masked pattern was here #### -POSTHOOK: query: select * from nzhang_part14 where value <> 'a' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@nzhang_part14 -POSTHOOK: Input: default@nzhang_part14@value=%2520 -POSTHOOK: Input: default@nzhang_part14@value=__HIVE_DEFAULT_PARTITION__ -#### A masked pattern was here #### -k1 __HIVE_DEFAULT_PARTITION__ -k1 __HIVE_DEFAULT_PARTITION__ -k2 __HIVE_DEFAULT_PARTITION__ -k2 __HIVE_DEFAULT_PARTITION__ -k3 %20 -k3 %20 diff --git ql/src/test/results/clientpositive/partition_timestamp2_win.q.out ql/src/test/results/clientpositive/partition_timestamp2_win.q.out deleted file mode 100755 index f39db1f..0000000 --- ql/src/test/results/clientpositive/partition_timestamp2_win.q.out +++ /dev/null @@ -1,399 +0,0 @@ -PREHOOK: query: -- Windows-specific due to space character being escaped in Hive paths on Windows. --- INCLUDE_OS_WINDOWS -drop table partition_timestamp2_1 -PREHOOK: type: DROPTABLE -POSTHOOK: query: -- Windows-specific due to space character being escaped in Hive paths on Windows. --- INCLUDE_OS_WINDOWS -drop table partition_timestamp2_1 -POSTHOOK: type: DROPTABLE -PREHOOK: query: create table partition_timestamp2_1 (key string, value string) partitioned by (dt timestamp, region int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@partition_timestamp2_1 -POSTHOOK: query: create table partition_timestamp2_1 (key string, value string) partitioned by (dt timestamp, region int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@partition_timestamp2_1 -PREHOOK: query: -- test timestamp literal syntax -from (select * from src tablesample (1 rows)) x -insert overwrite table partition_timestamp2_1 partition(dt=timestamp '2000-01-01 00:00:00', region=1) select * -insert overwrite table partition_timestamp2_1 partition(dt=timestamp '2000-01-01 01:00:00', region=1) select * -insert overwrite table partition_timestamp2_1 partition(dt=timestamp '1999-01-01 00:00:00', region=2) select * -insert overwrite table partition_timestamp2_1 partition(dt=timestamp '1999-01-01 01:00:00', region=2) select * -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -PREHOOK: Output: default@partition_timestamp2_1@dt=1999-01-01%2001%3A00%3A00.0/region=2 -PREHOOK: Output: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -PREHOOK: Output: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -POSTHOOK: query: -- test timestamp literal syntax -from (select * from src tablesample (1 rows)) x -insert overwrite table partition_timestamp2_1 partition(dt=timestamp '2000-01-01 00:00:00', region=1) select * -insert overwrite table partition_timestamp2_1 partition(dt=timestamp '2000-01-01 01:00:00', region=1) select * -insert overwrite table partition_timestamp2_1 partition(dt=timestamp '1999-01-01 00:00:00', region=2) select * -insert overwrite table partition_timestamp2_1 partition(dt=timestamp '1999-01-01 01:00:00', region=2) select * -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -POSTHOOK: Output: default@partition_timestamp2_1@dt=1999-01-01%2001%3A00%3A00.0/region=2 -POSTHOOK: Output: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -POSTHOOK: Output: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -POSTHOOK: Lineage: partition_timestamp2_1 PARTITION(dt=1999-01-01 00:00:00.0,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: partition_timestamp2_1 PARTITION(dt=1999-01-01 00:00:00.0,region=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: partition_timestamp2_1 PARTITION(dt=1999-01-01 01:00:00.0,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: partition_timestamp2_1 PARTITION(dt=1999-01-01 01:00:00.0,region=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: partition_timestamp2_1 PARTITION(dt=2000-01-01 00:00:00.0,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: partition_timestamp2_1 PARTITION(dt=2000-01-01 00:00:00.0,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: partition_timestamp2_1 PARTITION(dt=2000-01-01 01:00:00.0,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: partition_timestamp2_1 PARTITION(dt=2000-01-01 01:00:00.0,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: select distinct dt from partition_timestamp2_1 -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp2_1 -PREHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -PREHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2001%3A00%3A00.0/region=2 -PREHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -PREHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -POSTHOOK: query: select distinct dt from partition_timestamp2_1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp2_1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2001%3A00%3A00.0/region=2 -POSTHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -1999-01-01 00:00:00 -1999-01-01 01:00:00 -2000-01-01 00:00:00 -2000-01-01 01:00:00 -PREHOOK: query: select * from partition_timestamp2_1 -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp2_1 -PREHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -PREHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2001%3A00%3A00.0/region=2 -PREHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -PREHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -POSTHOOK: query: select * from partition_timestamp2_1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp2_1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2001%3A00%3A00.0/region=2 -POSTHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -238 val_238 1999-01-01 00:00:00 2 -238 val_238 1999-01-01 01:00:00 2 -238 val_238 2000-01-01 00:00:00 1 -238 val_238 2000-01-01 01:00:00 1 -PREHOOK: query: -- insert overwrite -insert overwrite table partition_timestamp2_1 partition(dt=timestamp '2000-01-01 00:00:00', region=1) - select 'changed_key', 'changed_value' from src tablesample (2 rows) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -POSTHOOK: query: -- insert overwrite -insert overwrite table partition_timestamp2_1 partition(dt=timestamp '2000-01-01 00:00:00', region=1) - select 'changed_key', 'changed_value' from src tablesample (2 rows) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -POSTHOOK: Lineage: partition_timestamp2_1 PARTITION(dt=2000-01-01 00:00:00.0,region=1).key SIMPLE [] -POSTHOOK: Lineage: partition_timestamp2_1 PARTITION(dt=2000-01-01 00:00:00.0,region=1).value SIMPLE [] -PREHOOK: query: select * from partition_timestamp2_1 -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp2_1 -PREHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -PREHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2001%3A00%3A00.0/region=2 -PREHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -PREHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -POSTHOOK: query: select * from partition_timestamp2_1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp2_1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2001%3A00%3A00.0/region=2 -POSTHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -238 val_238 1999-01-01 00:00:00 2 -238 val_238 1999-01-01 01:00:00 2 -changed_key changed_value 2000-01-01 00:00:00 1 -changed_key changed_value 2000-01-01 00:00:00 1 -238 val_238 2000-01-01 01:00:00 1 -PREHOOK: query: -- truncate -truncate table partition_timestamp2_1 partition(dt=timestamp '2000-01-01 00:00:00', region=1) -PREHOOK: type: TRUNCATETABLE -PREHOOK: Output: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -POSTHOOK: query: -- truncate -truncate table partition_timestamp2_1 partition(dt=timestamp '2000-01-01 00:00:00', region=1) -POSTHOOK: type: TRUNCATETABLE -POSTHOOK: Output: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -PREHOOK: query: select distinct dt from partition_timestamp2_1 -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp2_1 -PREHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -PREHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2001%3A00%3A00.0/region=2 -PREHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -PREHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -POSTHOOK: query: select distinct dt from partition_timestamp2_1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp2_1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2001%3A00%3A00.0/region=2 -POSTHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -1999-01-01 00:00:00 -1999-01-01 01:00:00 -2000-01-01 00:00:00 -2000-01-01 01:00:00 -PREHOOK: query: select * from partition_timestamp2_1 -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp2_1 -PREHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -PREHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2001%3A00%3A00.0/region=2 -PREHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -PREHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -POSTHOOK: query: select * from partition_timestamp2_1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp2_1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2001%3A00%3A00.0/region=2 -POSTHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -238 val_238 1999-01-01 00:00:00 2 -238 val_238 1999-01-01 01:00:00 2 -238 val_238 2000-01-01 01:00:00 1 -PREHOOK: query: -- alter table add partition -alter table partition_timestamp2_1 add partition (dt=timestamp '1980-01-02 00:00:00', region=3) -PREHOOK: type: ALTERTABLE_ADDPARTS -PREHOOK: Output: default@partition_timestamp2_1 -POSTHOOK: query: -- alter table add partition -alter table partition_timestamp2_1 add partition (dt=timestamp '1980-01-02 00:00:00', region=3) -POSTHOOK: type: ALTERTABLE_ADDPARTS -POSTHOOK: Output: default@partition_timestamp2_1 -POSTHOOK: Output: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -PREHOOK: query: select distinct dt from partition_timestamp2_1 -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp2_1 -PREHOOK: Input: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -PREHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -PREHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2001%3A00%3A00.0/region=2 -PREHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -PREHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -POSTHOOK: query: select distinct dt from partition_timestamp2_1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp2_1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2001%3A00%3A00.0/region=2 -POSTHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -1980-01-02 00:00:00 -1999-01-01 00:00:00 -1999-01-01 01:00:00 -2000-01-01 00:00:00 -2000-01-01 01:00:00 -PREHOOK: query: select * from partition_timestamp2_1 -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp2_1 -PREHOOK: Input: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -PREHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -PREHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2001%3A00%3A00.0/region=2 -PREHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -PREHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -POSTHOOK: query: select * from partition_timestamp2_1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp2_1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2001%3A00%3A00.0/region=2 -POSTHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -238 val_238 1999-01-01 00:00:00 2 -238 val_238 1999-01-01 01:00:00 2 -238 val_238 2000-01-01 01:00:00 1 -PREHOOK: query: -- alter table drop -alter table partition_timestamp2_1 drop partition (dt=timestamp '1999-01-01 01:00:00', region=2) -PREHOOK: type: ALTERTABLE_DROPPARTS -PREHOOK: Input: default@partition_timestamp2_1 -PREHOOK: Output: default@partition_timestamp2_1@dt=1999-01-01%2001%3A00%3A00.0/region=2 -POSTHOOK: query: -- alter table drop -alter table partition_timestamp2_1 drop partition (dt=timestamp '1999-01-01 01:00:00', region=2) -POSTHOOK: type: ALTERTABLE_DROPPARTS -POSTHOOK: Input: default@partition_timestamp2_1 -POSTHOOK: Output: default@partition_timestamp2_1@dt=1999-01-01%2001%3A00%3A00.0/region=2 -PREHOOK: query: select distinct dt from partition_timestamp2_1 -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp2_1 -PREHOOK: Input: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -PREHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -PREHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -PREHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -POSTHOOK: query: select distinct dt from partition_timestamp2_1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp2_1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -POSTHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -1980-01-02 00:00:00 -1999-01-01 00:00:00 -2000-01-01 00:00:00 -2000-01-01 01:00:00 -PREHOOK: query: select * from partition_timestamp2_1 -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp2_1 -PREHOOK: Input: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -PREHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -PREHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -PREHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -POSTHOOK: query: select * from partition_timestamp2_1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp2_1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -POSTHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -238 val_238 1999-01-01 00:00:00 2 -238 val_238 2000-01-01 01:00:00 1 -PREHOOK: query: -- alter table set serde -alter table partition_timestamp2_1 partition(dt=timestamp '1980-01-02 00:00:00', region=3) - set serde 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe' -PREHOOK: type: ALTERPARTITION_SERIALIZER -PREHOOK: Input: default@partition_timestamp2_1 -PREHOOK: Output: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -POSTHOOK: query: -- alter table set serde -alter table partition_timestamp2_1 partition(dt=timestamp '1980-01-02 00:00:00', region=3) - set serde 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe' -POSTHOOK: type: ALTERPARTITION_SERIALIZER -POSTHOOK: Input: default@partition_timestamp2_1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -POSTHOOK: Output: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -PREHOOK: query: -- alter table set fileformat -alter table partition_timestamp2_1 partition(dt=timestamp '1980-01-02 00:00:00', region=3) - set fileformat rcfile -PREHOOK: type: ALTERPARTITION_FILEFORMAT -PREHOOK: Input: default@partition_timestamp2_1 -PREHOOK: Output: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -POSTHOOK: query: -- alter table set fileformat -alter table partition_timestamp2_1 partition(dt=timestamp '1980-01-02 00:00:00', region=3) - set fileformat rcfile -POSTHOOK: type: ALTERPARTITION_FILEFORMAT -POSTHOOK: Input: default@partition_timestamp2_1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -POSTHOOK: Output: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -PREHOOK: query: describe extended partition_timestamp2_1 partition(dt=timestamp '1980-01-02 00:00:00', region=3) -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@partition_timestamp2_1 -POSTHOOK: query: describe extended partition_timestamp2_1 partition(dt=timestamp '1980-01-02 00:00:00', region=3) -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@partition_timestamp2_1 -key string -value string -dt timestamp -region int - -# Partition Information -# col_name data_type comment - -dt timestamp -region int - -#### A masked pattern was here #### -PREHOOK: query: insert overwrite table partition_timestamp2_1 partition(dt=timestamp '1980-01-02 00:00:00', region=3) - select * from src tablesample (2 rows) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -POSTHOOK: query: insert overwrite table partition_timestamp2_1 partition(dt=timestamp '1980-01-02 00:00:00', region=3) - select * from src tablesample (2 rows) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -POSTHOOK: Lineage: partition_timestamp2_1 PARTITION(dt=1980-01-02 00:00:00.0,region=3).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: partition_timestamp2_1 PARTITION(dt=1980-01-02 00:00:00.0,region=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: select * from partition_timestamp2_1 order by key,value,dt,region -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp2_1 -PREHOOK: Input: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -PREHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -PREHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -PREHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -POSTHOOK: query: select * from partition_timestamp2_1 order by key,value,dt,region -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp2_1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1999-01-01%2000%3A00%3A00.0/region=2 -POSTHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2000%3A00%3A00.0/region=1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -238 val_238 1980-01-02 00:00:00 3 -238 val_238 1999-01-01 00:00:00 2 -238 val_238 2000-01-01 01:00:00 1 -86 val_86 1980-01-02 00:00:00 3 -PREHOOK: query: -- alter table set location -alter table partition_timestamp2_1 partition(dt=timestamp '1980-01-02 00:00:00', region=3) -#### A masked pattern was here #### -PREHOOK: type: ALTERPARTITION_LOCATION -PREHOOK: Input: default@partition_timestamp2_1 -PREHOOK: Output: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -#### A masked pattern was here #### -POSTHOOK: query: -- alter table set location -alter table partition_timestamp2_1 partition(dt=timestamp '1980-01-02 00:00:00', region=3) -#### A masked pattern was here #### -POSTHOOK: type: ALTERPARTITION_LOCATION -POSTHOOK: Input: default@partition_timestamp2_1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -POSTHOOK: Output: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -#### A masked pattern was here #### -PREHOOK: query: describe extended partition_timestamp2_1 partition(dt=timestamp '1980-01-02 00:00:00', region=3) -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@partition_timestamp2_1 -POSTHOOK: query: describe extended partition_timestamp2_1 partition(dt=timestamp '1980-01-02 00:00:00', region=3) -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@partition_timestamp2_1 -key string -value string -dt timestamp -region int - -# Partition Information -# col_name data_type comment - -dt timestamp -region int - -#### A masked pattern was here #### -PREHOOK: query: -- alter table touch -alter table partition_timestamp2_1 touch partition(dt=timestamp '1980-01-02 00:00:00', region=3) -PREHOOK: type: ALTERTABLE_TOUCH -PREHOOK: Input: default@partition_timestamp2_1 -PREHOOK: Output: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -POSTHOOK: query: -- alter table touch -alter table partition_timestamp2_1 touch partition(dt=timestamp '1980-01-02 00:00:00', region=3) -POSTHOOK: type: ALTERTABLE_TOUCH -POSTHOOK: Input: default@partition_timestamp2_1 -POSTHOOK: Input: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -POSTHOOK: Output: default@partition_timestamp2_1@dt=1980-01-02%2000%3A00%3A00.0/region=3 -PREHOOK: query: drop table partition_timestamp2_1 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@partition_timestamp2_1 -PREHOOK: Output: default@partition_timestamp2_1 -POSTHOOK: query: drop table partition_timestamp2_1 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@partition_timestamp2_1 -POSTHOOK: Output: default@partition_timestamp2_1 diff --git ql/src/test/results/clientpositive/partition_timestamp_win.q.out ql/src/test/results/clientpositive/partition_timestamp_win.q.out deleted file mode 100755 index bbb9773..0000000 --- ql/src/test/results/clientpositive/partition_timestamp_win.q.out +++ /dev/null @@ -1,316 +0,0 @@ -PREHOOK: query: -- Windows-specific test due to space character being escaped in Hive paths on Windows. --- INCLUDE_OS_WINDOWS -drop table partition_timestamp_1 -PREHOOK: type: DROPTABLE -POSTHOOK: query: -- Windows-specific test due to space character being escaped in Hive paths on Windows. --- INCLUDE_OS_WINDOWS -drop table partition_timestamp_1 -POSTHOOK: type: DROPTABLE -PREHOOK: query: create table partition_timestamp_1 (key string, value string) partitioned by (dt timestamp, region string) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@partition_timestamp_1 -POSTHOOK: query: create table partition_timestamp_1 (key string, value string) partitioned by (dt timestamp, region string) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@partition_timestamp_1 -PREHOOK: query: insert overwrite table partition_timestamp_1 partition(dt='2000-01-01 01:00:00', region= '1') - select * from src tablesample (10 rows) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@partition_timestamp_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -POSTHOOK: query: insert overwrite table partition_timestamp_1 partition(dt='2000-01-01 01:00:00', region= '1') - select * from src tablesample (10 rows) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@partition_timestamp_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -POSTHOOK: Lineage: partition_timestamp_1 PARTITION(dt=2000-01-01 01:00:00.0,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: partition_timestamp_1 PARTITION(dt=2000-01-01 01:00:00.0,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_timestamp_1 partition(dt='2000-01-01 02:00:00', region= '2') - select * from src tablesample (5 rows) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@partition_timestamp_1@dt=2000-01-01%2002%3A00%3A00.0/region=2 -POSTHOOK: query: insert overwrite table partition_timestamp_1 partition(dt='2000-01-01 02:00:00', region= '2') - select * from src tablesample (5 rows) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@partition_timestamp_1@dt=2000-01-01%2002%3A00%3A00.0/region=2 -POSTHOOK: Lineage: partition_timestamp_1 PARTITION(dt=2000-01-01 02:00:00.0,region=2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: partition_timestamp_1 PARTITION(dt=2000-01-01 02:00:00.0,region=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_timestamp_1 partition(dt='2001-01-01 01:00:00', region= '2020-20-20') - select * from src tablesample (5 rows) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@partition_timestamp_1@dt=2001-01-01%2001%3A00%3A00.0/region=2020-20-20 -POSTHOOK: query: insert overwrite table partition_timestamp_1 partition(dt='2001-01-01 01:00:00', region= '2020-20-20') - select * from src tablesample (5 rows) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@partition_timestamp_1@dt=2001-01-01%2001%3A00%3A00.0/region=2020-20-20 -POSTHOOK: Lineage: partition_timestamp_1 PARTITION(dt=2001-01-01 01:00:00.0,region=2020-20-20).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: partition_timestamp_1 PARTITION(dt=2001-01-01 01:00:00.0,region=2020-20-20).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_timestamp_1 partition(dt='2001-01-01 02:00:00', region= '1') - select * from src tablesample (20 rows) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@partition_timestamp_1@dt=2001-01-01%2002%3A00%3A00.0/region=1 -POSTHOOK: query: insert overwrite table partition_timestamp_1 partition(dt='2001-01-01 02:00:00', region= '1') - select * from src tablesample (20 rows) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@partition_timestamp_1@dt=2001-01-01%2002%3A00%3A00.0/region=1 -POSTHOOK: Lineage: partition_timestamp_1 PARTITION(dt=2001-01-01 02:00:00.0,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: partition_timestamp_1 PARTITION(dt=2001-01-01 02:00:00.0,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table partition_timestamp_1 partition(dt='2001-01-01 03:00:00', region= '10') - select * from src tablesample (11 rows) -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@partition_timestamp_1@dt=2001-01-01%2003%3A00%3A00.0/region=10 -POSTHOOK: query: insert overwrite table partition_timestamp_1 partition(dt='2001-01-01 03:00:00', region= '10') - select * from src tablesample (11 rows) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@partition_timestamp_1@dt=2001-01-01%2003%3A00%3A00.0/region=10 -POSTHOOK: Lineage: partition_timestamp_1 PARTITION(dt=2001-01-01 03:00:00.0,region=10).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: partition_timestamp_1 PARTITION(dt=2001-01-01 03:00:00.0,region=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: select distinct dt from partition_timestamp_1 -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp_1 -PREHOOK: Input: default@partition_timestamp_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -PREHOOK: Input: default@partition_timestamp_1@dt=2000-01-01%2002%3A00%3A00.0/region=2 -PREHOOK: Input: default@partition_timestamp_1@dt=2001-01-01%2001%3A00%3A00.0/region=2020-20-20 -PREHOOK: Input: default@partition_timestamp_1@dt=2001-01-01%2002%3A00%3A00.0/region=1 -PREHOOK: Input: default@partition_timestamp_1@dt=2001-01-01%2003%3A00%3A00.0/region=10 -#### A masked pattern was here #### -POSTHOOK: query: select distinct dt from partition_timestamp_1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp_1 -POSTHOOK: Input: default@partition_timestamp_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -POSTHOOK: Input: default@partition_timestamp_1@dt=2000-01-01%2002%3A00%3A00.0/region=2 -POSTHOOK: Input: default@partition_timestamp_1@dt=2001-01-01%2001%3A00%3A00.0/region=2020-20-20 -POSTHOOK: Input: default@partition_timestamp_1@dt=2001-01-01%2002%3A00%3A00.0/region=1 -POSTHOOK: Input: default@partition_timestamp_1@dt=2001-01-01%2003%3A00%3A00.0/region=10 -#### A masked pattern was here #### -2000-01-01 01:00:00 -2000-01-01 02:00:00 -2001-01-01 01:00:00 -2001-01-01 02:00:00 -2001-01-01 03:00:00 -PREHOOK: query: select * from partition_timestamp_1 where dt = '2000-01-01 01:00:00' and region = '2' order by key,value -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp_1 -#### A masked pattern was here #### -POSTHOOK: query: select * from partition_timestamp_1 where dt = '2000-01-01 01:00:00' and region = '2' order by key,value -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp_1 -#### A masked pattern was here #### -PREHOOK: query: -- 10 -select count(*) from partition_timestamp_1 where dt = timestamp '2000-01-01 01:00:00' -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp_1 -PREHOOK: Input: default@partition_timestamp_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -POSTHOOK: query: -- 10 -select count(*) from partition_timestamp_1 where dt = timestamp '2000-01-01 01:00:00' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp_1 -POSTHOOK: Input: default@partition_timestamp_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -10 -PREHOOK: query: -- 10. Also try with string value in predicate -select count(*) from partition_timestamp_1 where dt = '2000-01-01 01:00:00' -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp_1 -PREHOOK: Input: default@partition_timestamp_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -POSTHOOK: query: -- 10. Also try with string value in predicate -select count(*) from partition_timestamp_1 where dt = '2000-01-01 01:00:00' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp_1 -POSTHOOK: Input: default@partition_timestamp_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -10 -PREHOOK: query: -- 5 -select count(*) from partition_timestamp_1 where dt = timestamp '2000-01-01 02:00:00' and region = '2' -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp_1 -PREHOOK: Input: default@partition_timestamp_1@dt=2000-01-01%2002%3A00%3A00.0/region=2 -#### A masked pattern was here #### -POSTHOOK: query: -- 5 -select count(*) from partition_timestamp_1 where dt = timestamp '2000-01-01 02:00:00' and region = '2' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp_1 -POSTHOOK: Input: default@partition_timestamp_1@dt=2000-01-01%2002%3A00%3A00.0/region=2 -#### A masked pattern was here #### -5 -PREHOOK: query: -- 11 -select count(*) from partition_timestamp_1 where dt = timestamp '2001-01-01 03:00:00' and region = '10' -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp_1 -PREHOOK: Input: default@partition_timestamp_1@dt=2001-01-01%2003%3A00%3A00.0/region=10 -#### A masked pattern was here #### -POSTHOOK: query: -- 11 -select count(*) from partition_timestamp_1 where dt = timestamp '2001-01-01 03:00:00' and region = '10' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp_1 -POSTHOOK: Input: default@partition_timestamp_1@dt=2001-01-01%2003%3A00%3A00.0/region=10 -#### A masked pattern was here #### -11 -PREHOOK: query: -- 30 -select count(*) from partition_timestamp_1 where region = '1' -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp_1 -PREHOOK: Input: default@partition_timestamp_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -PREHOOK: Input: default@partition_timestamp_1@dt=2001-01-01%2002%3A00%3A00.0/region=1 -#### A masked pattern was here #### -POSTHOOK: query: -- 30 -select count(*) from partition_timestamp_1 where region = '1' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp_1 -POSTHOOK: Input: default@partition_timestamp_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -POSTHOOK: Input: default@partition_timestamp_1@dt=2001-01-01%2002%3A00%3A00.0/region=1 -#### A masked pattern was here #### -30 -PREHOOK: query: -- 0 -select count(*) from partition_timestamp_1 where dt = timestamp '2000-01-01 01:00:00' and region = '3' -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp_1 -#### A masked pattern was here #### -POSTHOOK: query: -- 0 -select count(*) from partition_timestamp_1 where dt = timestamp '2000-01-01 01:00:00' and region = '3' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp_1 -#### A masked pattern was here #### -0 -PREHOOK: query: -- 0 -select count(*) from partition_timestamp_1 where dt = timestamp '1999-01-01 01:00:00' -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp_1 -#### A masked pattern was here #### -POSTHOOK: query: -- 0 -select count(*) from partition_timestamp_1 where dt = timestamp '1999-01-01 01:00:00' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp_1 -#### A masked pattern was here #### -0 -PREHOOK: query: -- Try other comparison operations - --- 20 -select count(*) from partition_timestamp_1 where dt > timestamp '2000-01-01 01:00:00' and region = '1' -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp_1 -PREHOOK: Input: default@partition_timestamp_1@dt=2001-01-01%2002%3A00%3A00.0/region=1 -#### A masked pattern was here #### -POSTHOOK: query: -- Try other comparison operations - --- 20 -select count(*) from partition_timestamp_1 where dt > timestamp '2000-01-01 01:00:00' and region = '1' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp_1 -POSTHOOK: Input: default@partition_timestamp_1@dt=2001-01-01%2002%3A00%3A00.0/region=1 -#### A masked pattern was here #### -20 -PREHOOK: query: -- 10 -select count(*) from partition_timestamp_1 where dt < timestamp '2000-01-02 01:00:00' and region = '1' -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp_1 -PREHOOK: Input: default@partition_timestamp_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -POSTHOOK: query: -- 10 -select count(*) from partition_timestamp_1 where dt < timestamp '2000-01-02 01:00:00' and region = '1' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp_1 -POSTHOOK: Input: default@partition_timestamp_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -10 -PREHOOK: query: -- 20 -select count(*) from partition_timestamp_1 where dt >= timestamp '2000-01-02 01:00:00' and region = '1' -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp_1 -PREHOOK: Input: default@partition_timestamp_1@dt=2001-01-01%2002%3A00%3A00.0/region=1 -#### A masked pattern was here #### -POSTHOOK: query: -- 20 -select count(*) from partition_timestamp_1 where dt >= timestamp '2000-01-02 01:00:00' and region = '1' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp_1 -POSTHOOK: Input: default@partition_timestamp_1@dt=2001-01-01%2002%3A00%3A00.0/region=1 -#### A masked pattern was here #### -20 -PREHOOK: query: -- 10 -select count(*) from partition_timestamp_1 where dt <= timestamp '2000-01-01 01:00:00' and region = '1' -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp_1 -PREHOOK: Input: default@partition_timestamp_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -POSTHOOK: query: -- 10 -select count(*) from partition_timestamp_1 where dt <= timestamp '2000-01-01 01:00:00' and region = '1' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp_1 -POSTHOOK: Input: default@partition_timestamp_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -10 -PREHOOK: query: -- 20 -select count(*) from partition_timestamp_1 where dt <> timestamp '2000-01-01 01:00:00' and region = '1' -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp_1 -PREHOOK: Input: default@partition_timestamp_1@dt=2001-01-01%2002%3A00%3A00.0/region=1 -#### A masked pattern was here #### -POSTHOOK: query: -- 20 -select count(*) from partition_timestamp_1 where dt <> timestamp '2000-01-01 01:00:00' and region = '1' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp_1 -POSTHOOK: Input: default@partition_timestamp_1@dt=2001-01-01%2002%3A00%3A00.0/region=1 -#### A masked pattern was here #### -20 -PREHOOK: query: -- 10 -select count(*) from partition_timestamp_1 where dt between timestamp '1999-12-30 12:00:00' and timestamp '2000-01-03 12:00:00' and region = '1' -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp_1 -PREHOOK: Input: default@partition_timestamp_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -POSTHOOK: query: -- 10 -select count(*) from partition_timestamp_1 where dt between timestamp '1999-12-30 12:00:00' and timestamp '2000-01-03 12:00:00' and region = '1' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp_1 -POSTHOOK: Input: default@partition_timestamp_1@dt=2000-01-01%2001%3A00%3A00.0/region=1 -#### A masked pattern was here #### -10 -PREHOOK: query: -- Try a string key with timestamp-like strings - --- 5 -select count(*) from partition_timestamp_1 where region = '2020-20-20' -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp_1 -PREHOOK: Input: default@partition_timestamp_1@dt=2001-01-01%2001%3A00%3A00.0/region=2020-20-20 -#### A masked pattern was here #### -POSTHOOK: query: -- Try a string key with timestamp-like strings - --- 5 -select count(*) from partition_timestamp_1 where region = '2020-20-20' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp_1 -POSTHOOK: Input: default@partition_timestamp_1@dt=2001-01-01%2001%3A00%3A00.0/region=2020-20-20 -#### A masked pattern was here #### -5 -PREHOOK: query: -- 5 -select count(*) from partition_timestamp_1 where region > '2010-01-01' -PREHOOK: type: QUERY -PREHOOK: Input: default@partition_timestamp_1 -PREHOOK: Input: default@partition_timestamp_1@dt=2001-01-01%2001%3A00%3A00.0/region=2020-20-20 -#### A masked pattern was here #### -POSTHOOK: query: -- 5 -select count(*) from partition_timestamp_1 where region > '2010-01-01' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@partition_timestamp_1 -POSTHOOK: Input: default@partition_timestamp_1@dt=2001-01-01%2001%3A00%3A00.0/region=2020-20-20 -#### A masked pattern was here #### -5 -PREHOOK: query: drop table partition_timestamp_1 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@partition_timestamp_1 -PREHOOK: Output: default@partition_timestamp_1 -POSTHOOK: query: drop table partition_timestamp_1 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@partition_timestamp_1 -POSTHOOK: Output: default@partition_timestamp_1 diff --git ql/src/test/results/clientpositive/scriptfile1_win.q.out ql/src/test/results/clientpositive/scriptfile1_win.q.out deleted file mode 100644 index 0c9c546..0000000 --- ql/src/test/results/clientpositive/scriptfile1_win.q.out +++ /dev/null @@ -1,55 +0,0 @@ -PREHOOK: query: -- INCLUDE_OS_WINDOWS - -CREATE TABLE dest1(key INT, value STRING) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dest1 -POSTHOOK: query: -- INCLUDE_OS_WINDOWS - -CREATE TABLE dest1(key INT, value STRING) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dest1 -PREHOOK: query: FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value) - USING 'testgrep_win.bat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -INSERT OVERWRITE TABLE dest1 SELECT tmap.tkey, tmap.tvalue -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@dest1 -POSTHOOK: query: FROM ( - FROM src - SELECT TRANSFORM(src.key, src.value) - USING 'testgrep_win.bat' AS (tkey, tvalue) - CLUSTER BY tkey -) tmap -INSERT OVERWRITE TABLE dest1 SELECT tmap.tkey, tmap.tvalue -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@dest1 -POSTHOOK: Lineage: dest1.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: dest1.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT dest1.* FROM dest1 -PREHOOK: type: QUERY -PREHOOK: Input: default@dest1 -#### A masked pattern was here #### -POSTHOOK: query: SELECT dest1.* FROM dest1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dest1 -#### A masked pattern was here #### -NULL NULL -NULL NULL -10 val_10 -100 val_100 -100 val_100 -103 val_103 -103 val_103 -104 val_104 -104 val_104 -105 val_105 -310 val_310 -NULL NULL -NULL NULL diff --git ql/src/test/results/clientpositive/vector_partitioned_date_time_win.q.out ql/src/test/results/clientpositive/vector_partitioned_date_time_win.q.out deleted file mode 100755 index 580e552..0000000 --- ql/src/test/results/clientpositive/vector_partitioned_date_time_win.q.out +++ /dev/null @@ -1,2036 +0,0 @@ -PREHOOK: query: -- Windows-specific test due to space character being escaped in Hive paths on Windows. --- INCLUDE_OS_WINDOWS - --- Check if vectorization code is handling partitioning on DATE and the other data types. - - -CREATE TABLE flights_tiny ( - origin_city_name STRING, - dest_city_name STRING, - fl_date DATE, - arr_delay FLOAT, - fl_num INT -) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@flights_tiny -POSTHOOK: query: -- Windows-specific test due to space character being escaped in Hive paths on Windows. --- INCLUDE_OS_WINDOWS - --- Check if vectorization code is handling partitioning on DATE and the other data types. - - -CREATE TABLE flights_tiny ( - origin_city_name STRING, - dest_city_name STRING, - fl_date DATE, - arr_delay FLOAT, - fl_num INT -) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@flights_tiny -PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt.1' OVERWRITE INTO TABLE flights_tiny -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@flights_tiny -POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt.1' OVERWRITE INTO TABLE flights_tiny -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@flights_tiny -PREHOOK: query: CREATE TABLE flights_tiny_orc STORED AS ORC AS -SELECT origin_city_name, dest_city_name, fl_date, to_utc_timestamp(fl_date, 'America/Los_Angeles') as fl_time, arr_delay, fl_num -FROM flights_tiny -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@flights_tiny -PREHOOK: Output: database:default -PREHOOK: Output: default@flights_tiny_orc -POSTHOOK: query: CREATE TABLE flights_tiny_orc STORED AS ORC AS -SELECT origin_city_name, dest_city_name, fl_date, to_utc_timestamp(fl_date, 'America/Los_Angeles') as fl_time, arr_delay, fl_num -FROM flights_tiny -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@flights_tiny -POSTHOOK: Output: database:default -POSTHOOK: Output: default@flights_tiny_orc -PREHOOK: query: SELECT * FROM flights_tiny_orc -PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny_orc -#### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM flights_tiny_orc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@flights_tiny_orc -#### A masked pattern was here #### -Baltimore New York 2010-10-20 2010-10-20 07:00:00 -30.0 1064 -Baltimore New York 2010-10-20 2010-10-20 07:00:00 23.0 1142 -Baltimore New York 2010-10-20 2010-10-20 07:00:00 6.0 1599 -Chicago New York 2010-10-20 2010-10-20 07:00:00 42.0 361 -Chicago New York 2010-10-20 2010-10-20 07:00:00 24.0 897 -Chicago New York 2010-10-20 2010-10-20 07:00:00 15.0 1531 -Chicago New York 2010-10-20 2010-10-20 07:00:00 -6.0 1610 -Chicago New York 2010-10-20 2010-10-20 07:00:00 -2.0 3198 -Baltimore New York 2010-10-21 2010-10-21 07:00:00 17.0 1064 -Baltimore New York 2010-10-21 2010-10-21 07:00:00 105.0 1142 -Baltimore New York 2010-10-21 2010-10-21 07:00:00 28.0 1599 -Chicago New York 2010-10-21 2010-10-21 07:00:00 142.0 361 -Chicago New York 2010-10-21 2010-10-21 07:00:00 77.0 897 -Chicago New York 2010-10-21 2010-10-21 07:00:00 53.0 1531 -Chicago New York 2010-10-21 2010-10-21 07:00:00 -5.0 1610 -Chicago New York 2010-10-21 2010-10-21 07:00:00 51.0 3198 -Baltimore New York 2010-10-22 2010-10-22 07:00:00 -12.0 1064 -Baltimore New York 2010-10-22 2010-10-22 07:00:00 54.0 1142 -Baltimore New York 2010-10-22 2010-10-22 07:00:00 18.0 1599 -Chicago New York 2010-10-22 2010-10-22 07:00:00 2.0 361 -Chicago New York 2010-10-22 2010-10-22 07:00:00 24.0 897 -Chicago New York 2010-10-22 2010-10-22 07:00:00 16.0 1531 -Chicago New York 2010-10-22 2010-10-22 07:00:00 -6.0 1610 -Chicago New York 2010-10-22 2010-10-22 07:00:00 -11.0 3198 -Baltimore New York 2010-10-23 2010-10-23 07:00:00 18.0 272 -Baltimore New York 2010-10-23 2010-10-23 07:00:00 -10.0 1805 -Baltimore New York 2010-10-23 2010-10-23 07:00:00 6.0 3171 -Chicago New York 2010-10-23 2010-10-23 07:00:00 3.0 384 -Chicago New York 2010-10-23 2010-10-23 07:00:00 32.0 426 -Chicago New York 2010-10-23 2010-10-23 07:00:00 1.0 650 -Chicago New York 2010-10-23 2010-10-23 07:00:00 11.0 3085 -Baltimore New York 2010-10-24 2010-10-24 07:00:00 12.0 1599 -Baltimore New York 2010-10-24 2010-10-24 07:00:00 20.0 2571 -Chicago New York 2010-10-24 2010-10-24 07:00:00 10.0 361 -Chicago New York 2010-10-24 2010-10-24 07:00:00 113.0 897 -Chicago New York 2010-10-24 2010-10-24 07:00:00 -5.0 1531 -Chicago New York 2010-10-24 2010-10-24 07:00:00 -17.0 1610 -Chicago New York 2010-10-24 2010-10-24 07:00:00 -3.0 3198 -Baltimore New York 2010-10-25 2010-10-25 07:00:00 -25.0 1064 -Baltimore New York 2010-10-25 2010-10-25 07:00:00 92.0 1142 -Baltimore New York 2010-10-25 2010-10-25 07:00:00 106.0 1599 -Chicago New York 2010-10-25 2010-10-25 07:00:00 31.0 361 -Chicago New York 2010-10-25 2010-10-25 07:00:00 -1.0 897 -Chicago New York 2010-10-25 2010-10-25 07:00:00 43.0 1531 -Chicago New York 2010-10-25 2010-10-25 07:00:00 6.0 1610 -Chicago New York 2010-10-25 2010-10-25 07:00:00 -16.0 3198 -Baltimore New York 2010-10-26 2010-10-26 07:00:00 -22.0 1064 -Baltimore New York 2010-10-26 2010-10-26 07:00:00 123.0 1142 -Baltimore New York 2010-10-26 2010-10-26 07:00:00 90.0 1599 -Chicago New York 2010-10-26 2010-10-26 07:00:00 12.0 361 -Chicago New York 2010-10-26 2010-10-26 07:00:00 0.0 897 -Chicago New York 2010-10-26 2010-10-26 07:00:00 29.0 1531 -Chicago New York 2010-10-26 2010-10-26 07:00:00 -17.0 1610 -Chicago New York 2010-10-26 2010-10-26 07:00:00 6.0 3198 -Baltimore New York 2010-10-27 2010-10-27 07:00:00 -18.0 1064 -Baltimore New York 2010-10-27 2010-10-27 07:00:00 49.0 1142 -Baltimore New York 2010-10-27 2010-10-27 07:00:00 92.0 1599 -Chicago New York 2010-10-27 2010-10-27 07:00:00 148.0 361 -Chicago New York 2010-10-27 2010-10-27 07:00:00 -11.0 897 -Chicago New York 2010-10-27 2010-10-27 07:00:00 70.0 1531 -Chicago New York 2010-10-27 2010-10-27 07:00:00 8.0 1610 -Chicago New York 2010-10-27 2010-10-27 07:00:00 21.0 3198 -Baltimore New York 2010-10-28 2010-10-28 07:00:00 -4.0 1064 -Baltimore New York 2010-10-28 2010-10-28 07:00:00 -14.0 1142 -Baltimore New York 2010-10-28 2010-10-28 07:00:00 -14.0 1599 -Chicago New York 2010-10-28 2010-10-28 07:00:00 2.0 361 -Chicago New York 2010-10-28 2010-10-28 07:00:00 2.0 897 -Chicago New York 2010-10-28 2010-10-28 07:00:00 -11.0 1531 -Chicago New York 2010-10-28 2010-10-28 07:00:00 3.0 1610 -Chicago New York 2010-10-28 2010-10-28 07:00:00 -18.0 3198 -Baltimore New York 2010-10-29 2010-10-29 07:00:00 -24.0 1064 -Baltimore New York 2010-10-29 2010-10-29 07:00:00 21.0 1142 -Baltimore New York 2010-10-29 2010-10-29 07:00:00 -2.0 1599 -Chicago New York 2010-10-29 2010-10-29 07:00:00 -12.0 361 -Chicago New York 2010-10-29 2010-10-29 07:00:00 -11.0 897 -Chicago New York 2010-10-29 2010-10-29 07:00:00 15.0 1531 -Chicago New York 2010-10-29 2010-10-29 07:00:00 -18.0 1610 -Chicago New York 2010-10-29 2010-10-29 07:00:00 -4.0 3198 -Baltimore New York 2010-10-30 2010-10-30 07:00:00 14.0 272 -Baltimore New York 2010-10-30 2010-10-30 07:00:00 -1.0 1805 -Baltimore New York 2010-10-30 2010-10-30 07:00:00 5.0 3171 -Chicago New York 2010-10-30 2010-10-30 07:00:00 -6.0 384 -Chicago New York 2010-10-30 2010-10-30 07:00:00 -10.0 426 -Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 650 -Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 3085 -Baltimore New York 2010-10-31 2010-10-31 07:00:00 -1.0 1599 -Baltimore New York 2010-10-31 2010-10-31 07:00:00 -14.0 2571 -Chicago New York 2010-10-31 2010-10-31 07:00:00 -25.0 361 -Chicago New York 2010-10-31 2010-10-31 07:00:00 -18.0 897 -Chicago New York 2010-10-31 2010-10-31 07:00:00 -4.0 1531 -Chicago New York 2010-10-31 2010-10-31 07:00:00 -22.0 1610 -Chicago New York 2010-10-31 2010-10-31 07:00:00 -15.0 3198 -Cleveland New York 2010-10-30 2010-10-30 07:00:00 -23.0 2018 -Cleveland New York 2010-10-30 2010-10-30 07:00:00 -12.0 2932 -Cleveland New York 2010-10-29 2010-10-29 07:00:00 -4.0 2630 -Cleveland New York 2010-10-29 2010-10-29 07:00:00 -19.0 2646 -Cleveland New York 2010-10-29 2010-10-29 07:00:00 -12.0 3014 -Cleveland New York 2010-10-28 2010-10-28 07:00:00 3.0 2630 -Cleveland New York 2010-10-28 2010-10-28 07:00:00 -6.0 2646 -Cleveland New York 2010-10-28 2010-10-28 07:00:00 1.0 3014 -Cleveland New York 2010-10-27 2010-10-27 07:00:00 16.0 2630 -Cleveland New York 2010-10-27 2010-10-27 07:00:00 27.0 3014 -Cleveland New York 2010-10-26 2010-10-26 07:00:00 4.0 2630 -Cleveland New York 2010-10-26 2010-10-26 07:00:00 -27.0 2646 -Cleveland New York 2010-10-26 2010-10-26 07:00:00 -11.0 2662 -Cleveland New York 2010-10-26 2010-10-26 07:00:00 13.0 3014 -Cleveland New York 2010-10-25 2010-10-25 07:00:00 -4.0 2630 -Cleveland New York 2010-10-25 2010-10-25 07:00:00 81.0 2646 -Cleveland New York 2010-10-25 2010-10-25 07:00:00 42.0 3014 -Cleveland New York 2010-10-24 2010-10-24 07:00:00 5.0 2254 -Cleveland New York 2010-10-24 2010-10-24 07:00:00 -11.0 2630 -Cleveland New York 2010-10-24 2010-10-24 07:00:00 -20.0 2646 -Cleveland New York 2010-10-24 2010-10-24 07:00:00 -9.0 3014 -Cleveland New York 2010-10-23 2010-10-23 07:00:00 -21.0 2932 -Cleveland New York 2010-10-22 2010-10-22 07:00:00 1.0 2630 -Cleveland New York 2010-10-22 2010-10-22 07:00:00 -25.0 2646 -Cleveland New York 2010-10-22 2010-10-22 07:00:00 -3.0 3014 -Cleveland New York 2010-10-21 2010-10-21 07:00:00 3.0 2630 -Cleveland New York 2010-10-21 2010-10-21 07:00:00 29.0 2646 -Cleveland New York 2010-10-21 2010-10-21 07:00:00 72.0 3014 -Cleveland New York 2010-10-20 2010-10-20 07:00:00 -8.0 2630 -Cleveland New York 2010-10-20 2010-10-20 07:00:00 -15.0 3014 -Washington New York 2010-10-23 2010-10-23 07:00:00 -25.0 5832 -Washington New York 2010-10-23 2010-10-23 07:00:00 -21.0 5904 -Washington New York 2010-10-23 2010-10-23 07:00:00 -18.0 5917 -Washington New York 2010-10-30 2010-10-30 07:00:00 -27.0 5904 -Washington New York 2010-10-30 2010-10-30 07:00:00 -16.0 5917 -Washington New York 2010-10-20 2010-10-20 07:00:00 -2.0 7291 -Washington New York 2010-10-21 2010-10-21 07:00:00 22.0 7291 -Washington New York 2010-10-23 2010-10-23 07:00:00 -16.0 7274 -Washington New York 2010-10-24 2010-10-24 07:00:00 -26.0 7282 -Washington New York 2010-10-25 2010-10-25 07:00:00 9.0 7291 -Washington New York 2010-10-26 2010-10-26 07:00:00 4.0 7291 -Washington New York 2010-10-27 2010-10-27 07:00:00 26.0 7291 -Washington New York 2010-10-28 2010-10-28 07:00:00 45.0 7291 -Washington New York 2010-10-29 2010-10-29 07:00:00 1.0 7291 -Washington New York 2010-10-31 2010-10-31 07:00:00 -18.0 7282 -PREHOOK: query: select * from flights_tiny_orc sort by fl_num, fl_date limit 25 -PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny_orc -#### A masked pattern was here #### -POSTHOOK: query: select * from flights_tiny_orc sort by fl_num, fl_date limit 25 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@flights_tiny_orc -#### A masked pattern was here #### -Baltimore New York 2010-10-23 2010-10-23 07:00:00 18.0 272 -Baltimore New York 2010-10-30 2010-10-30 07:00:00 14.0 272 -Chicago New York 2010-10-20 2010-10-20 07:00:00 42.0 361 -Chicago New York 2010-10-21 2010-10-21 07:00:00 142.0 361 -Chicago New York 2010-10-22 2010-10-22 07:00:00 2.0 361 -Chicago New York 2010-10-24 2010-10-24 07:00:00 10.0 361 -Chicago New York 2010-10-25 2010-10-25 07:00:00 31.0 361 -Chicago New York 2010-10-26 2010-10-26 07:00:00 12.0 361 -Chicago New York 2010-10-27 2010-10-27 07:00:00 148.0 361 -Chicago New York 2010-10-28 2010-10-28 07:00:00 2.0 361 -Chicago New York 2010-10-29 2010-10-29 07:00:00 -12.0 361 -Chicago New York 2010-10-31 2010-10-31 07:00:00 -25.0 361 -Chicago New York 2010-10-23 2010-10-23 07:00:00 3.0 384 -Chicago New York 2010-10-30 2010-10-30 07:00:00 -6.0 384 -Chicago New York 2010-10-23 2010-10-23 07:00:00 32.0 426 -Chicago New York 2010-10-30 2010-10-30 07:00:00 -10.0 426 -Chicago New York 2010-10-23 2010-10-23 07:00:00 1.0 650 -Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 650 -Chicago New York 2010-10-20 2010-10-20 07:00:00 24.0 897 -Chicago New York 2010-10-21 2010-10-21 07:00:00 77.0 897 -Chicago New York 2010-10-22 2010-10-22 07:00:00 24.0 897 -Chicago New York 2010-10-24 2010-10-24 07:00:00 113.0 897 -Chicago New York 2010-10-25 2010-10-25 07:00:00 -1.0 897 -Chicago New York 2010-10-26 2010-10-26 07:00:00 0.0 897 -Chicago New York 2010-10-27 2010-10-27 07:00:00 -11.0 897 -PREHOOK: query: select fl_date, count(*) from flights_tiny_orc group by fl_date -PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny_orc -#### A masked pattern was here #### -POSTHOOK: query: select fl_date, count(*) from flights_tiny_orc group by fl_date -POSTHOOK: type: QUERY -POSTHOOK: Input: default@flights_tiny_orc -#### A masked pattern was here #### -2010-10-20 11 -2010-10-21 12 -2010-10-22 11 -2010-10-23 12 -2010-10-24 12 -2010-10-25 12 -2010-10-26 13 -2010-10-27 11 -2010-10-28 12 -2010-10-29 12 -2010-10-30 11 -2010-10-31 8 -PREHOOK: query: explain -select * from flights_tiny_orc sort by fl_num, fl_date limit 25 -PREHOOK: type: QUERY -POSTHOOK: query: explain -select * from flights_tiny_orc sort by fl_num, fl_date limit 25 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: flights_tiny_orc - Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: origin_city_name (type: string), dest_city_name (type: string), fl_date (type: date), fl_time (type: timestamp), arr_delay (type: float), fl_num (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col5 (type: int), _col2 (type: date) - sort order: ++ - Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: timestamp), _col4 (type: float) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey1 (type: date), VALUE._col2 (type: timestamp), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 25 - Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col5 (type: int), _col2 (type: date) - sort order: ++ - Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: timestamp), _col4 (type: float) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey1 (type: date), VALUE._col2 (type: timestamp), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 25 - Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: 25 - Processor Tree: - ListSink - -PREHOOK: query: select * from flights_tiny_orc sort by fl_num, fl_date limit 25 -PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny_orc -#### A masked pattern was here #### -POSTHOOK: query: select * from flights_tiny_orc sort by fl_num, fl_date limit 25 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@flights_tiny_orc -#### A masked pattern was here #### -Baltimore New York 2010-10-23 2010-10-23 07:00:00 18.0 272 -Baltimore New York 2010-10-30 2010-10-30 07:00:00 14.0 272 -Chicago New York 2010-10-20 2010-10-20 07:00:00 42.0 361 -Chicago New York 2010-10-21 2010-10-21 07:00:00 142.0 361 -Chicago New York 2010-10-22 2010-10-22 07:00:00 2.0 361 -Chicago New York 2010-10-24 2010-10-24 07:00:00 10.0 361 -Chicago New York 2010-10-25 2010-10-25 07:00:00 31.0 361 -Chicago New York 2010-10-26 2010-10-26 07:00:00 12.0 361 -Chicago New York 2010-10-27 2010-10-27 07:00:00 148.0 361 -Chicago New York 2010-10-28 2010-10-28 07:00:00 2.0 361 -Chicago New York 2010-10-29 2010-10-29 07:00:00 -12.0 361 -Chicago New York 2010-10-31 2010-10-31 07:00:00 -25.0 361 -Chicago New York 2010-10-23 2010-10-23 07:00:00 3.0 384 -Chicago New York 2010-10-30 2010-10-30 07:00:00 -6.0 384 -Chicago New York 2010-10-23 2010-10-23 07:00:00 32.0 426 -Chicago New York 2010-10-30 2010-10-30 07:00:00 -10.0 426 -Chicago New York 2010-10-23 2010-10-23 07:00:00 1.0 650 -Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 650 -Chicago New York 2010-10-20 2010-10-20 07:00:00 24.0 897 -Chicago New York 2010-10-21 2010-10-21 07:00:00 77.0 897 -Chicago New York 2010-10-22 2010-10-22 07:00:00 24.0 897 -Chicago New York 2010-10-24 2010-10-24 07:00:00 113.0 897 -Chicago New York 2010-10-25 2010-10-25 07:00:00 -1.0 897 -Chicago New York 2010-10-26 2010-10-26 07:00:00 0.0 897 -Chicago New York 2010-10-27 2010-10-27 07:00:00 -11.0 897 -PREHOOK: query: explain -select fl_date, count(*) from flights_tiny_orc group by fl_date -PREHOOK: type: QUERY -POSTHOOK: query: explain -select fl_date, count(*) from flights_tiny_orc group by fl_date -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: flights_tiny_orc - Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: fl_date (type: date) - outputColumnNames: _col0 - Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: _col0 (type: date) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: date) - sort order: + - Map-reduce partition columns: _col0 (type: date) - Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: date) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 68 Data size: 19584 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 68 Data size: 19584 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select fl_date, count(*) from flights_tiny_orc group by fl_date -PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny_orc -#### A masked pattern was here #### -POSTHOOK: query: select fl_date, count(*) from flights_tiny_orc group by fl_date -POSTHOOK: type: QUERY -POSTHOOK: Input: default@flights_tiny_orc -#### A masked pattern was here #### -2010-10-20 11 -2010-10-21 12 -2010-10-22 11 -2010-10-23 12 -2010-10-24 12 -2010-10-25 12 -2010-10-26 13 -2010-10-27 11 -2010-10-28 12 -2010-10-29 12 -2010-10-30 11 -2010-10-31 8 -PREHOOK: query: CREATE TABLE flights_tiny_orc_partitioned_date ( - origin_city_name STRING, - dest_city_name STRING, - fl_time TIMESTAMP, - arr_delay FLOAT, - fl_num INT -) -PARTITIONED BY (fl_date DATE) -STORED AS ORC -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@flights_tiny_orc_partitioned_date -POSTHOOK: query: CREATE TABLE flights_tiny_orc_partitioned_date ( - origin_city_name STRING, - dest_city_name STRING, - fl_time TIMESTAMP, - arr_delay FLOAT, - fl_num INT -) -PARTITIONED BY (fl_date DATE) -STORED AS ORC -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@flights_tiny_orc_partitioned_date -PREHOOK: query: INSERT INTO TABLE flights_tiny_orc_partitioned_date -PARTITION (fl_date) -SELECT origin_city_name, dest_city_name, fl_time, arr_delay, fl_num, fl_date -FROM flights_tiny_orc -PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny_orc -PREHOOK: Output: default@flights_tiny_orc_partitioned_date -POSTHOOK: query: INSERT INTO TABLE flights_tiny_orc_partitioned_date -PARTITION (fl_date) -SELECT origin_city_name, dest_city_name, fl_time, arr_delay, fl_num, fl_date -FROM flights_tiny_orc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@flights_tiny_orc -POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-20).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-20).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-20).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-20).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-20).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-21).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-21).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-21).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-21).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-21).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-22).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-22).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-22).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-22).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-22).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-23).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-23).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-23).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-23).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-23).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-24).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-24).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-24).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-24).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-24).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-25).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-25).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-25).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-25).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-25).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-26).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-26).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-26).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-26).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-26).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-27).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-27).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-27).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-27).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-27).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-28).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-28).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-28).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-28).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-28).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-29).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-29).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-29).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-29).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-29).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-30).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-30).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-30).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-30).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-30).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-31).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-31).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-31).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-31).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-31).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -PREHOOK: query: select * from flights_tiny_orc_partitioned_date -PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny_orc_partitioned_date -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 -#### A masked pattern was here #### -POSTHOOK: query: select * from flights_tiny_orc_partitioned_date -POSTHOOK: type: QUERY -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 -#### A masked pattern was here #### -Baltimore New York 2010-10-20 07:00:00 -30.0 1064 2010-10-20 -Baltimore New York 2010-10-20 07:00:00 23.0 1142 2010-10-20 -Baltimore New York 2010-10-20 07:00:00 6.0 1599 2010-10-20 -Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 -Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 -Chicago New York 2010-10-20 07:00:00 15.0 1531 2010-10-20 -Chicago New York 2010-10-20 07:00:00 -6.0 1610 2010-10-20 -Chicago New York 2010-10-20 07:00:00 -2.0 3198 2010-10-20 -Cleveland New York 2010-10-20 07:00:00 -8.0 2630 2010-10-20 -Cleveland New York 2010-10-20 07:00:00 -15.0 3014 2010-10-20 -Washington New York 2010-10-20 07:00:00 -2.0 7291 2010-10-20 -Baltimore New York 2010-10-21 07:00:00 17.0 1064 2010-10-21 -Baltimore New York 2010-10-21 07:00:00 105.0 1142 2010-10-21 -Baltimore New York 2010-10-21 07:00:00 28.0 1599 2010-10-21 -Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 -Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 -Chicago New York 2010-10-21 07:00:00 53.0 1531 2010-10-21 -Chicago New York 2010-10-21 07:00:00 -5.0 1610 2010-10-21 -Chicago New York 2010-10-21 07:00:00 51.0 3198 2010-10-21 -Cleveland New York 2010-10-21 07:00:00 3.0 2630 2010-10-21 -Cleveland New York 2010-10-21 07:00:00 29.0 2646 2010-10-21 -Cleveland New York 2010-10-21 07:00:00 72.0 3014 2010-10-21 -Washington New York 2010-10-21 07:00:00 22.0 7291 2010-10-21 -Baltimore New York 2010-10-22 07:00:00 -12.0 1064 2010-10-22 -Baltimore New York 2010-10-22 07:00:00 54.0 1142 2010-10-22 -Baltimore New York 2010-10-22 07:00:00 18.0 1599 2010-10-22 -Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 -Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 -Chicago New York 2010-10-22 07:00:00 16.0 1531 2010-10-22 -Chicago New York 2010-10-22 07:00:00 -6.0 1610 2010-10-22 -Chicago New York 2010-10-22 07:00:00 -11.0 3198 2010-10-22 -Cleveland New York 2010-10-22 07:00:00 1.0 2630 2010-10-22 -Cleveland New York 2010-10-22 07:00:00 -25.0 2646 2010-10-22 -Cleveland New York 2010-10-22 07:00:00 -3.0 3014 2010-10-22 -Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 -Baltimore New York 2010-10-23 07:00:00 -10.0 1805 2010-10-23 -Baltimore New York 2010-10-23 07:00:00 6.0 3171 2010-10-23 -Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 -Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 -Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 -Chicago New York 2010-10-23 07:00:00 11.0 3085 2010-10-23 -Cleveland New York 2010-10-23 07:00:00 -21.0 2932 2010-10-23 -Washington New York 2010-10-23 07:00:00 -25.0 5832 2010-10-23 -Washington New York 2010-10-23 07:00:00 -21.0 5904 2010-10-23 -Washington New York 2010-10-23 07:00:00 -18.0 5917 2010-10-23 -Washington New York 2010-10-23 07:00:00 -16.0 7274 2010-10-23 -Baltimore New York 2010-10-24 07:00:00 12.0 1599 2010-10-24 -Baltimore New York 2010-10-24 07:00:00 20.0 2571 2010-10-24 -Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 -Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 -Chicago New York 2010-10-24 07:00:00 -5.0 1531 2010-10-24 -Chicago New York 2010-10-24 07:00:00 -17.0 1610 2010-10-24 -Chicago New York 2010-10-24 07:00:00 -3.0 3198 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 5.0 2254 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 -11.0 2630 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 -20.0 2646 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 -9.0 3014 2010-10-24 -Washington New York 2010-10-24 07:00:00 -26.0 7282 2010-10-24 -Baltimore New York 2010-10-25 07:00:00 -25.0 1064 2010-10-25 -Baltimore New York 2010-10-25 07:00:00 92.0 1142 2010-10-25 -Baltimore New York 2010-10-25 07:00:00 106.0 1599 2010-10-25 -Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 -Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 -Chicago New York 2010-10-25 07:00:00 43.0 1531 2010-10-25 -Chicago New York 2010-10-25 07:00:00 6.0 1610 2010-10-25 -Chicago New York 2010-10-25 07:00:00 -16.0 3198 2010-10-25 -Cleveland New York 2010-10-25 07:00:00 -4.0 2630 2010-10-25 -Cleveland New York 2010-10-25 07:00:00 81.0 2646 2010-10-25 -Cleveland New York 2010-10-25 07:00:00 42.0 3014 2010-10-25 -Washington New York 2010-10-25 07:00:00 9.0 7291 2010-10-25 -Baltimore New York 2010-10-26 07:00:00 -22.0 1064 2010-10-26 -Baltimore New York 2010-10-26 07:00:00 123.0 1142 2010-10-26 -Baltimore New York 2010-10-26 07:00:00 90.0 1599 2010-10-26 -Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 -Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 -Chicago New York 2010-10-26 07:00:00 29.0 1531 2010-10-26 -Chicago New York 2010-10-26 07:00:00 -17.0 1610 2010-10-26 -Chicago New York 2010-10-26 07:00:00 6.0 3198 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 4.0 2630 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 -27.0 2646 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 -11.0 2662 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 13.0 3014 2010-10-26 -Washington New York 2010-10-26 07:00:00 4.0 7291 2010-10-26 -Baltimore New York 2010-10-27 07:00:00 -18.0 1064 2010-10-27 -Baltimore New York 2010-10-27 07:00:00 49.0 1142 2010-10-27 -Baltimore New York 2010-10-27 07:00:00 92.0 1599 2010-10-27 -Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 -Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 -Chicago New York 2010-10-27 07:00:00 70.0 1531 2010-10-27 -Chicago New York 2010-10-27 07:00:00 8.0 1610 2010-10-27 -Chicago New York 2010-10-27 07:00:00 21.0 3198 2010-10-27 -Cleveland New York 2010-10-27 07:00:00 16.0 2630 2010-10-27 -Cleveland New York 2010-10-27 07:00:00 27.0 3014 2010-10-27 -Washington New York 2010-10-27 07:00:00 26.0 7291 2010-10-27 -Baltimore New York 2010-10-28 07:00:00 -4.0 1064 2010-10-28 -Baltimore New York 2010-10-28 07:00:00 -14.0 1142 2010-10-28 -Baltimore New York 2010-10-28 07:00:00 -14.0 1599 2010-10-28 -Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 -Chicago New York 2010-10-28 07:00:00 2.0 897 2010-10-28 -Chicago New York 2010-10-28 07:00:00 -11.0 1531 2010-10-28 -Chicago New York 2010-10-28 07:00:00 3.0 1610 2010-10-28 -Chicago New York 2010-10-28 07:00:00 -18.0 3198 2010-10-28 -Cleveland New York 2010-10-28 07:00:00 3.0 2630 2010-10-28 -Cleveland New York 2010-10-28 07:00:00 -6.0 2646 2010-10-28 -Cleveland New York 2010-10-28 07:00:00 1.0 3014 2010-10-28 -Washington New York 2010-10-28 07:00:00 45.0 7291 2010-10-28 -Baltimore New York 2010-10-29 07:00:00 -24.0 1064 2010-10-29 -Baltimore New York 2010-10-29 07:00:00 21.0 1142 2010-10-29 -Baltimore New York 2010-10-29 07:00:00 -2.0 1599 2010-10-29 -Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 -Chicago New York 2010-10-29 07:00:00 -11.0 897 2010-10-29 -Chicago New York 2010-10-29 07:00:00 15.0 1531 2010-10-29 -Chicago New York 2010-10-29 07:00:00 -18.0 1610 2010-10-29 -Chicago New York 2010-10-29 07:00:00 -4.0 3198 2010-10-29 -Cleveland New York 2010-10-29 07:00:00 -4.0 2630 2010-10-29 -Cleveland New York 2010-10-29 07:00:00 -19.0 2646 2010-10-29 -Cleveland New York 2010-10-29 07:00:00 -12.0 3014 2010-10-29 -Washington New York 2010-10-29 07:00:00 1.0 7291 2010-10-29 -Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 -Baltimore New York 2010-10-30 07:00:00 -1.0 1805 2010-10-30 -Baltimore New York 2010-10-30 07:00:00 5.0 3171 2010-10-30 -Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 -Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 -Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 -Chicago New York 2010-10-30 07:00:00 -5.0 3085 2010-10-30 -Cleveland New York 2010-10-30 07:00:00 -23.0 2018 2010-10-30 -Cleveland New York 2010-10-30 07:00:00 -12.0 2932 2010-10-30 -Washington New York 2010-10-30 07:00:00 -27.0 5904 2010-10-30 -Washington New York 2010-10-30 07:00:00 -16.0 5917 2010-10-30 -Baltimore New York 2010-10-31 07:00:00 -1.0 1599 2010-10-31 -Baltimore New York 2010-10-31 07:00:00 -14.0 2571 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -18.0 897 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -4.0 1531 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -22.0 1610 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -15.0 3198 2010-10-31 -Washington New York 2010-10-31 07:00:00 -18.0 7282 2010-10-31 -PREHOOK: query: select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25 -PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny_orc_partitioned_date -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 -#### A masked pattern was here #### -POSTHOOK: query: select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 -#### A masked pattern was here #### -Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 -Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 -Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 -Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 -Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 -Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 -Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 -Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 -Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 -Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 -Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 -Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 -Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 -Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 -Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 -Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 -Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 -Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 -Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 -Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 -Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 -Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 -Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 -Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 -Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 -PREHOOK: query: select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date -PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny_orc_partitioned_date -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 -#### A masked pattern was here #### -POSTHOOK: query: select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date -POSTHOOK: type: QUERY -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 -#### A masked pattern was here #### -2010-10-20 11 -2010-10-21 12 -2010-10-22 11 -2010-10-23 12 -2010-10-24 12 -2010-10-25 12 -2010-10-26 13 -2010-10-27 11 -2010-10-28 12 -2010-10-29 12 -2010-10-30 11 -2010-10-31 8 -PREHOOK: query: explain -select * from flights_tiny_orc_partitioned_date -PREHOOK: type: QUERY -POSTHOOK: query: explain -select * from flights_tiny_orc_partitioned_date -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: flights_tiny_orc_partitioned_date - Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: origin_city_name (type: string), dest_city_name (type: string), fl_time (type: timestamp), arr_delay (type: float), fl_num (type: int), fl_date (type: date) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE - ListSink - -PREHOOK: query: select * from flights_tiny_orc_partitioned_date -PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny_orc_partitioned_date -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 -#### A masked pattern was here #### -POSTHOOK: query: select * from flights_tiny_orc_partitioned_date -POSTHOOK: type: QUERY -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 -#### A masked pattern was here #### -Baltimore New York 2010-10-20 07:00:00 -30.0 1064 2010-10-20 -Baltimore New York 2010-10-20 07:00:00 23.0 1142 2010-10-20 -Baltimore New York 2010-10-20 07:00:00 6.0 1599 2010-10-20 -Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 -Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 -Chicago New York 2010-10-20 07:00:00 15.0 1531 2010-10-20 -Chicago New York 2010-10-20 07:00:00 -6.0 1610 2010-10-20 -Chicago New York 2010-10-20 07:00:00 -2.0 3198 2010-10-20 -Cleveland New York 2010-10-20 07:00:00 -8.0 2630 2010-10-20 -Cleveland New York 2010-10-20 07:00:00 -15.0 3014 2010-10-20 -Washington New York 2010-10-20 07:00:00 -2.0 7291 2010-10-20 -Baltimore New York 2010-10-21 07:00:00 17.0 1064 2010-10-21 -Baltimore New York 2010-10-21 07:00:00 105.0 1142 2010-10-21 -Baltimore New York 2010-10-21 07:00:00 28.0 1599 2010-10-21 -Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 -Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 -Chicago New York 2010-10-21 07:00:00 53.0 1531 2010-10-21 -Chicago New York 2010-10-21 07:00:00 -5.0 1610 2010-10-21 -Chicago New York 2010-10-21 07:00:00 51.0 3198 2010-10-21 -Cleveland New York 2010-10-21 07:00:00 3.0 2630 2010-10-21 -Cleveland New York 2010-10-21 07:00:00 29.0 2646 2010-10-21 -Cleveland New York 2010-10-21 07:00:00 72.0 3014 2010-10-21 -Washington New York 2010-10-21 07:00:00 22.0 7291 2010-10-21 -Baltimore New York 2010-10-22 07:00:00 -12.0 1064 2010-10-22 -Baltimore New York 2010-10-22 07:00:00 54.0 1142 2010-10-22 -Baltimore New York 2010-10-22 07:00:00 18.0 1599 2010-10-22 -Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 -Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 -Chicago New York 2010-10-22 07:00:00 16.0 1531 2010-10-22 -Chicago New York 2010-10-22 07:00:00 -6.0 1610 2010-10-22 -Chicago New York 2010-10-22 07:00:00 -11.0 3198 2010-10-22 -Cleveland New York 2010-10-22 07:00:00 1.0 2630 2010-10-22 -Cleveland New York 2010-10-22 07:00:00 -25.0 2646 2010-10-22 -Cleveland New York 2010-10-22 07:00:00 -3.0 3014 2010-10-22 -Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 -Baltimore New York 2010-10-23 07:00:00 -10.0 1805 2010-10-23 -Baltimore New York 2010-10-23 07:00:00 6.0 3171 2010-10-23 -Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 -Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 -Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 -Chicago New York 2010-10-23 07:00:00 11.0 3085 2010-10-23 -Cleveland New York 2010-10-23 07:00:00 -21.0 2932 2010-10-23 -Washington New York 2010-10-23 07:00:00 -25.0 5832 2010-10-23 -Washington New York 2010-10-23 07:00:00 -21.0 5904 2010-10-23 -Washington New York 2010-10-23 07:00:00 -18.0 5917 2010-10-23 -Washington New York 2010-10-23 07:00:00 -16.0 7274 2010-10-23 -Baltimore New York 2010-10-24 07:00:00 12.0 1599 2010-10-24 -Baltimore New York 2010-10-24 07:00:00 20.0 2571 2010-10-24 -Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 -Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 -Chicago New York 2010-10-24 07:00:00 -5.0 1531 2010-10-24 -Chicago New York 2010-10-24 07:00:00 -17.0 1610 2010-10-24 -Chicago New York 2010-10-24 07:00:00 -3.0 3198 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 5.0 2254 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 -11.0 2630 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 -20.0 2646 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 -9.0 3014 2010-10-24 -Washington New York 2010-10-24 07:00:00 -26.0 7282 2010-10-24 -Baltimore New York 2010-10-25 07:00:00 -25.0 1064 2010-10-25 -Baltimore New York 2010-10-25 07:00:00 92.0 1142 2010-10-25 -Baltimore New York 2010-10-25 07:00:00 106.0 1599 2010-10-25 -Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 -Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 -Chicago New York 2010-10-25 07:00:00 43.0 1531 2010-10-25 -Chicago New York 2010-10-25 07:00:00 6.0 1610 2010-10-25 -Chicago New York 2010-10-25 07:00:00 -16.0 3198 2010-10-25 -Cleveland New York 2010-10-25 07:00:00 -4.0 2630 2010-10-25 -Cleveland New York 2010-10-25 07:00:00 81.0 2646 2010-10-25 -Cleveland New York 2010-10-25 07:00:00 42.0 3014 2010-10-25 -Washington New York 2010-10-25 07:00:00 9.0 7291 2010-10-25 -Baltimore New York 2010-10-26 07:00:00 -22.0 1064 2010-10-26 -Baltimore New York 2010-10-26 07:00:00 123.0 1142 2010-10-26 -Baltimore New York 2010-10-26 07:00:00 90.0 1599 2010-10-26 -Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 -Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 -Chicago New York 2010-10-26 07:00:00 29.0 1531 2010-10-26 -Chicago New York 2010-10-26 07:00:00 -17.0 1610 2010-10-26 -Chicago New York 2010-10-26 07:00:00 6.0 3198 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 4.0 2630 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 -27.0 2646 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 -11.0 2662 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 13.0 3014 2010-10-26 -Washington New York 2010-10-26 07:00:00 4.0 7291 2010-10-26 -Baltimore New York 2010-10-27 07:00:00 -18.0 1064 2010-10-27 -Baltimore New York 2010-10-27 07:00:00 49.0 1142 2010-10-27 -Baltimore New York 2010-10-27 07:00:00 92.0 1599 2010-10-27 -Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 -Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 -Chicago New York 2010-10-27 07:00:00 70.0 1531 2010-10-27 -Chicago New York 2010-10-27 07:00:00 8.0 1610 2010-10-27 -Chicago New York 2010-10-27 07:00:00 21.0 3198 2010-10-27 -Cleveland New York 2010-10-27 07:00:00 16.0 2630 2010-10-27 -Cleveland New York 2010-10-27 07:00:00 27.0 3014 2010-10-27 -Washington New York 2010-10-27 07:00:00 26.0 7291 2010-10-27 -Baltimore New York 2010-10-28 07:00:00 -4.0 1064 2010-10-28 -Baltimore New York 2010-10-28 07:00:00 -14.0 1142 2010-10-28 -Baltimore New York 2010-10-28 07:00:00 -14.0 1599 2010-10-28 -Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 -Chicago New York 2010-10-28 07:00:00 2.0 897 2010-10-28 -Chicago New York 2010-10-28 07:00:00 -11.0 1531 2010-10-28 -Chicago New York 2010-10-28 07:00:00 3.0 1610 2010-10-28 -Chicago New York 2010-10-28 07:00:00 -18.0 3198 2010-10-28 -Cleveland New York 2010-10-28 07:00:00 3.0 2630 2010-10-28 -Cleveland New York 2010-10-28 07:00:00 -6.0 2646 2010-10-28 -Cleveland New York 2010-10-28 07:00:00 1.0 3014 2010-10-28 -Washington New York 2010-10-28 07:00:00 45.0 7291 2010-10-28 -Baltimore New York 2010-10-29 07:00:00 -24.0 1064 2010-10-29 -Baltimore New York 2010-10-29 07:00:00 21.0 1142 2010-10-29 -Baltimore New York 2010-10-29 07:00:00 -2.0 1599 2010-10-29 -Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 -Chicago New York 2010-10-29 07:00:00 -11.0 897 2010-10-29 -Chicago New York 2010-10-29 07:00:00 15.0 1531 2010-10-29 -Chicago New York 2010-10-29 07:00:00 -18.0 1610 2010-10-29 -Chicago New York 2010-10-29 07:00:00 -4.0 3198 2010-10-29 -Cleveland New York 2010-10-29 07:00:00 -4.0 2630 2010-10-29 -Cleveland New York 2010-10-29 07:00:00 -19.0 2646 2010-10-29 -Cleveland New York 2010-10-29 07:00:00 -12.0 3014 2010-10-29 -Washington New York 2010-10-29 07:00:00 1.0 7291 2010-10-29 -Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 -Baltimore New York 2010-10-30 07:00:00 -1.0 1805 2010-10-30 -Baltimore New York 2010-10-30 07:00:00 5.0 3171 2010-10-30 -Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 -Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 -Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 -Chicago New York 2010-10-30 07:00:00 -5.0 3085 2010-10-30 -Cleveland New York 2010-10-30 07:00:00 -23.0 2018 2010-10-30 -Cleveland New York 2010-10-30 07:00:00 -12.0 2932 2010-10-30 -Washington New York 2010-10-30 07:00:00 -27.0 5904 2010-10-30 -Washington New York 2010-10-30 07:00:00 -16.0 5917 2010-10-30 -Baltimore New York 2010-10-31 07:00:00 -1.0 1599 2010-10-31 -Baltimore New York 2010-10-31 07:00:00 -14.0 2571 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -18.0 897 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -4.0 1531 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -22.0 1610 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -15.0 3198 2010-10-31 -Washington New York 2010-10-31 07:00:00 -18.0 7282 2010-10-31 -PREHOOK: query: explain -select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25 -PREHOOK: type: QUERY -POSTHOOK: query: explain -select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: flights_tiny_orc_partitioned_date - Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: origin_city_name (type: string), dest_city_name (type: string), fl_time (type: timestamp), arr_delay (type: float), fl_num (type: int), fl_date (type: date) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col4 (type: int), _col5 (type: date) - sort order: ++ - Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: timestamp), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: date) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 25 - Statistics: Num rows: 25 Data size: 5775 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col4 (type: int), _col5 (type: date) - sort order: ++ - Statistics: Num rows: 25 Data size: 5775 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: timestamp), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: date) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 25 Data size: 5775 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 25 - Statistics: Num rows: 25 Data size: 5775 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 25 Data size: 5775 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: 25 - Processor Tree: - ListSink - -PREHOOK: query: select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25 -PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny_orc_partitioned_date -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 -#### A masked pattern was here #### -POSTHOOK: query: select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 -#### A masked pattern was here #### -Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 -Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 -Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 -Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 -Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 -Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 -Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 -Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 -Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 -Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 -Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 -Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 -Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 -Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 -Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 -Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 -Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 -Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 -Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 -Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 -Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 -Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 -Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 -Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 -Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 -PREHOOK: query: explain -select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date -PREHOOK: type: QUERY -POSTHOOK: query: explain -select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: flights_tiny_orc_partitioned_date - Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: fl_date (type: date) - outputColumnNames: _col0 - Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: _col0 (type: date) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: date) - sort order: + - Map-reduce partition columns: _col0 (type: date) - Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: date) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 68 Data size: 15772 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 68 Data size: 15772 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date -PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny_orc_partitioned_date -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 -PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 -#### A masked pattern was here #### -POSTHOOK: query: select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date -POSTHOOK: type: QUERY -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 -#### A masked pattern was here #### -2010-10-20 11 -2010-10-21 12 -2010-10-22 11 -2010-10-23 12 -2010-10-24 12 -2010-10-25 12 -2010-10-26 13 -2010-10-27 11 -2010-10-28 12 -2010-10-29 12 -2010-10-30 11 -2010-10-31 8 -PREHOOK: query: CREATE TABLE flights_tiny_orc_partitioned_timestamp ( - origin_city_name STRING, - dest_city_name STRING, - fl_date DATE, - arr_delay FLOAT, - fl_num INT -) -PARTITIONED BY (fl_time TIMESTAMP) -STORED AS ORC -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@flights_tiny_orc_partitioned_timestamp -POSTHOOK: query: CREATE TABLE flights_tiny_orc_partitioned_timestamp ( - origin_city_name STRING, - dest_city_name STRING, - fl_date DATE, - arr_delay FLOAT, - fl_num INT -) -PARTITIONED BY (fl_time TIMESTAMP) -STORED AS ORC -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp -PREHOOK: query: INSERT INTO TABLE flights_tiny_orc_partitioned_timestamp -PARTITION (fl_time) -SELECT origin_city_name, dest_city_name, fl_date, arr_delay, fl_num, fl_time -FROM flights_tiny_orc -PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny_orc -PREHOOK: Output: default@flights_tiny_orc_partitioned_timestamp -POSTHOOK: query: INSERT INTO TABLE flights_tiny_orc_partitioned_timestamp -PARTITION (fl_time) -SELECT origin_city_name, dest_city_name, fl_date, arr_delay, fl_num, fl_time -FROM flights_tiny_orc -POSTHOOK: type: QUERY -POSTHOOK: Input: default@flights_tiny_orc -POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20%2007%3A00%3A00 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21%2007%3A00%3A00 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22%2007%3A00%3A00 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23%2007%3A00%3A00 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24%2007%3A00%3A00 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25%2007%3A00%3A00 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26%2007%3A00%3A00 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27%2007%3A00%3A00 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28%2007%3A00%3A00 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29%2007%3A00%3A00 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30%2007%3A00%3A00 -POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31%2007%3A00%3A00 -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-20 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-20 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-20 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-20 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-20 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-21 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-21 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-21 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-21 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-21 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-22 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-22 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-22 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-22 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-22 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-23 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-23 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-23 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-23 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-23 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-24 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-24 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-24 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-24 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-24 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-25 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-25 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-25 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-25 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-25 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-26 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-26 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-26 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-26 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-26 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-27 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-27 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-27 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-27 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-27 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-28 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-28 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-28 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-28 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-28 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-29 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-29 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-29 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-29 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-29 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-30 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-30 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-30 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-30 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-30 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-31 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-31 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-31 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-31 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] -POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-31 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] -PREHOOK: query: select * from flights_tiny_orc_partitioned_timestamp -PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31%2007%3A00%3A00 -#### A masked pattern was here #### -POSTHOOK: query: select * from flights_tiny_orc_partitioned_timestamp -POSTHOOK: type: QUERY -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31%2007%3A00%3A00 -#### A masked pattern was here #### -Baltimore New York 2010-10-20 -30.0 1064 2010-10-20 07:00:00 -Baltimore New York 2010-10-20 23.0 1142 2010-10-20 07:00:00 -Baltimore New York 2010-10-20 6.0 1599 2010-10-20 07:00:00 -Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 -Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 -Chicago New York 2010-10-20 15.0 1531 2010-10-20 07:00:00 -Chicago New York 2010-10-20 -6.0 1610 2010-10-20 07:00:00 -Chicago New York 2010-10-20 -2.0 3198 2010-10-20 07:00:00 -Cleveland New York 2010-10-20 -8.0 2630 2010-10-20 07:00:00 -Cleveland New York 2010-10-20 -15.0 3014 2010-10-20 07:00:00 -Washington New York 2010-10-20 -2.0 7291 2010-10-20 07:00:00 -Baltimore New York 2010-10-21 17.0 1064 2010-10-21 07:00:00 -Baltimore New York 2010-10-21 105.0 1142 2010-10-21 07:00:00 -Baltimore New York 2010-10-21 28.0 1599 2010-10-21 07:00:00 -Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 -Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 -Chicago New York 2010-10-21 53.0 1531 2010-10-21 07:00:00 -Chicago New York 2010-10-21 -5.0 1610 2010-10-21 07:00:00 -Chicago New York 2010-10-21 51.0 3198 2010-10-21 07:00:00 -Cleveland New York 2010-10-21 3.0 2630 2010-10-21 07:00:00 -Cleveland New York 2010-10-21 29.0 2646 2010-10-21 07:00:00 -Cleveland New York 2010-10-21 72.0 3014 2010-10-21 07:00:00 -Washington New York 2010-10-21 22.0 7291 2010-10-21 07:00:00 -Baltimore New York 2010-10-22 -12.0 1064 2010-10-22 07:00:00 -Baltimore New York 2010-10-22 54.0 1142 2010-10-22 07:00:00 -Baltimore New York 2010-10-22 18.0 1599 2010-10-22 07:00:00 -Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 -Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 -Chicago New York 2010-10-22 16.0 1531 2010-10-22 07:00:00 -Chicago New York 2010-10-22 -6.0 1610 2010-10-22 07:00:00 -Chicago New York 2010-10-22 -11.0 3198 2010-10-22 07:00:00 -Cleveland New York 2010-10-22 1.0 2630 2010-10-22 07:00:00 -Cleveland New York 2010-10-22 -25.0 2646 2010-10-22 07:00:00 -Cleveland New York 2010-10-22 -3.0 3014 2010-10-22 07:00:00 -Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 -Baltimore New York 2010-10-23 -10.0 1805 2010-10-23 07:00:00 -Baltimore New York 2010-10-23 6.0 3171 2010-10-23 07:00:00 -Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 -Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 -Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 -Chicago New York 2010-10-23 11.0 3085 2010-10-23 07:00:00 -Cleveland New York 2010-10-23 -21.0 2932 2010-10-23 07:00:00 -Washington New York 2010-10-23 -25.0 5832 2010-10-23 07:00:00 -Washington New York 2010-10-23 -21.0 5904 2010-10-23 07:00:00 -Washington New York 2010-10-23 -18.0 5917 2010-10-23 07:00:00 -Washington New York 2010-10-23 -16.0 7274 2010-10-23 07:00:00 -Baltimore New York 2010-10-24 12.0 1599 2010-10-24 07:00:00 -Baltimore New York 2010-10-24 20.0 2571 2010-10-24 07:00:00 -Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 -Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 -Chicago New York 2010-10-24 -5.0 1531 2010-10-24 07:00:00 -Chicago New York 2010-10-24 -17.0 1610 2010-10-24 07:00:00 -Chicago New York 2010-10-24 -3.0 3198 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 5.0 2254 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 -11.0 2630 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 -20.0 2646 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 -9.0 3014 2010-10-24 07:00:00 -Washington New York 2010-10-24 -26.0 7282 2010-10-24 07:00:00 -Baltimore New York 2010-10-25 -25.0 1064 2010-10-25 07:00:00 -Baltimore New York 2010-10-25 92.0 1142 2010-10-25 07:00:00 -Baltimore New York 2010-10-25 106.0 1599 2010-10-25 07:00:00 -Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 -Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 -Chicago New York 2010-10-25 43.0 1531 2010-10-25 07:00:00 -Chicago New York 2010-10-25 6.0 1610 2010-10-25 07:00:00 -Chicago New York 2010-10-25 -16.0 3198 2010-10-25 07:00:00 -Cleveland New York 2010-10-25 -4.0 2630 2010-10-25 07:00:00 -Cleveland New York 2010-10-25 81.0 2646 2010-10-25 07:00:00 -Cleveland New York 2010-10-25 42.0 3014 2010-10-25 07:00:00 -Washington New York 2010-10-25 9.0 7291 2010-10-25 07:00:00 -Baltimore New York 2010-10-26 -22.0 1064 2010-10-26 07:00:00 -Baltimore New York 2010-10-26 123.0 1142 2010-10-26 07:00:00 -Baltimore New York 2010-10-26 90.0 1599 2010-10-26 07:00:00 -Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 -Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 -Chicago New York 2010-10-26 29.0 1531 2010-10-26 07:00:00 -Chicago New York 2010-10-26 -17.0 1610 2010-10-26 07:00:00 -Chicago New York 2010-10-26 6.0 3198 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 4.0 2630 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 -27.0 2646 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 -11.0 2662 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 13.0 3014 2010-10-26 07:00:00 -Washington New York 2010-10-26 4.0 7291 2010-10-26 07:00:00 -Baltimore New York 2010-10-27 -18.0 1064 2010-10-27 07:00:00 -Baltimore New York 2010-10-27 49.0 1142 2010-10-27 07:00:00 -Baltimore New York 2010-10-27 92.0 1599 2010-10-27 07:00:00 -Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 -Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 -Chicago New York 2010-10-27 70.0 1531 2010-10-27 07:00:00 -Chicago New York 2010-10-27 8.0 1610 2010-10-27 07:00:00 -Chicago New York 2010-10-27 21.0 3198 2010-10-27 07:00:00 -Cleveland New York 2010-10-27 16.0 2630 2010-10-27 07:00:00 -Cleveland New York 2010-10-27 27.0 3014 2010-10-27 07:00:00 -Washington New York 2010-10-27 26.0 7291 2010-10-27 07:00:00 -Baltimore New York 2010-10-28 -4.0 1064 2010-10-28 07:00:00 -Baltimore New York 2010-10-28 -14.0 1142 2010-10-28 07:00:00 -Baltimore New York 2010-10-28 -14.0 1599 2010-10-28 07:00:00 -Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 -Chicago New York 2010-10-28 2.0 897 2010-10-28 07:00:00 -Chicago New York 2010-10-28 -11.0 1531 2010-10-28 07:00:00 -Chicago New York 2010-10-28 3.0 1610 2010-10-28 07:00:00 -Chicago New York 2010-10-28 -18.0 3198 2010-10-28 07:00:00 -Cleveland New York 2010-10-28 3.0 2630 2010-10-28 07:00:00 -Cleveland New York 2010-10-28 -6.0 2646 2010-10-28 07:00:00 -Cleveland New York 2010-10-28 1.0 3014 2010-10-28 07:00:00 -Washington New York 2010-10-28 45.0 7291 2010-10-28 07:00:00 -Baltimore New York 2010-10-29 -24.0 1064 2010-10-29 07:00:00 -Baltimore New York 2010-10-29 21.0 1142 2010-10-29 07:00:00 -Baltimore New York 2010-10-29 -2.0 1599 2010-10-29 07:00:00 -Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 -Chicago New York 2010-10-29 -11.0 897 2010-10-29 07:00:00 -Chicago New York 2010-10-29 15.0 1531 2010-10-29 07:00:00 -Chicago New York 2010-10-29 -18.0 1610 2010-10-29 07:00:00 -Chicago New York 2010-10-29 -4.0 3198 2010-10-29 07:00:00 -Cleveland New York 2010-10-29 -4.0 2630 2010-10-29 07:00:00 -Cleveland New York 2010-10-29 -19.0 2646 2010-10-29 07:00:00 -Cleveland New York 2010-10-29 -12.0 3014 2010-10-29 07:00:00 -Washington New York 2010-10-29 1.0 7291 2010-10-29 07:00:00 -Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 -Baltimore New York 2010-10-30 -1.0 1805 2010-10-30 07:00:00 -Baltimore New York 2010-10-30 5.0 3171 2010-10-30 07:00:00 -Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 -Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 -Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 -Chicago New York 2010-10-30 -5.0 3085 2010-10-30 07:00:00 -Cleveland New York 2010-10-30 -23.0 2018 2010-10-30 07:00:00 -Cleveland New York 2010-10-30 -12.0 2932 2010-10-30 07:00:00 -Washington New York 2010-10-30 -27.0 5904 2010-10-30 07:00:00 -Washington New York 2010-10-30 -16.0 5917 2010-10-30 07:00:00 -Baltimore New York 2010-10-31 -1.0 1599 2010-10-31 07:00:00 -Baltimore New York 2010-10-31 -14.0 2571 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -18.0 897 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -4.0 1531 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -22.0 1610 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -15.0 3198 2010-10-31 07:00:00 -Washington New York 2010-10-31 -18.0 7282 2010-10-31 07:00:00 -PREHOOK: query: select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25 -PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31%2007%3A00%3A00 -#### A masked pattern was here #### -POSTHOOK: query: select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31%2007%3A00%3A00 -#### A masked pattern was here #### -Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 -Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 -Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 -Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 -Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 -Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 -Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 -Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 -Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 -Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 -Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 -Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 -Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 -Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 -Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 -Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 -Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 -Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 -Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 -Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 -Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 -Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 -Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 -Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 -Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 -PREHOOK: query: select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time -PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31%2007%3A00%3A00 -#### A masked pattern was here #### -POSTHOOK: query: select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time -POSTHOOK: type: QUERY -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31%2007%3A00%3A00 -#### A masked pattern was here #### -2010-10-20 07:00:00 11 -2010-10-21 07:00:00 12 -2010-10-22 07:00:00 11 -2010-10-23 07:00:00 12 -2010-10-24 07:00:00 12 -2010-10-25 07:00:00 12 -2010-10-26 07:00:00 13 -2010-10-27 07:00:00 11 -2010-10-28 07:00:00 12 -2010-10-29 07:00:00 12 -2010-10-30 07:00:00 11 -2010-10-31 07:00:00 8 -PREHOOK: query: explain -select * from flights_tiny_orc_partitioned_timestamp -PREHOOK: type: QUERY -POSTHOOK: query: explain -select * from flights_tiny_orc_partitioned_timestamp -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: flights_tiny_orc_partitioned_timestamp - Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: origin_city_name (type: string), dest_city_name (type: string), fl_date (type: date), arr_delay (type: float), fl_num (type: int), fl_time (type: timestamp) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE - ListSink - -PREHOOK: query: select * from flights_tiny_orc_partitioned_timestamp -PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31%2007%3A00%3A00 -#### A masked pattern was here #### -POSTHOOK: query: select * from flights_tiny_orc_partitioned_timestamp -POSTHOOK: type: QUERY -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31%2007%3A00%3A00 -#### A masked pattern was here #### -Baltimore New York 2010-10-20 -30.0 1064 2010-10-20 07:00:00 -Baltimore New York 2010-10-20 23.0 1142 2010-10-20 07:00:00 -Baltimore New York 2010-10-20 6.0 1599 2010-10-20 07:00:00 -Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 -Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 -Chicago New York 2010-10-20 15.0 1531 2010-10-20 07:00:00 -Chicago New York 2010-10-20 -6.0 1610 2010-10-20 07:00:00 -Chicago New York 2010-10-20 -2.0 3198 2010-10-20 07:00:00 -Cleveland New York 2010-10-20 -8.0 2630 2010-10-20 07:00:00 -Cleveland New York 2010-10-20 -15.0 3014 2010-10-20 07:00:00 -Washington New York 2010-10-20 -2.0 7291 2010-10-20 07:00:00 -Baltimore New York 2010-10-21 17.0 1064 2010-10-21 07:00:00 -Baltimore New York 2010-10-21 105.0 1142 2010-10-21 07:00:00 -Baltimore New York 2010-10-21 28.0 1599 2010-10-21 07:00:00 -Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 -Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 -Chicago New York 2010-10-21 53.0 1531 2010-10-21 07:00:00 -Chicago New York 2010-10-21 -5.0 1610 2010-10-21 07:00:00 -Chicago New York 2010-10-21 51.0 3198 2010-10-21 07:00:00 -Cleveland New York 2010-10-21 3.0 2630 2010-10-21 07:00:00 -Cleveland New York 2010-10-21 29.0 2646 2010-10-21 07:00:00 -Cleveland New York 2010-10-21 72.0 3014 2010-10-21 07:00:00 -Washington New York 2010-10-21 22.0 7291 2010-10-21 07:00:00 -Baltimore New York 2010-10-22 -12.0 1064 2010-10-22 07:00:00 -Baltimore New York 2010-10-22 54.0 1142 2010-10-22 07:00:00 -Baltimore New York 2010-10-22 18.0 1599 2010-10-22 07:00:00 -Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 -Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 -Chicago New York 2010-10-22 16.0 1531 2010-10-22 07:00:00 -Chicago New York 2010-10-22 -6.0 1610 2010-10-22 07:00:00 -Chicago New York 2010-10-22 -11.0 3198 2010-10-22 07:00:00 -Cleveland New York 2010-10-22 1.0 2630 2010-10-22 07:00:00 -Cleveland New York 2010-10-22 -25.0 2646 2010-10-22 07:00:00 -Cleveland New York 2010-10-22 -3.0 3014 2010-10-22 07:00:00 -Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 -Baltimore New York 2010-10-23 -10.0 1805 2010-10-23 07:00:00 -Baltimore New York 2010-10-23 6.0 3171 2010-10-23 07:00:00 -Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 -Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 -Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 -Chicago New York 2010-10-23 11.0 3085 2010-10-23 07:00:00 -Cleveland New York 2010-10-23 -21.0 2932 2010-10-23 07:00:00 -Washington New York 2010-10-23 -25.0 5832 2010-10-23 07:00:00 -Washington New York 2010-10-23 -21.0 5904 2010-10-23 07:00:00 -Washington New York 2010-10-23 -18.0 5917 2010-10-23 07:00:00 -Washington New York 2010-10-23 -16.0 7274 2010-10-23 07:00:00 -Baltimore New York 2010-10-24 12.0 1599 2010-10-24 07:00:00 -Baltimore New York 2010-10-24 20.0 2571 2010-10-24 07:00:00 -Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 -Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 -Chicago New York 2010-10-24 -5.0 1531 2010-10-24 07:00:00 -Chicago New York 2010-10-24 -17.0 1610 2010-10-24 07:00:00 -Chicago New York 2010-10-24 -3.0 3198 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 5.0 2254 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 -11.0 2630 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 -20.0 2646 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 -9.0 3014 2010-10-24 07:00:00 -Washington New York 2010-10-24 -26.0 7282 2010-10-24 07:00:00 -Baltimore New York 2010-10-25 -25.0 1064 2010-10-25 07:00:00 -Baltimore New York 2010-10-25 92.0 1142 2010-10-25 07:00:00 -Baltimore New York 2010-10-25 106.0 1599 2010-10-25 07:00:00 -Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 -Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 -Chicago New York 2010-10-25 43.0 1531 2010-10-25 07:00:00 -Chicago New York 2010-10-25 6.0 1610 2010-10-25 07:00:00 -Chicago New York 2010-10-25 -16.0 3198 2010-10-25 07:00:00 -Cleveland New York 2010-10-25 -4.0 2630 2010-10-25 07:00:00 -Cleveland New York 2010-10-25 81.0 2646 2010-10-25 07:00:00 -Cleveland New York 2010-10-25 42.0 3014 2010-10-25 07:00:00 -Washington New York 2010-10-25 9.0 7291 2010-10-25 07:00:00 -Baltimore New York 2010-10-26 -22.0 1064 2010-10-26 07:00:00 -Baltimore New York 2010-10-26 123.0 1142 2010-10-26 07:00:00 -Baltimore New York 2010-10-26 90.0 1599 2010-10-26 07:00:00 -Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 -Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 -Chicago New York 2010-10-26 29.0 1531 2010-10-26 07:00:00 -Chicago New York 2010-10-26 -17.0 1610 2010-10-26 07:00:00 -Chicago New York 2010-10-26 6.0 3198 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 4.0 2630 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 -27.0 2646 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 -11.0 2662 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 13.0 3014 2010-10-26 07:00:00 -Washington New York 2010-10-26 4.0 7291 2010-10-26 07:00:00 -Baltimore New York 2010-10-27 -18.0 1064 2010-10-27 07:00:00 -Baltimore New York 2010-10-27 49.0 1142 2010-10-27 07:00:00 -Baltimore New York 2010-10-27 92.0 1599 2010-10-27 07:00:00 -Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 -Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 -Chicago New York 2010-10-27 70.0 1531 2010-10-27 07:00:00 -Chicago New York 2010-10-27 8.0 1610 2010-10-27 07:00:00 -Chicago New York 2010-10-27 21.0 3198 2010-10-27 07:00:00 -Cleveland New York 2010-10-27 16.0 2630 2010-10-27 07:00:00 -Cleveland New York 2010-10-27 27.0 3014 2010-10-27 07:00:00 -Washington New York 2010-10-27 26.0 7291 2010-10-27 07:00:00 -Baltimore New York 2010-10-28 -4.0 1064 2010-10-28 07:00:00 -Baltimore New York 2010-10-28 -14.0 1142 2010-10-28 07:00:00 -Baltimore New York 2010-10-28 -14.0 1599 2010-10-28 07:00:00 -Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 -Chicago New York 2010-10-28 2.0 897 2010-10-28 07:00:00 -Chicago New York 2010-10-28 -11.0 1531 2010-10-28 07:00:00 -Chicago New York 2010-10-28 3.0 1610 2010-10-28 07:00:00 -Chicago New York 2010-10-28 -18.0 3198 2010-10-28 07:00:00 -Cleveland New York 2010-10-28 3.0 2630 2010-10-28 07:00:00 -Cleveland New York 2010-10-28 -6.0 2646 2010-10-28 07:00:00 -Cleveland New York 2010-10-28 1.0 3014 2010-10-28 07:00:00 -Washington New York 2010-10-28 45.0 7291 2010-10-28 07:00:00 -Baltimore New York 2010-10-29 -24.0 1064 2010-10-29 07:00:00 -Baltimore New York 2010-10-29 21.0 1142 2010-10-29 07:00:00 -Baltimore New York 2010-10-29 -2.0 1599 2010-10-29 07:00:00 -Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 -Chicago New York 2010-10-29 -11.0 897 2010-10-29 07:00:00 -Chicago New York 2010-10-29 15.0 1531 2010-10-29 07:00:00 -Chicago New York 2010-10-29 -18.0 1610 2010-10-29 07:00:00 -Chicago New York 2010-10-29 -4.0 3198 2010-10-29 07:00:00 -Cleveland New York 2010-10-29 -4.0 2630 2010-10-29 07:00:00 -Cleveland New York 2010-10-29 -19.0 2646 2010-10-29 07:00:00 -Cleveland New York 2010-10-29 -12.0 3014 2010-10-29 07:00:00 -Washington New York 2010-10-29 1.0 7291 2010-10-29 07:00:00 -Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 -Baltimore New York 2010-10-30 -1.0 1805 2010-10-30 07:00:00 -Baltimore New York 2010-10-30 5.0 3171 2010-10-30 07:00:00 -Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 -Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 -Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 -Chicago New York 2010-10-30 -5.0 3085 2010-10-30 07:00:00 -Cleveland New York 2010-10-30 -23.0 2018 2010-10-30 07:00:00 -Cleveland New York 2010-10-30 -12.0 2932 2010-10-30 07:00:00 -Washington New York 2010-10-30 -27.0 5904 2010-10-30 07:00:00 -Washington New York 2010-10-30 -16.0 5917 2010-10-30 07:00:00 -Baltimore New York 2010-10-31 -1.0 1599 2010-10-31 07:00:00 -Baltimore New York 2010-10-31 -14.0 2571 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -18.0 897 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -4.0 1531 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -22.0 1610 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -15.0 3198 2010-10-31 07:00:00 -Washington New York 2010-10-31 -18.0 7282 2010-10-31 07:00:00 -PREHOOK: query: explain -select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25 -PREHOOK: type: QUERY -POSTHOOK: query: explain -select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: flights_tiny_orc_partitioned_timestamp - Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: origin_city_name (type: string), dest_city_name (type: string), fl_date (type: date), arr_delay (type: float), fl_num (type: int), fl_time (type: timestamp) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col4 (type: int), _col5 (type: timestamp) - sort order: ++ - Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: date), _col3 (type: float) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: date), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: timestamp) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 25 - Statistics: Num rows: 25 Data size: 6175 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col4 (type: int), _col5 (type: timestamp) - sort order: ++ - Statistics: Num rows: 25 Data size: 6175 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: date), _col3 (type: float) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: date), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: timestamp) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 25 Data size: 6175 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 25 - Statistics: Num rows: 25 Data size: 6175 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 25 Data size: 6175 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: 25 - Processor Tree: - ListSink - -PREHOOK: query: select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25 -PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31%2007%3A00%3A00 -#### A masked pattern was here #### -POSTHOOK: query: select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31%2007%3A00%3A00 -#### A masked pattern was here #### -Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 -Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 -Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 -Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 -Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 -Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 -Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 -Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 -Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 -Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 -Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 -Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 -Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 -Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 -Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 -Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 -Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 -Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 -Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 -Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 -Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 -Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 -Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 -Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 -Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 -PREHOOK: query: explain -select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time -PREHOOK: type: QUERY -POSTHOOK: query: explain -select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: flights_tiny_orc_partitioned_timestamp - Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: fl_time (type: timestamp) - outputColumnNames: _col0 - Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: _col0 (type: timestamp) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: timestamp) - sort order: + - Map-reduce partition columns: _col0 (type: timestamp) - Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: timestamp) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 68 Data size: 16860 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 68 Data size: 16860 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time -PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30%2007%3A00%3A00 -PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31%2007%3A00%3A00 -#### A masked pattern was here #### -POSTHOOK: query: select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time -POSTHOOK: type: QUERY -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30%2007%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31%2007%3A00%3A00 -#### A masked pattern was here #### -2010-10-20 07:00:00 11 -2010-10-21 07:00:00 12 -2010-10-22 07:00:00 11 -2010-10-23 07:00:00 12 -2010-10-24 07:00:00 12 -2010-10-25 07:00:00 12 -2010-10-26 07:00:00 13 -2010-10-27 07:00:00 11 -2010-10-28 07:00:00 12 -2010-10-29 07:00:00 12 -2010-10-30 07:00:00 11 -2010-10-31 07:00:00 8 diff --git service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java index 844baf7..ebec165 100644 --- service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java +++ service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java @@ -107,7 +107,7 @@ public void run() { } connector.setPort(portNum); // Linux:yes, Windows:no - connector.setReuseAddress(!Shell.WINDOWS); + connector.setReuseAddress(true); int maxIdleTime = (int) hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_MAX_IDLE_TIME, TimeUnit.MILLISECONDS); connector.setMaxIdleTime(maxIdleTime); diff --git shims/common/src/main/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java shims/common/src/main/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java index 197b965..dc32190 100644 --- shims/common/src/main/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java +++ shims/common/src/main/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java @@ -59,14 +59,6 @@ public void initialize(URI name, Configuration conf) throws IOException { // from the supplied URI this.scheme = name.getScheme(); String nameUriString = name.toString(); - if (Shell.WINDOWS) { - // Replace the encoded backward slash with forward slash - // Remove the windows drive letter - nameUriString = - nameUriString.replaceAll("%5C", "/").replaceFirst("/[c-zC-Z]:", "/") - .replaceFirst("^[c-zC-Z]:", ""); - name = URI.create(nameUriString); - } String authority = name.getAuthority() != null ? name.getAuthority() : ""; String proxyUriString = scheme + "://" + authority + "/"; diff --git testutils/hadoop.cmd testutils/hadoop.cmd deleted file mode 100644 index 1ff147c..0000000 --- testutils/hadoop.cmd +++ /dev/null @@ -1,252 +0,0 @@ -@echo off -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. - - -@rem The Hadoop command script -@rem -@rem Environment Variables -@rem -@rem JAVA_HOME The java implementation to use. Overrides JAVA_HOME. -@rem -@rem HADOOP_CLASSPATH Extra Java CLASSPATH entries. -@rem -@rem HADOOP_HEAPSIZE The maximum amount of heap to use, in MB. -@rem Default is 1000. -@rem -@rem HADOOP_OPTS Extra Java runtime options. -@rem -@rem HADOOP_NAMENODE_OPTS These options are added to HADOOP_OPTS -@rem HADOOP_CLIENT_OPTS when the respective command is run. -@rem HADOOP_{COMMAND}_OPTS etc HADOOP_JT_OPTS applies to JobTracker -@rem for e.g. HADOOP_CLIENT_OPTS applies to -@rem more than one command (fs, dfs, fsck, -@rem dfsadmin etc) -@rem -@rem HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. -@rem -@rem HADOOP_ROOT_LOGGER The root appender. Default is INFO,console -@rem - -if not defined HADOOP_BIN_PATH ( - set HADOOP_BIN_PATH=%~dp0 -) - -if "%HADOOP_BIN_PATH:~-1%" == "\" ( - set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% -) -call :updatepath %HADOOP_BIN_PATH% - -set BIN=%~dp0 -for %%i in (%BIN%.) do ( - set BIN=%%~dpi -) -if "%BIN:~-1%" == "\" ( - set BIN=%BIN:~0,-1% -) - - -@rem -@rem setup java environment variables -@rem - -if not defined JAVA_HOME ( - echo Error: JAVA_HOME is not set. - goto :eof -) - -if not exist %JAVA_HOME%\bin\java.exe ( - echo Error: JAVA_HOME is incorrectly set. - goto :eof -) - -set JAVA=%JAVA_HOME%\bin\java -set JAVA_HEAP_MAX=-Xmx1000m - -@rem -@rem check envvars which might override default args -@rem - -if defined HADOOP_HEAPSIZE ( - set JAVA_HEAP_MAX=-Xmx%HADOOP_HEAPSIZE%m -) - -@rem -@rem CLASSPATH initially contains %HADOOP_CONF_DIR% -@rem - -set CLASSPATH=%HADOOP_CONF_DIR% -set CLASSPATH=%CLASSPATH%;%JAVA_HOME%\lib\tools.jar - - -set BUILD_ROOT="%BIN%"/build - - -if not defined HIVE_HADOOP_TEST_CLASSPATH ( - @echo Error: HIVE_HADOOP_TEST_CLASSPATH not defined. - goto :eof -) - - - -set CLASSPATH=%CLASSPATH%;%HIVE_HADOOP_TEST_CLASSPATH% -if not exist %BUILD_ROOT%/test/hadoop/logs ( - mkdir %BUILD_ROOT%/test/hadoop/logs -) - -@rem -@rem add user-specified CLASSPATH last -@rem - -if defined HADOOP_CLASSPATH ( - set CLASSPATH=%CLASSPATH%;%HADOOP_CLASSPATH% -) - -if not defined HADOOP_LOG_DIR ( - set HADOOP_LOG_DIR=%BUILD_ROOT%\logs -) - -if not defined HADOOP_LOGFILE ( - set HADOOP_LOGFILE=hadoop.log -) - -if not defined HADOOP_ROOT_LOGGER ( - set HADOOP_ROOT_LOGGER=INFO,console,DRFA -) - -@rem -@rem default policy file for service-level authorization -@rem - -if not defined HADOOP_POLICYFILE ( - set HADOOP_POLICYFILE=hadoop-policy.xml -) -set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.log.dir=%HADOOP_LOG_DIR% -set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.log.file=%HADOOP_LOGFILE% -set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.root.logger=%HADOOP_ROOT_LOGGER% - -if defined HADOOP_PREFIX ( - set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.home.dir=%HADOOP_PREFIX% -) - -if defined HADOOP_IDENT_STRING ( - set HADOOP_OPTS=%$HADOOP_OPTS% -Dhadoop.id.str=%HADOOP_IDENT_STRING% -) - -if defined JAVA_LIBRARY_PATH ( - set HADOOP_OPTS=%HADOOP_OPTS% -Djava.library.path=%JAVA_LIBRARY_PATH% -) -set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.policy.file=%HADOOP_POLICYFILE% - -@rem Disable ipv6 as it can cause issues -set HADOOP_OPTS=%HADOOP_OPTS% -Djava.net.preferIPv4Stack=true - -:main - setlocal enabledelayedexpansion - - set hadoop-command=%1 - if not defined hadoop-command ( - goto print_usage - ) - - call :make_command_arguments %* - set corecommands=fs version jar distcp daemonlog archive - for %%i in ( %corecommands% ) do ( - if %hadoop-command% == %%i set corecommand=true - ) - if defined corecommand ( - call :%hadoop-command% - ) else ( - set CLASSPATH=%CLASSPATH%;%CD% - set CLASS=%hadoop-command% - ) - call %JAVA% %JAVA_HEAP_MAX% %HADOOP_OPTS% -classpath %CLASSPATH% %CLASS% %hadoop-command-arguments% - exit /b %ERRORLEVEL% - goto :eof - -:version - set CLASS=org.apache.hadoop.util.VersionInfo - set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS% - goto :eof - -:jar - set CLASS=org.apache.hadoop.util.RunJar - goto :eof - -:distcp - set CLASS=org.apache.hadoop.tools.DistCp - set CLASSPATH=%CLASSPATH%;%TOOL_PATH% - set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS% - goto :eof - -:daemonlog - set CLASS=org.apache.hadoop.log.LogLevel - set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS% - goto :eof - -:archive - set CLASS=org.apache.hadoop.tools.HadoopArchives - set CLASSPATH=%CLASSPATH%;%TOOL_PATH% - set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS% - goto :eof - -:updatepath - set path_to_add=%* - set current_path_comparable=%path:(x86)=% - set current_path_comparable=%current_path_comparable: =_% - set path_to_add_comparable=%path_to_add:(x86)=% - set path_to_add_comparable=%path_to_add_comparable: =_% - for %%i in ( %current_path_comparable% ) do ( - if /i "%%i" == "%path_to_add_comparable%" ( - set path_to_add_exist=true - ) - ) - set system_path_comparable= - set path_to_add_comparable= - if not defined path_to_add_exist path=%path_to_add%;%path% - set path_to_add= - goto :eof - -:make_command_arguments - if "%2" == "" goto :eof - set _count=0 - set _shift=1 - for %%i in (%*) do ( - set /a _count=!_count!+1 - if !_count! GTR %_shift% ( - if not defined _arguments ( - set _arguments=%%i - ) else ( - set _arguments=!_arguments! %%i - ) - ) - ) - - set hadoop-command-arguments=%_arguments% - goto :eof - -:print_usage - @echo Usage: hadoop COMMAND - @echo where COMMAND is one of: - @echo fs run a generic filesystem user client - @echo version print the version - @echo jar ^ run a jar file - @echo. - @echo distcp ^ ^ copy file or directories recursively - @echo archive -archiveName NAME ^* ^ create a hadoop archive - @echo daemonlog get/set the log level for each daemon - @echo Most commands print help when invoked w/o parameters. - -endlocal