Index: oozie/distro/src/main/resources/LICENSE.txt =================================================================== --- oozie/distro/src/main/resources/LICENSE.txt (revision 0) +++ oozie/distro/src/main/resources/LICENSE.txt (revision 0) @@ -0,0 +1,203 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + Index: oozie/distro/src/main/resources/README.txt =================================================================== --- oozie/distro/src/main/resources/README.txt (revision 0) +++ oozie/distro/src/main/resources/README.txt (revision 0) @@ -0,0 +1,12 @@ +Oozie Distribution +======================== + +This tar ball contains the following Oozie distribution: + +Oozie version: ${project.version} +Hadoop version: ${hadoopVersion} + +Detailed License information can be found at: docs/index.html##LicenseInfo + +All the documentation can be found under the docs/ directory. + Index: oozie/distro/pom.xml =================================================================== --- oozie/distro/pom.xml (revision 0) +++ oozie/distro/pom.xml (revision 0) @@ -0,0 +1,80 @@ + + + 4.0.0 + + org.apache.oozie + oozie-main + ${hadoopVersion}${oozieVersion} + + + oozie + Oozie Distro + Oozie Distro + jar + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + + + + + + org.apache.oozie + oozie-core + ${project.version} + compile + + + org.apache.oozie + oozie-client + ${project.version} + compile + + + org.apache.oozie + oozie-webapp + ${project.version} + compile + war + + + org.apache.oozie + oozie-examples + ${project.version} + compile + + + + + + + src/main/resources + false + + README.txt + + + + src/main/resources + + README.txt + + true + + + + + maven-assembly-plugin + + + ../src/main/assemblies/distro.xml + + + + + + + + Index: oozie/src/main/assemblies/distro.xml =================================================================== --- oozie/src/main/assemblies/distro.xml (revision 0) +++ oozie/src/main/assemblies/distro.xml (revision 0) @@ -0,0 +1,132 @@ + + distro + + dir + tar.gz + + true + oozie-${artifact.version} + + + + ${basedir}/../webapp/target/classes/ + /conf + + **/* + + 0644 + + + + ${basedir}/target/classes/ + / + + **/* + + 0444 + + + + ${basedir}/../client/target/oozie-client-${artifact.version}-client.dir/bin + /bin + + * + + 0555 + + + ${basedir}/../client/target/oozie-client-${artifact.version}-client.dir/lib + /lib + + * + + 0444 + + + + ${basedir}/../target/site/apidocs + /docs/apidocs/ + + **/* + + 0444 + + + + ${basedir}/../docs/target/site/ + /docs/ + + **/* + + 0444 + + + + ${basedir}/../core/target/site/cobertura/ + /docs/cobertura/ + + **/* + + 0444 + + + + ${basedir}/../core/target/site/ + /docs/findbugs/ + + findbugs.html + css/* + images/* + + 0444 + + + + ${basedir}/../webapp/target/site/ + /docs/ + + dependencies.html + css/* + images/* + + 0444 + + + + + + ${basedir}/../core/target/classes/oozie-default.xml + /docs/ + oozie-default.xml.txt + 0444 + + + + ${basedir}/../webapp/target/classes/oozie-log4j.properties + /docs/ + oozie-log4j.properties.txt + 0444 + + + + ${basedir}/../webapp/target/oozie-webapp-${project.version}.war + / + oozie.war + 0444 + + + + ${basedir}/../client/target/oozie-client-${artifact.version}-client.tar.gz + / + oozie-client.tar.gz + 0444 + + + + ${basedir}/../examples/target/oozie-examples-${artifact.version}-examples.tar.gz + / + oozie-examples.tar.gz + 0444 + + + Index: oozie/src/main/assemblies/client.xml =================================================================== --- oozie/src/main/assemblies/client.xml (revision 0) +++ oozie/src/main/assemblies/client.xml (revision 0) @@ -0,0 +1,35 @@ + + client + + dir + tar.gz + + false + + + + ${basedir}/target/${artifact.artifactId}-${artifact.version}.jar + lib + 0644 + + + + + ${basedir}/src/main/bin + /bin + + * + + 0555 + + + + + + /lib + false + compile + 0644 + + + Index: oozie/src/main/assemblies/empty.xml =================================================================== --- oozie/src/main/assemblies/empty.xml (revision 0) +++ oozie/src/main/assemblies/empty.xml (revision 0) @@ -0,0 +1,4 @@ + + empty + + Index: oozie/src/main/assemblies/examples.xml =================================================================== --- oozie/src/main/assemblies/examples.xml (revision 0) +++ oozie/src/main/assemblies/examples.xml (revision 0) @@ -0,0 +1,73 @@ + + examples + + dir + tar.gz + + false + + + + ${basedir}/src/main/java + /examples/src + 0644 + + + ${basedir}/src/main/workflows + /examples/seed/workflows + 0644 + + + ${basedir}/src/main/input-data + /examples/seed/input-data + 0644 + + + + + + ${basedir}/src/main/bin/prepare-examples.sh + /examples/ + 0555 + + + ${basedir}/target/${artifact.artifactId}-${artifact.version}.jar + /examples/seed/lib + 0444 + + + + + + + /examples/seed/lib + false + compile + + commons-dbcp:commons-dbcp + commons-pool:commons-pool + hsqldb:hsqldb + jdom:jdom + jetty:org.mortbay.jetty + javax.servlet:jsp-api + log4j:log4j + org.apache.oozie:oozie-core + javax.servlet:servlet-api + org.apache.oozie:oozie-client + commons-cli:commons-cli + org.apache.hadoop:hadoop-core + commons-cli:commons-cli + commons-codec:commons-codec + commons-el:commons-el + commons-logging:commons-logging + commons-net:commons-net + json:json_simple + junit:junit + oro:oro + xmlenc:xmlenc + + 0444 + + + + Index: oozie/bin/mkdistro.sh =================================================================== --- oozie/bin/mkdistro.sh (revision 0) +++ oozie/bin/mkdistro.sh (revision 0) @@ -0,0 +1,103 @@ +#!/bin/sh + +# resolve links - $0 may be a softlink +PRG="${0}" + +while [ -h "${PRG}" ]; do + ls=`ls -ld "${PRG}"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "${PRG}"`/"$link" + fi +done + +BASEDIR=`dirname ${PRG}` +BASEDIR=`cd ${BASEDIR}/..;pwd` + + +if [ "$1" == "-full" ]; then + FULLDISTRO=true + shift +fi + +for arg in "$@" +do + if [[ ${arg} =~ ^\-Dhadoop\= ]]; then + HADOOPV=${arg:9} + fi + if [[ ${arg} =~ ^\-Dpig\= ]]; then + PIGV=${arg:9} + fi +done + +if [ "$HADOOPV" == "" ]; then + echo "-Dhadoop= must be specified" + exit -1 +fi + +if [ "$PIGV" == "" ]; then + echo "-Dpig= must be specified" + exit -1 +fi + +function checkExitStatus { + if [ "$?" != "0" ]; then + echo + echo "ERROR, Oozie distro could not be created ${1}" + echo + cleanUpLocalRepo + exit -1 + fi +} + +function cleanUpLocalRepo { + rm -rf ~/.m2/repository/org/apache/oozie/* +} + +export DATETIME=`date -u "+%Y.%m.%d-%H:%M:%SGMT"` +cd ${BASEDIR} +export SVNREV=`svn info | grep "Revision" | awk '{print $2}'` +export SVNURL=`svn info | grep "URL" | awk '{print $2}'` + +#clean up local repo +cleanUpLocalRepo + +MVN_OPTS="-Dbuild.time=${DATETIME} -Dsvn.revision=${SVNREV} -Dsvn.url=${SVNURL}" + +#clean, compile, test, package, install +mvn clean install ${MVN_OPTS} $* +checkExitStatus "running: clean compile, test, package, install" + +if [ "$FULLDISTRO" == "true" ]; then + #cobertura + mvn cobertura:cobertura ${MVN_OPTS} $* + checkExitStatus "running: cobertura" + + #dependencies report + mvn project-info-reports:dependencies ${MVN_OPTS} $* + checkExitStatus "running: dependencies" + + #TODO add findbugs and test reports +fi + +#javadocs +mvn javadoc:javadoc ${MVN_OPTS} $* +checkExitStatus "running: javadoc" + +cd docs +mvn clean +mvn site:site +checkExitStatus "running: docs site" +cd .. + +#putting together distro +mvn assembly:single ${MVN_OPTS} $* +checkExitStatus "running: assembly" + +cleanUpLocalRepo + +echo +echo "Oozie distro created, DATE[${DATETIME}] SVN-REV[${SVNREV}], available at [${BASEDIR}/distro/target]" +echo Property changes on: oozie/bin/mkdistro.sh ___________________________________________________________________ Added: svn:executable + * Index: oozie/bin/purgelocalrepo.sh =================================================================== --- oozie/bin/purgelocalrepo.sh (revision 0) +++ oozie/bin/purgelocalrepo.sh (revision 0) @@ -0,0 +1,9 @@ +#!/bin/sh + +function cleanUpLocalRepo { + rm -rf ~/.m2/repository/org/apache/oozie/* +} + +#clean up local repo +cleanUpLocalRepo + Property changes on: oozie/bin/purgelocalrepo.sh ___________________________________________________________________ Added: svn:executable + * Index: oozie/core/src/test/java/org/apache/oozie/test/TestEmbeddedServletContainer.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/test/TestEmbeddedServletContainer.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/test/TestEmbeddedServletContainer.java (revision 0) @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.test; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; + +public class TestEmbeddedServletContainer extends XTestCase { + + public void testEmbeddedServletContainer() throws Exception { + EmbeddedServletContainer container = new EmbeddedServletContainer("blah"); + container.addServletEndpoint("/ping/*", PingServlet.class); + try { + container.start(); + URL url = new URL(container.getServletURL("/ping/*") + "bla"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.connect(); + assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream())); + assertEquals("ping", reader.readLine()); + assertEquals(null, reader.readLine()); + + } + finally { + container.stop(); + } + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/test/PingServlet.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/test/PingServlet.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/test/PingServlet.java (revision 0) @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.test; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.Writer; + +/** + * Servlet that returns a 'ping'. Used to test the ServletTestCase + */ +public class PingServlet extends HttpServlet { + + protected void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + response.setStatus(HttpServletResponse.SC_OK); + Writer w = response.getWriter(); + w.write("ping"); + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/test/TestXTestCase.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/test/TestXTestCase.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/test/TestXTestCase.java (revision 0) @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.test; + +import junit.framework.TestCase; + +public class TestXTestCase extends TestCase { + static String SYS_PROP = "oozie.test.testProp"; + static String testBaseDir; + + protected void setUp() throws Exception { + super.setUp(); + testBaseDir = null; + } + + public void testBaseDir() throws Exception { + testBaseDir = System.getProperty(XTestCase.OOZIE_TEST_DIR); + try { + MyXTestCase testcase = new MyXTestCase(); + testcase.setName(getName()); + testcase.setUp(); + testcase.testBaseDir(); + testcase.tearDown(); + } + finally { + if (testBaseDir != null) { + System.getProperties().setProperty(XTestCase.OOZIE_TEST_DIR, testBaseDir); + } + } + } + + public void testSysPropSetting() throws Exception { + try { + System.getProperties().remove(SYS_PROP); + MyXTestCase testcase = new MyXTestCase(); + testcase.setName(getName()); + testcase.setUp(); + testcase.testUnsetSysProperty(); + assertEquals("A", System.getProperty(SYS_PROP)); + testcase.tearDown(); + assertNull(System.getProperty(SYS_PROP)); + + testcase = new MyXTestCase(); + testcase.setName(getName() + "A"); + testcase.setUp(); + System.getProperties().setProperty(SYS_PROP, "B"); + testcase.testSetSysProperty(); + assertEquals("C", System.getProperty(SYS_PROP)); + testcase.tearDown(); + assertEquals("B", System.getProperty(SYS_PROP)); + + } + finally { + System.getProperties().remove(SYS_PROP); + } + + } + + public void testWaitFor() throws Exception { + MyXTestCase testcase = new MyXTestCase(); + testcase.setName(getName()); + testcase.setUp(); + testcase.testWaitFor(); + testcase.tearDown(); + + testcase.setName(getName() + "A"); + testcase.setUp(); + testcase.testWaitForTimeOut(); + testcase.tearDown(); + } + + + public class MyXTestCase extends XTestCase { + + public void testBaseDir() { + assertTrue(TestXTestCase.testBaseDir == null || + getTestCaseDir().startsWith(TestXTestCase.testBaseDir)); + } + + public void testUnsetSysProperty() { + assertNull(System.getProperty(TestXTestCase.SYS_PROP)); + setSystemProperty(TestXTestCase.SYS_PROP, "A"); + assertEquals("A", System.getProperty(TestXTestCase.SYS_PROP)); + } + + public void testSetSysProperty() { + assertEquals("B", System.getProperty(TestXTestCase.SYS_PROP)); + setSystemProperty(TestXTestCase.SYS_PROP, "C"); + assertEquals("C", System.getProperty(TestXTestCase.SYS_PROP)); + } + + public void testWaitFor() { + long start = System.currentTimeMillis(); + long waited = waitFor(60 * 1000, new Predicate() { + public boolean evaluate() throws Exception { + return true; + } + }); + long end = System.currentTimeMillis(); + assertEquals(0, waited, 100); + assertEquals(0, end - start, 300); + } + + public void testWaitForTimeOut() { + long start = System.currentTimeMillis(); + long waited = waitFor(1000, new Predicate() { + public boolean evaluate() throws Exception { + return false; + } + }); + long end = System.currentTimeMillis(); + assertEquals(1000, waited, 100); + assertEquals(1000, end - start, 300); + } + + public void testHadoopSysProps() { + assertEquals("hdfs://localhost:9000", getNameNodeUri()); + assertEquals("localhost:9001", getJobTrackerUri()); + setSystemProperty(XTestCase.OOZIE_TEST_NAME_NODE, "hdfs://xyz:9000"); + setSystemProperty(XTestCase.OOZIE_TEST_JOB_TRACKER, "xyz:9001"); + assertEquals("hdfs://xyz:9000", getNameNodeUri()); + assertEquals("xyz:9001", getJobTrackerUri()); + } + + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/test/XTestCase.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/test/XTestCase.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/test/XTestCase.java (revision 0) @@ -0,0 +1,343 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.test; + +import junit.framework.TestCase; +import org.apache.commons.logging.LogFactory; +import org.apache.oozie.util.ParamChecker; +import org.apache.oozie.util.XLog; +import org.apache.oozie.service.DataSourceService; +import org.apache.oozie.dag.service.DBLiteWorkflowStoreService; +import org.apache.hadoop.conf.Configuration; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.DriverManager; +import java.sql.Statement; + +/** + * Base JUnit TestCase subclass used by all Oozie testcases. + *

+ * This class provides the following functionality: + *

+ *

+ *

+ * The base directory for the test working directory must be specified via the system property + * oozie.test.dir, there default value is '/tmp'. + *

+ * From within testcases, system properties must be changed using the {@link #setSystemProperty} method. + */ +public abstract class XTestCase extends TestCase { + private Map sysProps; + private String testCaseDir; + + /** + * System property to specify the parent directory for the 'oozietests' directory + * to be used as base for all test working directories. + *

+ * If this property is not set, the assumed value is '/tmp'. + */ + public static final String OOZIE_TEST_DIR = "oozie.test.dir"; + + /** + * System property to specify the Hadoop Job Tracker to use for testing. + *

+ * If this property is not set, the assumed value is 'locahost:9001'. + */ + public static final String OOZIE_TEST_JOB_TRACKER = "oozie.test.job.tracker"; + + /** + * System property to specify the Hadoop Name Node to use for testing. + *

+ * If this property is not set, the assumed value is 'locahost:9000'. + */ + public static final String OOZIE_TEST_NAME_NODE = "oozie.test.name.node"; + + /** + * Initialize the test working directory. + *

+ * If it does not exist it creates it, if it already exists it deletes all its contents. + *

+ * The test working directory it is not deleted after the test runs. + *

+ * @throws Exception if the test workflow working directory could not be created. + */ + protected void setUp() throws Exception { + super.setUp(); + String baseDir = System.getProperty(OOZIE_TEST_DIR, "/tmp"); + String msg = null; + if (!baseDir.startsWith("/")) { + msg = XLog.format("System property [{0}]=[{1}] must be set to an absolute path", OOZIE_TEST_DIR, baseDir); + } + else if (baseDir.length() < 4) { + msg = XLog.format("System property [{0}]=[{1}] path must be at least 4 chars", OOZIE_TEST_DIR, baseDir); + } + if (msg != null) { + throw new Error(msg); + } + sysProps = new HashMap(); + testCaseDir = createTestCaseDir(this, true); + if (System.getProperty("oozielocal.log") == null) { + setSystemProperty("oozielocal.log", "/tmp/oozielocal.log"); + } + } + + /** + * Clean up the test case. + */ + protected void tearDown() throws Exception { + resetSystemProperties(); + sysProps = null; + testCaseDir = null; + super.tearDown(); + } + + /** + * Return the test working directory. The directory name is the full class name of the test plus the test method + * name. + * + * @return the test working directory path, it is always an absolute path. + */ + protected String getTestCaseDir() { + return testCaseDir; + } + + /** + * Return the test working directory. + *

+ * It returns ${oozie.test.dir}/oozietests/TESTCLASSNAME/TESTMETHODNAME. + *

+ * + * @param testCase testcase instance to obtain the working directory. + * @return the test working directory. + */ + private String getTestCaseDirInternal(TestCase testCase) { + ParamChecker.notNull(testCase, "testCase"); + File dir = new File(System.getProperty(OOZIE_TEST_DIR, "/tmp")); + dir = new File(dir, "oozietests"); + dir = new File(dir, testCase.getClass().getName()); + dir = new File(dir, testCase.getName()); + return dir.getAbsolutePath(); + } + + private void delete(File file) throws IOException { + ParamChecker.notNull(file, "file"); + if (file.getAbsolutePath().length() < 5) { + throw new RuntimeException(XLog.format("path [{0}] is too short, not deleting", file.getAbsolutePath())); + } + if (file.exists()) { + if (file.isDirectory()) { + File[] children = file.listFiles(); + if (children != null) { + for (File child : children) { + delete(child); + } + } + } + if (!file.delete()) { + throw new RuntimeException(XLog.format("could not delete path [{0}]", file.getAbsolutePath())); + } + } + } + + /** + * Create the test working directory. + * + * @param testCase testcase instance to obtain the working directory. + * @param cleanup indicates if the directory should be cleaned up if it exists. + * @return return the path of the test working directory, it is always an absolute path. + * @throws Exception if the test working directory could not be created or cleaned up. + */ + private String createTestCaseDir(TestCase testCase, boolean cleanup) throws Exception { + String testCaseDir = getTestCaseDirInternal(testCase); + System.out.println(XLog.format("Setting testcase work dir[{0}]", testCaseDir)); + if (cleanup) { + delete(new File(testCaseDir)); + } + File dir = new File(testCaseDir); + if (!dir.mkdirs()) { + throw new RuntimeException(XLog.format("Could not create testcase dir[{0}]", testCaseDir)); + } + return testCaseDir; + } + + /** + * Create a Test case sub directory. + * + * @param subDirName sub directory name. + * @return the absolute path to the created directory. + */ + protected String createTestCaseSubDir(String subDirName) { + ParamChecker.notNull(subDirName, "subDirName"); + File dir = new File(testCaseDir, subDirName); + if (!dir.mkdirs()) { + throw new RuntimeException(XLog.format("Could not create testcase subdir[{0}]", dir)); + } + return dir.getAbsolutePath(); + } + + /** + * Set a system property for the duration of the method test case. + *

+ * After the test method ends the orginal value is restored. + * + * @param name system property name. + * @param value value to set. + */ + protected void setSystemProperty(String name, String value) { + ParamChecker.notNull(value, "value"); + if (!sysProps.containsKey(name)) { + String currentValue = System.getProperty(name); + sysProps.put(name, currentValue); + } + System.setProperty(name, value); + } + + /** + * Reset changed system properties to their original values. + *

+ * Called from {@link #tearDown}. + */ + private void resetSystemProperties() { + for (Map.Entry entry : sysProps.entrySet()) { + if (entry.getValue() != null) { + System.setProperty(entry.getKey(), entry.getValue()); + } + else { + System.getProperties().remove(entry.getKey()); + } + } + sysProps.clear(); + } + + /** + * A predicate 'closure' used by {@link XTestCase#waitFor} method. + */ + public static interface Predicate { + + /** + * Perform a predicate evaluation. + * + * @return the boolean result of the evaluation. + * @throws Exception thrown if the predicate evaluation could not evaluate. + */ + public boolean evaluate() throws Exception; + } + + /** + * Wait for a condition, expressed via a {@link Predicate} to become true. + * + * @param timeout maximum time in milliseconds to wait for the predicate to become true. + * @param predicate predicate waiting on. + * @return the waited time. + */ + protected long waitFor(int timeout, Predicate predicate) { + ParamChecker.notNull(predicate, "predicate"); + XLog log = new XLog(LogFactory.getLog(getClass())); + long started = System.currentTimeMillis(); + long mustEnd = System.currentTimeMillis() + timeout; + long lastEcho = 0; + try { + long waiting = mustEnd - System.currentTimeMillis(); + log.info("Waiting up to [{0}] msec", waiting); + boolean eval; + while (!(eval = predicate.evaluate()) && System.currentTimeMillis() < mustEnd) { + if ((System.currentTimeMillis() - lastEcho) > 1000) { + waiting = mustEnd - System.currentTimeMillis(); + log.info("Waiting up to [{0}] msec", waiting); + lastEcho = System.currentTimeMillis(); + } + Thread.sleep(1000); + } + if (!eval) { + log.info("Waiting timed out after [{0}] msec", timeout); + } + return System.currentTimeMillis() - started; + } + catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + /** + * Return the Hadoop Job Tracker to use for testing. + *

+ * The value is taken from the Java sytem property {@link #OOZIE_TEST_JOB_TRACKER}, if this property is not set, + * the assumed value is 'locahost:9001'. + * + * @return the job tracker URI. + */ + protected String getJobTrackerUri() { + return System.getProperty(OOZIE_TEST_JOB_TRACKER, "localhost:9001"); + } + + /** + * Return the Hadoop Name Node to use for testing. + *

+ * The value is taken from the Java sytem property {@link #OOZIE_TEST_NAME_NODE}, if this property is not set, + * the assumed value is 'locahost:9000'. + * + * @return the name node URI. + */ + protected String getNameNodeUri() { + return System.getProperty(OOZIE_TEST_NAME_NODE, "hdfs://localhost:9000"); + } + + private Connection getConnection(Configuration conf) throws SQLException { + String driver = conf.get(DataSourceService.CONF_DRIVER, "org.hsqldb.jdbcDriver"); + String url = conf.get(DataSourceService.CONF_URL, "jdbc:hsqldb:mem:testdb"); + String user = conf.get(DataSourceService.CONF_USERNAME, "sa"); + String password = conf.get(DataSourceService.CONF_PASSWORD, "").trim(); + try { + Class.forName(driver); + } + catch (ClassNotFoundException ex) { + throw new RuntimeException(ex); + } + return DriverManager.getConnection(url, user, password); + } + + //TODO Fix this + protected void cleanUpDB(Configuration conf) throws Exception { + String dbName = conf.get(DBLiteWorkflowStoreService.CONF_SCHEMA_NAME); + Connection conn = getConnection(conf); + Statement st = conn.createStatement(); + try { + st.executeUpdate("DROP SCHEMA " + dbName + " CASCADE"); + } + catch (SQLException ex) { + try { + st.executeUpdate("DROP DATABASE " + dbName); + } + catch (SQLException ex1) { + // nop + } + } + st.close(); + conn.close(); + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/test/TestXFsTestCase.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/test/TestXFsTestCase.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/test/TestXFsTestCase.java (revision 0) @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.test; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + + +public class TestXFsTestCase extends XFsTestCase { + + public void testFsDir() throws Exception { + assertNotNull(getFsTestCaseDir()); + assertNotNull(getFileSystem()); + + String testDir = getTestCaseDir(); + String nameNode = getNameNodeUri(); + String user = System.getProperty("user.name"); + Path fsTestDir = getFsTestCaseDir(); + + assertTrue(fsTestDir.toString().startsWith(nameNode)); + assertTrue(fsTestDir.toString().contains(user + testDir)); + + FileSystem fs = getFileSystem(); + assertTrue(fs.getUri().toString().startsWith(getNameNodeUri())); + + assertTrue(fs.exists(fsTestDir)); + assertTrue(fs.listStatus(fsTestDir).length == 0); + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/test/XFsTestCase.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/test/XFsTestCase.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/test/XFsTestCase.java (revision 0) @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.test; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.oozie.dag.service.WorkflowAppService; +import org.apache.oozie.util.XLog; + +import java.io.IOException; +import java.net.URI; + +/** + * Base JUnit TestCase subclass used by all Oozie testcases that need Hadoop FS access. + *

+ * As part of its setup, this testcase class creates a unique test working directory per test method in the FS. + *

+ * The URI of the FS namenode must be specified via the {@link XTestCase#OOZIE_TEST_NAME_NODE} system property. The + * default value is 'hdfs://localhost:9000'. + * + * The test working directory is created in the specified FS URI, under the current user name home directory, under + * the subdirectory name specified wit the system property {@link XTestCase#OOZIE_TEST_DIR}. The default value is + * '/tmp'. + *

+ * The path of the test working directory is: + * '$FS_URI/user/$USER/$OOZIE_TEST_DIR/oozietest/$TEST_CASE_CLASS/$TEST_CASE_METHOD/' + *

+ * For example: 'hdfs://localhost:9000/user/tucu/tmp/oozietest/org.apache.oozie.service.TestELService/testEL/' + * + */ +public abstract class XFsTestCase extends XTestCase { + private FileSystem fileSystem; + private Path fsTestDir; + + /** + * Set up the testcase. + * + * @throws Exception thrown if the test case could no be set up. + */ + protected void setUp() throws Exception { + super.setUp(); + Configuration conf = new Configuration(); + conf.set(WorkflowAppService.HADOOP_USER, System.getProperty("user.name")); + conf.set(WorkflowAppService.HADOOP_UGI, System.getProperty("user.name") + ",other"); + fileSystem = FileSystem.get(new URI(getNameNodeUri()), conf); + Path path = new Path(fileSystem.getWorkingDirectory(), getTestCaseDir().substring(1)); + fsTestDir = fileSystem.makeQualified(path); + System.out.println(XLog.format("Setting FS testcase work dir[{0}]", fsTestDir)); + fileSystem.delete(fsTestDir, true); + if (!fileSystem.mkdirs(path)) { + throw new IOException(XLog.format("Could not create FS testcase dir [{0}]", fsTestDir)); + } + } + + /** + * Tear down the testcase. + */ + protected void tearDown() throws Exception { + fileSystem = null; + fsTestDir = null; + super.tearDown(); + } + + /** + * Return the file system used by the tescase. + * + * @return the file system used by the tescase. + */ + protected FileSystem getFileSystem() { + return fileSystem; + } + + /** + * Return the FS test working directory. The directory name is the full class name of the test plus the test method + * name. + * + * @return the test working directory path, it is always an full and absolute path. + */ + protected Path getFsTestCaseDir() { + return fsTestDir; + } + +} Index: oozie/core/src/test/java/org/apache/oozie/service/TestSchedulerService.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/service/TestSchedulerService.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/service/TestSchedulerService.java (revision 0) @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.service; + +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.IOUtils; + +import java.io.File; +import java.io.FileOutputStream; +import java.util.concurrent.atomic.AtomicInteger; + +public class TestSchedulerService extends XTestCase { + + public void testInstrumentation() throws Exception { + Services services = new Services(); + services.init(); + assertNotNull(services.get(SchedulerService.class)); + SchedulerService ss = services.get(SchedulerService.class); + final AtomicInteger counter = new AtomicInteger(); + ss.schedule(new Runnable() { + public void run() { + counter.incrementAndGet(); + } + }, 0, 1, SchedulerService.Unit.SEC); + + waitFor(2 * 1000, new Predicate() { + public boolean evaluate() throws Exception { + return false; + } + }); + assertTrue(counter.get() > 1); + services.destroy(); + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/service/TestAuthorizationService.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/service/TestAuthorizationService.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/service/TestAuthorizationService.java (revision 0) @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.service; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.oozie.client.WorkflowClient; +import org.apache.oozie.dag.DagEngine; +import org.apache.oozie.dag.ForTestingActionExecutor; +import org.apache.oozie.dag.service.ActionService; +import org.apache.oozie.dag.service.WorkflowSchemaService; +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.IOUtils; +import org.apache.oozie.util.XConfiguration; +import org.apache.oozie.util.XLog; + +import java.io.FileWriter; +import java.io.IOException; +import java.io.Reader; +import java.io.Writer; +import java.net.URI; + +/** + * Tests the authorization service. + */ +public class TestAuthorizationService extends XTestCase { + + private Services services; + + @Override + public void setUp() throws Exception { + super.setUp(); + setSystemProperty(WorkflowSchemaService.CONF_EXT_SCHEMAS, "wf-ext-schema.xsd"); + + Reader adminListReader = IOUtils.getResourceAsReader("adminusers.txt", -1); + Writer adminListWriter = new FileWriter(getTestCaseDir() + "/adminusers.txt"); + IOUtils.copyCharStream(adminListReader, adminListWriter); + + Reader logPropReader = IOUtils.getResourceAsReader("oozie-log4j.properties", -1); + Writer logPropWriter = new FileWriter(getTestCaseDir() + "/oozie-log4j.properties"); + IOUtils.copyCharStream(logPropReader, logPropWriter); + + services = new Services(); + services.init(); + setSystemProperty(ConfigurationService.CONFIG_PATH, getTestCaseDir()); + services.getConf().setBoolean(AuthorizationService.CONF_SECURITY_ENABLED, true); + services.get(AuthorizationService.class).init(services); + services.get(ActionService.class).register(ForTestingActionExecutor.class); + } + + public void tearDown() { + services.destroy(); + } + + /** + * Tests the Authorization Service API. + */ + public void testAuthorizationService() throws Exception { + + Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1); + Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); + IOUtils.copyCharStream(reader, writer); + + final DagEngine engine = new DagEngine("u", "g", "a"); + Configuration jobConf = new XConfiguration(); + jobConf.set(WorkflowClient.APP_PATH, getTestCaseDir()); + jobConf.set(WorkflowClient.USER_NAME, "u"); + jobConf.set(WorkflowClient.GROUP_NAME, "g"); + jobConf.set(WorkflowClient.LOG_TOKEN, "t"); + + jobConf.set("external-status", "ok"); + jobConf.set("signal-value", "based_on_action_status"); + + final String jobId = engine.submitJob(jobConf, true); + + Configuration conf = new Configuration(); + conf.set("hadoop.job.ugi", System.getProperty("user.name") + "," + "others"); + FileSystem fileSystem = FileSystem.get(new URI(getNameNodeUri()), conf); + Path path = new Path(fileSystem.getWorkingDirectory(), getTestCaseDir().substring(1)); + Path fsTestDir = fileSystem.makeQualified(path); + System.out.println(XLog.format("Setting FS testcase work dir[{0}]", fsTestDir)); + fileSystem.delete(fsTestDir, true); + if (!fileSystem.mkdirs(path)) { + throw new IOException(XLog.format("Could not create FS testcase dir [{0}]", fsTestDir)); + } + + String appPath = fsTestDir.toString() + "/app"; + + Path jobXmlPath = new Path(appPath, "workflow.xml"); + fileSystem.create(jobXmlPath); + FsPermission permissions = new FsPermission(FsAction.READ_WRITE, FsAction.READ, FsAction.NONE); + fileSystem.setPermission(jobXmlPath, permissions); + + AuthorizationService as = services.get(AuthorizationService.class); + assertNotNull(as); + assertTrue(as.verifyUserGroup("u", "g")); + + assertTrue(as.authorizeForAdmin("admin", "g", false)); + assertTrue(as.authorizeForAdmin("admin", "g", true)); + assertFalse(as.authorizeForAdmin("u", "g", true)); + assertFalse(as.authorizeForAdmin("u", "g", false)); + + assertFalse(as.authorizeForApp("u", "g", appPath)); + assertTrue(as.authorizeForApp(System.getProperty("user.name"), "others", appPath)); + + assertTrue(as.authorizeForJob("u", "g", "j", false)); + assertTrue(as.authorizeForJob("u", "g", jobId, true)); + assertFalse(as.authorizeForJob("blah", "g", jobId, true)); + services.destroy(); + } +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/service/TestELService.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/service/TestELService.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/service/TestELService.java (revision 0) @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.service; + +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.ELEvaluator; + +public class TestELService extends XTestCase { + + public void testEL() throws Exception { + Services services = new Services(); + services.init(); + assertNotNull(services.get(ELService.class)); + ELEvaluator eval = services.get(ELService.class).createEvaluator(); + assertNotNull(eval.evaluate("${KB}", Long.class)); + assertNotNull(eval.evaluate("${MB}", Long.class)); + assertNotNull(eval.evaluate("${GB}", Long.class)); + assertNotNull(eval.evaluate("${TB}", Long.class)); + assertNotNull(eval.evaluate("${PB}", Long.class)); + assertNotNull(eval.evaluate("${trim(' ')}", String.class)); + assertNotNull(eval.evaluate("${concat('a', 'b')}", String.class)); + assertNotNull(eval.evaluate("${firstNotNull(null, 'b')}", String.class)); + assertNotNull(eval.evaluate("${timestamp()}", String.class)); + assertNotNull(eval.evaluate("${urlEncode('abc')}", String.class)); + services.destroy(); + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/service/TestServices.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/service/TestServices.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/service/TestServices.java (revision 0) @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.service; + +import org.apache.oozie.test.XTestCase; + +import java.io.File; + +public class TestServices extends XTestCase { + + public void testDefaultServices() throws Exception { + setSystemProperty(ConfigurationService.CONFIG_FILE, "oozie-site1.xml"); + Services services = new Services(); + services.init(); + assertNotNull(services.get(XLogService.class)); + assertNotNull(services.get(ConfigurationService.class)); + + assertEquals("oozie-" + System.getProperty("user.name"), services.getSystemId()); + assertNotNull(services.getRuntimeDir()); + assertTrue(new File(services.getRuntimeDir()).exists()); + + services.destroy(); + } + +} Index: oozie/core/src/test/java/org/apache/oozie/service/TestInstrumentationService.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/service/TestInstrumentationService.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/service/TestInstrumentationService.java (revision 0) @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.service; + +import org.apache.oozie.test.XTestCase; + +public class TestInstrumentationService extends XTestCase { + + public void testInstrumentation() throws Exception { + Services services = new Services(); + services.init(); + assertNotNull(services.get(InstrumentationService.class)); + assertNotNull(services.get(InstrumentationService.class).get()); + services.destroy(); + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/service/TestCallableQueueService.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/service/TestCallableQueueService.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/service/TestCallableQueueService.java (revision 0) @@ -0,0 +1,159 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.service; + +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.XCallable; + +import java.util.Arrays; +import java.util.concurrent.atomic.AtomicLong; + +public class TestCallableQueueService extends XTestCase { + static AtomicLong EXEC_ORDER = new AtomicLong(); + + public class MyCallable implements XCallable { + int priority; + long executed = 0; + int wait; + long order; + + public MyCallable() { + this(0, 0); + } + + public String getName() { + return "myCallable"; + } + + public MyCallable(int priority, int wait) { + this.priority = priority; + this.wait = wait; + } + + public int getPriority() { + return 0; + } + + public Void call() throws Exception { + order = EXEC_ORDER.getAndIncrement(); + Thread.sleep(wait); + executed = System.currentTimeMillis(); + return null; + } + + } + + public void testQueuing() throws Exception { + Services services = new Services(); + services.init(); + + CallableQueueService queueservice = services.get(CallableQueueService.class); + + final MyCallable callable = new MyCallable(); + queueservice.queue(callable); + waitFor(1000, new Predicate() { + public boolean evaluate() throws Exception { + return callable.executed != 0; + } + }); + assertTrue(callable.executed != 0); + + services.destroy(); + + } + + public void testDelayedQueuing() throws Exception { + Services services = new Services(); + services.init(); + + CallableQueueService queueservice = services.get(CallableQueueService.class); + + final MyCallable callable = new MyCallable(); + long scheduled = System.currentTimeMillis(); + queueservice.queue(callable, 1000); + waitFor(3000, new Predicate() { + public boolean evaluate() throws Exception { + return callable.executed != 0; + } + }); + assertTrue(callable.executed >= scheduled + 1000); + + services.destroy(); + } + + public void testPriorityExecution() throws Exception { + EXEC_ORDER = new AtomicLong(); + setSystemProperty(CallableQueueService.CONF_THREADS, "1"); + Services services = new Services(); + services.init(); + + CallableQueueService queueservice = services.get(CallableQueueService.class); + + final MyCallable callable1 = new MyCallable(0, 200); + final MyCallable callable2 = new MyCallable(0, 200); + final MyCallable callable3 = new MyCallable(0, 200); + final MyCallable callableLow = new MyCallable(); + final MyCallable callableHigh = new MyCallable(1, 10); + + queueservice.queue(callable1); + queueservice.queue(callable2); + queueservice.queue(callable3); + queueservice.queue(callableLow); + queueservice.queue(callableHigh); + + waitFor(3000, new Predicate() { + public boolean evaluate() throws Exception { + return callable1.executed != 0 && callable2.executed != 0 && callable3.executed != 0 && + callableLow.executed != 0 && callableHigh.executed != 0; + } + }); + assertTrue(callable1.executed >= 0); + assertTrue(callable2.executed >= 0); + assertTrue(callable3.executed >= 0); + assertTrue(callableLow.executed >= 0); + assertTrue(callableHigh.executed >= 0); + assertTrue(callableHigh.order < callableLow.order); + + services.destroy(); + + } + + public void testQueueSerial() throws Exception { + EXEC_ORDER = new AtomicLong(); + Services services = new Services(); + services.init(); + final MyCallable callable1 = new MyCallable(0, 10); + final MyCallable callable2 = new MyCallable(0, 10); + final MyCallable callable3 = new MyCallable(0, 10); + + CallableQueueService queueservice = services.get(CallableQueueService.class); + + queueservice.queueSerial(Arrays.asList(callable1, callable2, callable3)); + waitFor(100, new Predicate() { + public boolean evaluate() throws Exception { + return callable1.executed != 0 && callable2.executed != 0 && callable3.executed != 0; + } + }); + assertEquals(0, callable1.order); + assertEquals(1, callable2.order); + assertEquals(2, callable3.order); + + services.destroy(); + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/service/TestMemoryLocksService.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/service/TestMemoryLocksService.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/service/TestMemoryLocksService.java (revision 0) @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.service; + +import org.apache.oozie.test.XTestCase; + +public class TestMemoryLocksService extends XTestCase { + + public void testService() throws Exception { + new Services().init(); + assertNotNull(Services.get().get(MemoryLocksService.class)); + Services.get().destroy(); + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/service/TestUUIDService.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/service/TestUUIDService.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/service/TestUUIDService.java (revision 0) @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.service; + +import org.apache.oozie.test.XTestCase; + +public class TestUUIDService extends XTestCase { + + public void testConfiguration() throws Exception { + setSystemProperty(UUIDService.CONF_GENERATOR, "counter"); + Services services = new Services(); + services.init(); + services.destroy(); + + setSystemProperty(UUIDService.CONF_GENERATOR, "random"); + services = new Services(); + services.init(); + services.destroy(); + + try { + setSystemProperty(UUIDService.CONF_GENERATOR, "x"); + services = new Services(); + services.init(); + services.destroy(); + fail(); + } + catch (ServiceException ex) { + //nop + } + } + + public void testChildId() throws Exception { + setSystemProperty(UUIDService.CONF_GENERATOR, "counter"); + Services services = new Services(); + services.init(); + UUIDService uuid = services.get(UUIDService.class); + String id = uuid.generateId(); + String childId = uuid.generateChildId(id, "a"); + assertEquals(id, uuid.getId(childId)); + assertEquals("a", uuid.getChildName(childId)); + services.destroy(); + + setSystemProperty(UUIDService.CONF_GENERATOR, "random"); + services = new Services(); + services.init(); + uuid = services.get(UUIDService.class); + id = uuid.generateId(); + childId = uuid.generateChildId(id, "a"); + assertEquals(id, uuid.getId(childId)); + assertEquals("a", uuid.getChildName(childId)); + services.destroy(); + } + +} Index: oozie/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java (revision 0) @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.service; + +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.IOUtils; + +import java.io.File; +import java.io.FileOutputStream; + +public class TestConfigurationService extends XTestCase { + + public void testOriginalDefault() throws Exception { + ConfigurationService cl = new ConfigurationService(); + cl.init(null); + assertNotNull(cl.getConf().get("oozie.safemode")); + cl.destroy(); + } + + public void testDefault() throws Exception { + ConfigurationService cl = new ConfigurationService(); + ConfigurationService.testingDefaultFile = true; + cl.init(null); + assertEquals("DEFAULT", cl.getConf().get("oozie.dummy")); + cl.destroy(); + } + + public void testSiteFromClasspath() throws Exception { + setSystemProperty(ConfigurationService.CONFIG_FILE, "oozie-site1.xml"); + ConfigurationService cl = new ConfigurationService(); + ConfigurationService.testingDefaultFile = true; + cl.init(null); + assertEquals("SITE1", cl.getConf().get("oozie.dummy")); + cl.destroy(); + } + + public void testSiteFromDir() throws Exception { + String dir = getTestCaseDir(); + IOUtils.copyStream(IOUtils.getResourceAsStream("test-oozie-log4j.properties", -1), + new FileOutputStream(new File(dir, "test-oozie-log4j.properties"))); + setSystemProperty(ConfigurationService.CONFIG_FILE, "oozie-site.xml"); + IOUtils.copyStream(IOUtils.getResourceAsStream("oozie-site2.xml", -1), + new FileOutputStream(new File(dir, "oozie-site.xml"))); + setSystemProperty(ConfigurationService.CONFIG_PATH, dir); + ConfigurationService cl = new ConfigurationService(); + ConfigurationService.testingDefaultFile = true; + cl.init(null); + assertEquals("SITE2", cl.getConf().get("oozie.dummy")); + cl.destroy(); + } + + public void testCustomSiteFromDir() throws Exception { + String dir = getTestCaseDir(); + IOUtils.copyStream(IOUtils.getResourceAsStream("test-oozie-log4j.properties", -1), + new FileOutputStream(new File(dir, "test-oozie-log4j.properties"))); + IOUtils.copyStream(IOUtils.getResourceAsStream("oozie-site2.xml", -1), + new FileOutputStream(new File(dir, "oozie-site2.xml"))); + setSystemProperty(ConfigurationService.CONFIG_PATH, dir); + setSystemProperty(ConfigurationService.CONFIG_FILE, "oozie-site2.xml"); + ConfigurationService cl = new ConfigurationService(); + ConfigurationService.testingDefaultFile = true; + cl.init(null); + assertEquals("SITE2", cl.getConf().get("oozie.dummy")); + cl.destroy(); + } + + public void testSysPropOverride() throws Exception { + setSystemProperty("oozie.dummy", "OVERRIDE"); + ConfigurationService cl = new ConfigurationService(); + ConfigurationService.testingDefaultFile = true; + cl.init(null); + assertEquals("OVERRIDE", cl.getConf().get("oozie.dummy")); + cl.destroy(); + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/service/TestXLogService.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/service/TestXLogService.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/service/TestXLogService.java (revision 0) @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.service; + +import junit.framework.Assert; +import org.apache.commons.logging.LogFactory; +import org.apache.log4j.LogManager; +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.IOUtils; +import org.apache.oozie.util.XLog; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; + +public class TestXLogService extends XTestCase { + + public void setUp() throws Exception { + super.setUp(); + LogFactory.getFactory().release(); + LogManager.resetConfiguration(); + } + + public void tearDown() throws Exception { + LogFactory.getFactory().release(); + LogManager.resetConfiguration(); + super.tearDown(); + } + + public void testDefaultFileViaClassLoader() throws Exception { + XLogService ls = new XLogService(); + XLogService.testingDefaultFile = true; + ls.init(null); + Assert.assertTrue(LogFactory.getLog("a").isTraceEnabled()); + ls.destroy(); + } + + public void testCustomFileViaClassLoader() throws Exception { + setSystemProperty(XLogService.LOG4J_FILE, "test-custom-log4j.properties"); + XLogService ls = new XLogService(); + XLogService.testingDefaultFile = true; + ls.init(null); + Assert.assertFalse(LogFactory.getLog("a").isTraceEnabled()); + ls.destroy(); + } + + public void testDefaultFileViaConfigPath() throws Exception { + File file = new File(getTestCaseDir(), "test-oozie-log4j.properties"); + InputStream is = Thread.currentThread().getContextClassLoader(). + getResourceAsStream("test-oozie-log4j.properties"); + IOUtils.copyStream(is, new FileOutputStream(file)); + setSystemProperty(ConfigurationService.CONFIG_PATH, file.getParent()); + XLogService ls = new XLogService(); + XLogService.testingDefaultFile = true; + ls.init(null); + Assert.assertTrue(LogFactory.getLog("a").isTraceEnabled()); + ls.destroy(); + } + + public void testCustomFileViaConfigPath() throws Exception { + File file = new File(getTestCaseDir(), "test-custom-log4j.properties"); + InputStream is = Thread.currentThread().getContextClassLoader(). + getResourceAsStream("test-custom-log4j.properties"); + IOUtils.copyStream(is, new FileOutputStream(file)); + setSystemProperty(ConfigurationService.CONFIG_PATH, file.getParent()); + setSystemProperty(XLogService.LOG4J_FILE, "test-custom-log4j.properties"); + XLogService ls = new XLogService(); + XLogService.testingDefaultFile = true; + ls.init(null); + Assert.assertFalse(LogFactory.getLog("a").isTraceEnabled()); + ls.destroy(); + } + + public void _testReload() throws Exception { + File file = new File(getTestCaseDir(), "reload-log4j.properties"); + + InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream( + "test-oozie-log4j.properties"); + IOUtils.copyStream(is, new FileOutputStream(file)); + + setSystemProperty(ConfigurationService.CONFIG_PATH, getTestCaseDir()); + setSystemProperty(XLogService.RELOAD_INTERVAL, "1"); + setSystemProperty(XLogService.LOG4J_FILE, "reload-log4j.properties"); + XLogService ls = new XLogService(); + XLogService.testingDefaultFile = true; + ls.init(null); + assertTrue(LogFactory.getLog("a").isTraceEnabled()); + + is = Thread.currentThread().getContextClassLoader().getResourceAsStream("test-custom-log4j.properties"); + IOUtils.copyStream(is, new FileOutputStream(file)); + + waitFor(5 * 1000, new Predicate() { + public boolean evaluate() throws Exception { + return !LogFactory.getLog("a").isTraceEnabled(); + } + }); + + assertFalse(LogFactory.getLog("a").isTraceEnabled()); + + ls.destroy(); + } + + public void testInfoParameters() throws Exception { + XLogService ls = new XLogService(); + ls.init(null); + assertEquals("USER[-] GROUP[-]", XLog.Info.get().createPrefix()); + ls.destroy(); + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/service/TestDataSourceService.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/service/TestDataSourceService.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/service/TestDataSourceService.java (revision 0) @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.service; + +import org.apache.oozie.test.XTestCase; + +import javax.sql.DataSource; +import java.sql.Connection; + +public class TestDataSourceService extends XTestCase { + + public void testDataSource() throws Exception { + Services services = new Services(); + services.init(); + assertNotNull(services.get(DataSourceService.class)); + services.destroy(); + } + + public void testRawConnection() throws Exception { + Services services = new Services(); + services.init(); + assertNotNull(services.get(DataSourceService.class)); + Connection conn = services.get(DataSourceService.class).getRawConnection(); + assertNotNull(conn); + conn.close(); + services.destroy(); + } + + public void testManageConnection() throws Exception { + Services services = new Services(); + services.init(); + assertNotNull(services.get(DataSourceService.class)); + Connection conn = services.get(DataSourceService.class).getConnection(); + assertNotNull(conn); + conn.close(); + services.destroy(); + } +} Index: oozie/core/src/test/java/org/apache/oozie/servlet/MyJsonRestServlet.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/servlet/MyJsonRestServlet.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/servlet/MyJsonRestServlet.java (revision 0) @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.servlet; + +import org.json.simple.JSONObject; +import org.json.simple.JSONArray; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; + +public class MyJsonRestServlet extends JsonRestServlet { + static ResourceInfo[] EMPTY = new ResourceInfo[0]; + + static ResourceInfo[] NO_RESOURCE_NO_PARAMS = { + new ResourceInfo("", Arrays.asList("GET"), Collections.EMPTY_LIST)}; + + static ResourceInfo[] PARAMS_REQUIRED = { + new ResourceInfo("", Arrays.asList("GET"), Arrays.asList( + new ParameterInfo("required", Boolean.class, true, Arrays.asList("GET")), + new ParameterInfo("optional", Boolean.class, false, Arrays.asList("GET"))))}; + + static ResourceInfo[] PARAM_TYPES = { + new ResourceInfo("", Arrays.asList("GET"), Arrays.asList( + new ParameterInfo("boolean", Boolean.class, false, Arrays.asList("GET")), + new ParameterInfo("integer", Integer.class, false, Arrays.asList("GET")), + new ParameterInfo("string", String.class, false, Arrays.asList("GET"))))}; + + static ResourceInfo[] RESOURCE_GET_POST_PARAM_GET = { + new ResourceInfo("", Arrays.asList("GET", "POST"), Arrays.asList( + new ParameterInfo("param", Boolean.class, true, Arrays.asList("GET"))))}; + + static ResourceInfo[] FIXED_RESOURCE = { + new ResourceInfo("resource", Arrays.asList("GET"), Collections.EMPTY_LIST)}; + + static ResourceInfo[] WILDCARD_RESOURCE = { + new ResourceInfo("*", Arrays.asList("GET"), Collections.EMPTY_LIST)}; + + static ResourceInfo[] MULTIPLE_RESOURCES = { + new ResourceInfo("resource1", Arrays.asList("GET"), Collections.EMPTY_LIST), + new ResourceInfo("resource2", Arrays.asList("POST"), Collections.EMPTY_LIST)}; + + static ResourceInfo[] MULTIPLE_RESOURCES_NO_RESOURCE = { + new ResourceInfo("resource1", Arrays.asList("GET"), Collections.EMPTY_LIST), + new ResourceInfo("resource2", Arrays.asList("GET"), Collections.EMPTY_LIST), + new ResourceInfo("", Arrays.asList("POST"), Collections.EMPTY_LIST), + }; + + static ResourceInfo[] MULTIPLE_RESOURCES_WILDCARD = { + new ResourceInfo("resource1", Arrays.asList("GET"), Collections.EMPTY_LIST), + new ResourceInfo("resource2", Arrays.asList("GET"), Collections.EMPTY_LIST), + new ResourceInfo("*", Arrays.asList("POST"), Collections.EMPTY_LIST), + }; + + static ResourceInfo[] CONTENT_TYPE_JSON_CRON_TEST = { + new ResourceInfo("", Arrays.asList("GET"), + Arrays.asList(new ParameterInfo("json", String.class, true, Arrays.asList("GET"))))}; + + static ResourceInfo[] ACTIVE = NO_RESOURCE_NO_PARAMS; + + public MyJsonRestServlet() { + super("my", ACTIVE); + } + + protected void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + if (ACTIVE != CONTENT_TYPE_JSON_CRON_TEST) { + response.setStatus(HttpServletResponse.SC_OK); + } + else { + try { + stopCron(); + validateContentType(request, "application/xml"); + } + finally { + startCron(); + } + if (request.getParameter("json").equals("object")) { + JSONObject json = new JSONObject(); + json.put("a", "object"); + sendJsonResponse(response, HttpServletResponse.SC_OK, json); + } + else + if (request.getParameter("json").equals("array")) { + JSONArray json = new JSONArray(); + json.add("array"); + sendJsonResponse(response, HttpServletResponse.SC_OK, json); + } + } + } + + protected void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + response.setStatus(HttpServletResponse.SC_OK); + } + +} Index: oozie/core/src/test/java/org/apache/oozie/servlet/TestJsonRestServlet.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/servlet/TestJsonRestServlet.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/servlet/TestJsonRestServlet.java (revision 0) @@ -0,0 +1,246 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.servlet; + +import org.apache.oozie.test.EmbeddedServletContainer; +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.service.Services; + +import javax.servlet.http.HttpServletResponse; +import java.net.URL; +import java.net.HttpURLConnection; +import java.util.concurrent.Callable; +import java.io.BufferedReader; +import java.io.InputStreamReader; + +public class TestJsonRestServlet extends XTestCase { + + static { + new MyJsonRestServlet(); + } + + EmbeddedServletContainer container; + + private int invoke(String method, String resource, String queryString) throws Exception { + return invoke(method, resource, queryString, "dummy"); + } + + private int invoke(String method, String resource, String queryString, String contentType) throws Exception { + String s = container.getServletURL("/dummy"); + if (resource != null) { + s += resource; + } + if (queryString != null) { + s += "?" + queryString; + } + HttpURLConnection conn = (HttpURLConnection) new URL(s).openConnection(); + conn.setRequestProperty("content-type", contentType); + conn.setRequestMethod(method); + conn.connect(); + return conn.getResponseCode(); + } + + private String invokeAndGetResponse(String method, String resource, String queryString, String contentType) + throws Exception { + String s = container.getServletURL("/dummy"); + if (resource != null) { + s += resource; + } + if (queryString != null) { + s += "?" + queryString; + } + HttpURLConnection conn = (HttpURLConnection) new URL(s).openConnection(); + conn.setRequestProperty("content-type", contentType); + conn.setRequestMethod(method); + conn.connect(); + StringBuilder sb = new StringBuilder(); + BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream())); + String line = reader.readLine(); + while (line != null) { + sb.append(line); + line = reader.readLine(); + } + return sb.toString(); + } + + private void runTest(JsonRestServlet.ResourceInfo[] resourceInfo, Callable assertions) throws Exception { + container = new EmbeddedServletContainer("test"); + Services services = new Services(); + try { + services.init(); + MyJsonRestServlet.ACTIVE = resourceInfo; + container.addServletEndpoint("/dummy/*", MyJsonRestServlet.class); + container.start(); + assertions.call(); + } + finally { + container.stop(); + services.destroy(); + } + } + + public void testEmptyResources() { + try { + MyJsonRestServlet.ACTIVE = MyJsonRestServlet.EMPTY; + new MyJsonRestServlet(); + fail(); + } + catch (IllegalArgumentException ex) { + //nop + } + } + + public void testNoResourceNoParams() throws Exception { + runTest(MyJsonRestServlet.NO_RESOURCE_NO_PARAMS, new Callable() { + public Void call() throws Exception { + assertEquals(HttpServletResponse.SC_OK, invoke("GET", null, null)); + assertEquals(HttpServletResponse.SC_OK, invoke("GET", "", null)); + assertEquals(HttpServletResponse.SC_OK, invoke("GET", "", "a=A")); + assertEquals(HttpServletResponse.SC_OK, invoke("GET", "/", null)); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, invoke("GET", "/hello", null)); + return null; + } + }); + } + + public void testParamsRequired() throws Exception { + runTest(MyJsonRestServlet.PARAMS_REQUIRED, new Callable() { + public Void call() throws Exception { + assertEquals(HttpServletResponse.SC_OK, invoke("GET", "", "required=true")); + assertEquals(HttpServletResponse.SC_OK, invoke("GET", "", "required=true&optional=true")); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, invoke("GET", "", "optional=true")); + return null; + } + }); + } + + public void testParamTypes() throws Exception { + runTest(MyJsonRestServlet.PARAM_TYPES, new Callable() { + public Void call() throws Exception { + assertEquals(HttpServletResponse.SC_OK, invoke("GET", "", "boolean=true")); + assertEquals(HttpServletResponse.SC_OK, invoke("GET", "", "boolean=false")); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, invoke("GET", "", "boolean=x")); + assertEquals(HttpServletResponse.SC_OK, invoke("GET", "", "integer=1")); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, invoke("GET", "", "integer=x")); + assertEquals(HttpServletResponse.SC_OK, invoke("GET", "", "string=a")); + return null; + } + }); + } + + public void testResourceGetPostParamGet() throws Exception { + runTest(MyJsonRestServlet.RESOURCE_GET_POST_PARAM_GET, new Callable() { + public Void call() throws Exception { + assertEquals(HttpServletResponse.SC_OK, invoke("GET", "", "param=true")); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, invoke("POST", "", "param=true")); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, invoke("GET", "", "")); + assertEquals(HttpServletResponse.SC_OK, invoke("POST", "", "")); + return null; + } + }); + } + + public void testFixedResource() throws Exception { + runTest(MyJsonRestServlet.FIXED_RESOURCE, new Callable() { + public Void call() throws Exception { + assertEquals(HttpServletResponse.SC_BAD_REQUEST, invoke("GET", "", "")); + assertEquals(HttpServletResponse.SC_OK, invoke("GET", "/resource", "")); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, invoke("POST", "/resource", "")); + return null; + } + }); + } + + public void testWildCardResource() throws Exception { + runTest(MyJsonRestServlet.WILDCARD_RESOURCE, new Callable() { + public Void call() throws Exception { + assertEquals(HttpServletResponse.SC_OK, invoke("GET", "/any", "")); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, invoke("POST", "/any", "")); + return null; + } + }); + } + + public void testInvalidResource() throws Exception { + runTest(MyJsonRestServlet.WILDCARD_RESOURCE, new Callable() { + public Void call() throws Exception { + assertEquals(HttpServletResponse.SC_BAD_REQUEST, invoke("GET", "/any/any", "")); + return null; + } + }); + } + + + public void testMultipleResources() throws Exception { + runTest(MyJsonRestServlet.MULTIPLE_RESOURCES, new Callable() { + public Void call() throws Exception { + assertEquals(HttpServletResponse.SC_OK, invoke("GET", "/resource1", "")); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, invoke("POST", "/resource1", "")); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, invoke("GET", "/resource2", "")); + assertEquals(HttpServletResponse.SC_OK, invoke("POST", "/resource2", "")); + return null; + } + }); + } + + public void testMultipleResourcesNoResource() throws Exception { + runTest(MyJsonRestServlet.MULTIPLE_RESOURCES_NO_RESOURCE, new Callable() { + public Void call() throws Exception { + assertEquals(HttpServletResponse.SC_OK, invoke("GET", "/resource1", "")); + assertEquals(HttpServletResponse.SC_OK, invoke("GET", "/resource2", "")); + assertEquals(HttpServletResponse.SC_OK, invoke("POST", "", "")); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, invoke("POST", "/resource1", "")); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, invoke("POST", "/resource2", "")); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, invoke("GET", "", "")); + return null; + } + }); + } + + public void testMultipleResourcesWildCard() throws Exception { + runTest(MyJsonRestServlet.MULTIPLE_RESOURCES_WILDCARD, new Callable() { + public Void call() throws Exception { + assertEquals(HttpServletResponse.SC_OK, invoke("GET", "/resource1", "")); + assertEquals(HttpServletResponse.SC_OK, invoke("GET", "/resource2", "")); + assertEquals(HttpServletResponse.SC_OK, invoke("POST", "/any", "")); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, invoke("POST", "/resource1", "")); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, invoke("POST", "/resource2", "")); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, invoke("GET", "/any", "")); + return null; + } + }); + } + + public void testContentTypeJsonCron() throws Exception { + runTest(MyJsonRestServlet.CONTENT_TYPE_JSON_CRON_TEST, new Callable() { + public Void call() throws Exception { + assertEquals(HttpServletResponse.SC_OK, invoke("GET", "", "json=object", "application/xml")); + assertEquals(HttpServletResponse.SC_OK, invoke("GET", "", "json=object", "application/xml; param=x")); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, invoke("GET", "", "json=object", "")); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, invoke("GET", "", "json=object", "application/json")); + String response = invokeAndGetResponse("GET", "", "json=object", "application/xml"); + assertTrue(response.contains("object")); + response = invokeAndGetResponse("GET", "", "json=array", "application/xml"); + assertTrue(response.contains("array")); + return null; + } + }); + } + + +} Index: oozie/core/src/test/java/org/apache/oozie/dag/service/TestDagXLogInfoService.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/service/TestDagXLogInfoService.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/service/TestDagXLogInfoService.java (revision 0) @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.service; + +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.service.Services; +import org.apache.oozie.util.XLog; + +public class TestDagXLogInfoService extends XTestCase { + + public void testLogInfo() throws Exception { + Services services = new Services(); + services.init(); + assertNotNull(services.get(DagXLogInfoService.class)); + String prefix = XLog.Info.get().createPrefix(); + assertTrue(prefix.contains(DagXLogInfoService.TOKEN)); + assertTrue(prefix.contains(DagXLogInfoService.APP)); + assertTrue(prefix.contains(DagXLogInfoService.JOB)); + assertTrue(prefix.contains(DagXLogInfoService.ACTION)); + services.destroy(); + } +} Index: oozie/core/src/test/java/org/apache/oozie/dag/service/TestActionCheckerService.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/service/TestActionCheckerService.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/service/TestActionCheckerService.java (revision 0) @@ -0,0 +1,182 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.service; + +import java.io.FileWriter; +import java.io.Reader; +import java.io.Writer; +import java.util.List; +import java.util.Date; + +import org.apache.hadoop.conf.Configuration; +import org.apache.oozie.client.Workflow; +import org.apache.oozie.client.WorkflowClient; +import org.apache.oozie.dag.ActionBean; +import org.apache.oozie.dag.DagEngine; +import org.apache.oozie.dag.ForTestingActionExecutor; +import org.apache.oozie.dag.service.ActionCheckerService.ActionCheckRunnable; +import org.apache.oozie.dag.store.WorkflowStore; +import org.apache.oozie.service.Services; +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.IOUtils; +import org.apache.oozie.util.XConfiguration; + +/** + * Test cases for the Action Checker Service. + * + */ +public class TestActionCheckerService extends XTestCase { + + private Services services; + + @Override + public void setUp() throws Exception { + super.setUp(); + setSystemProperty(WorkflowSchemaService.CONF_EXT_SCHEMAS, "wf-ext-schema.xsd"); + services = new Services(); + cleanUpDB(services.getConf()); + services.init(); + services.get(ActionService.class).register(ForTestingActionExecutor.class); + } + + public void tearDown() { + services.destroy(); + } + + /** + * Tests functionality of the Action Checker Service Runnable. + *

+ * Starts an action which behaves like an Async Action (Action and Job state + * set to Running). Verifies the action status to be RUNNING. + *

+ * Runs the ActionCheck runnable, and checks for thw job to complete. + * + * @throws Exception + */ + public void testActionCheckerService() throws Exception { + Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1); + Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); + IOUtils.copyCharStream(reader, writer); + + final DagEngine engine = new DagEngine("u", "g", "a"); + Configuration conf = new XConfiguration(); + conf.set(WorkflowClient.APP_PATH, getTestCaseDir()); + conf.set(WorkflowClient.USER_NAME, "u"); + conf.set(WorkflowClient.GROUP_NAME, "g"); + conf.set(WorkflowClient.LOG_TOKEN, "t"); + + conf.set("external-status", "ok"); + conf.set("signal-value", "based_on_action_status"); + conf.set("running-mode", "async"); + + final String jobId = engine.submitJob(conf, true); + Thread.sleep(200); + + waitFor(5000, new Predicate() { + public boolean evaluate() throws Exception { + return (engine.getJob(jobId).getStatus() == Workflow.Status.RUNNING); + } + }); + String actionId = null; + final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create(); + List actions = store.getActionsForWorkflow(jobId, false); + ActionBean action = actions.get(0); + actionId = action.getId(); + assertEquals(ActionBean.Status.RUNNING, action.getStatus()); + store.close(); + + Thread.sleep(2000); + Runnable actionCheckRunnable = new ActionCheckRunnable(0); + actionCheckRunnable.run(); + + waitFor(20000, new Predicate() { + public boolean evaluate() throws Exception { + return (engine.getJob(jobId).getStatus() == Workflow.Status.SUCCEEDED); + } + }); + + final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create(); + List actions2 = store2.getActionsForWorkflow(jobId, false); + ActionBean action2 = actions2.get(0); + assertEquals(ActionBean.Status.OK, action2.getStatus()); + store2.close(); + } + + /** + * Tests the delayed check functionality of the Action Check Service + * Runnable. + *

+ * Starts an action which behaves like an Async Action (Action and Job state + * set to Running). Verifies the action status to be RUNNING. + *

+ * Updates the last check time to now, and attempts to run the + * ActionCheckRunnable with the delay configured to 20 seconds. + * + * @throws Exception + */ + public void testActionCheckerServiceDelay() throws Exception { + Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1); + Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); + IOUtils.copyCharStream(reader, writer); + + final DagEngine engine = new DagEngine("u", "g", "a"); + Configuration conf = new XConfiguration(); + conf.set(WorkflowClient.APP_PATH, getTestCaseDir()); + conf.set(WorkflowClient.USER_NAME, "u"); + conf.set(WorkflowClient.GROUP_NAME, "g"); + conf.set(WorkflowClient.LOG_TOKEN, "t"); + + conf.set("external-status", "ok"); + conf.set("signal-value", "based_on_action_status"); + conf.set("running-mode", "async"); + + final String jobId = engine.submitJob(conf, true); + Thread.sleep(200); + + + waitFor(5000, new Predicate() { + public boolean evaluate() throws Exception { + return (engine.getJob(jobId).getStatus() == Workflow.Status.RUNNING); + } + }); + + Thread.sleep(100); + final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create(); + List actions = store.getActionsForWorkflow(jobId, false); + ActionBean action = actions.get(0); + assertEquals(ActionBean.Status.RUNNING, action.getStatus()); + + action.setLastCheckTime(new Date()); + store.updateAction(action); + store.commit(); + store.close(); + + int actionCheckDelay = 20; + + Runnable actionCheckRunnable = new ActionCheckRunnable(actionCheckDelay); + actionCheckRunnable.run(); + + Thread.sleep(3000); + final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create(); + List actions2 = store2.getActionsForWorkflow(jobId, false); + ActionBean action2 = actions2.get(0); + assertEquals(ActionBean.Status.RUNNING, action2.getStatus()); + store2.close(); + assertEquals(Workflow.Status.RUNNING, engine.getJob(jobId).getStatus()); + } +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/dag/service/TestPurgeService.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/service/TestPurgeService.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/service/TestPurgeService.java (revision 0) @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.service; + +import java.io.FileWriter; +import java.io.Reader; +import java.io.Writer; +import java.util.Date; +import org.apache.hadoop.conf.Configuration; +import org.apache.oozie.client.Workflow; +import org.apache.oozie.client.WorkflowClient; +import org.apache.oozie.dag.DagEngine; +import org.apache.oozie.dag.DagEngineException; +import org.apache.oozie.dag.ForTestingActionExecutor; +import org.apache.oozie.dag.WorkflowBean; +import org.apache.oozie.dag.command.PurgeCommand; +import org.apache.oozie.dag.service.PurgeService.PurgeRunnable; +import org.apache.oozie.dag.store.WorkflowStore; +import org.apache.oozie.service.Services; +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.IOUtils; +import org.apache.oozie.util.XConfiguration; +import org.apache.oozie.dag.store.StoreException; + +/** + * Test cases for checking the correct functionality of the PurgeService. + */ +public class TestPurgeService extends XTestCase { + private Services services; + + @Override + public void setUp() throws Exception { + super.setUp(); + setSystemProperty(WorkflowSchemaService.CONF_EXT_SCHEMAS, "wf-ext-schema.xsd"); + services = new Services(); + services.init(); + services.get(ActionService.class).register(ForTestingActionExecutor.class); + } + + public void tearDown() { + services.destroy(); + } + + /** + * Tests the {@link PurgeService}. + *

+ * Creates and runs a new job to completion. Attempts to purge jobs older + * than a day. Verifies the presence of the job in the system. + *

+ * Sets the end date for the same job to make it qualify for the purge + * criteria. Calls the purge service, and ensure the job does not exist in + * the system. + */ + public void testPurgeService() throws Exception { + Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1); + Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); + IOUtils.copyCharStream(reader, writer); + + final DagEngine engine = new DagEngine("u", "g", "a"); + Configuration conf = new XConfiguration(); + conf.set(WorkflowClient.APP_PATH, getTestCaseDir()); + conf.set(WorkflowClient.USER_NAME, "u"); + conf.set(WorkflowClient.GROUP_NAME, "g"); + conf.set(WorkflowClient.LOG_TOKEN, "t"); + + conf.set("external-status", "ok"); + conf.set("signal-value", "based_on_action_status"); + + final String jobId = engine.submitJob(conf, true); + + waitFor(5000, new Predicate() { + public boolean evaluate() throws Exception { + return (engine.getJob(jobId).getStatus() == Workflow.Status.SUCCEEDED); + } + }); + + assertEquals(Workflow.Status.SUCCEEDED, engine.getJob(jobId).getStatus()); + + new PurgeCommand(1).call(); + assertEquals(Workflow.Status.SUCCEEDED, engine.getJob(jobId).getStatus()); + + final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create(); + WorkflowBean wfBean = store.getWorkflow(jobId, true); + Date endDate = new Date(System.currentTimeMillis() - 2 * 24 * 60 * 60 * 1000); + wfBean.setEndTime(endDate); + store.updateWorkflow(wfBean); + store.commit(); + store.close(); + + Runnable purgeRunnable = new PurgeRunnable(1); + purgeRunnable.run(); + + waitFor(5000, new Predicate() { + public boolean evaluate() throws Exception { + try { + engine.getJob(jobId).getStatus(); + } + catch (Exception ex) { + return true; + } + return false; + } + }); + + try { + engine.getJob(jobId).getStatus(); + assertTrue(false); + } + catch (Exception ex) { + assertEquals(ex.getClass(), DagEngineException.class); + DagEngineException dex = (DagEngineException) ex; + assertEquals(StoreException.ErrorCode.E1204, dex.getErrorCode()); + } + } +} Index: oozie/core/src/test/java/org/apache/oozie/dag/service/TestLiteWorkflowStoreService.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/service/TestLiteWorkflowStoreService.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/service/TestLiteWorkflowStoreService.java (revision 0) @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.service; + +import org.apache.oozie.service.Services; +import org.apache.oozie.test.XTestCase; + +public class TestLiteWorkflowStoreService extends XTestCase { + + protected void setUp() throws Exception { + super.setUp(); + new Services().init(); + } + + protected void tearDown() throws Exception { + super.tearDown(); + Services.get().destroy(); + } + + public void testService() throws Exception { + assertNotNull(Services.get().get(WorkflowStoreService.class)); + } + + public void testCreateStore() throws Exception { + WorkflowStoreService wls = Services.get().get(WorkflowStoreService.class); + assertNotNull(wls); + assertNotNull(wls.create()); + } + + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/dag/service/TestLiteWorkflowAppService.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/service/TestLiteWorkflowAppService.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/service/TestLiteWorkflowAppService.java (revision 0) @@ -0,0 +1,233 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.service; + +import org.apache.hadoop.conf.Configuration; +import org.apache.oozie.client.WorkflowClient; +import org.apache.oozie.dag.workflow.WorkflowApp; +import org.apache.oozie.dag.workflow.WorkflowException; +import org.apache.oozie.dag.workflow.lite.LiteWorkflowApp; +import org.apache.oozie.service.Services; +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.IOUtils; +import org.apache.oozie.util.XConfiguration; + +import java.io.FileWriter; +import java.io.Reader; +import java.io.Writer; + +public class TestLiteWorkflowAppService extends XTestCase { + + public void testService() throws Exception { + Services services = new Services(); + try { + services.init(); + assertNotNull(services.get(WorkflowAppService.class)); + } + finally { + services.destroy(); + } + } + + public void testReadDefinition() throws Exception { + Services services = new Services(); + try { + services.init(); + + Reader reader = IOUtils.getResourceAsReader("wf-schema-valid.xml", -1); + Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); + IOUtils.copyCharStream(reader, writer); + + WorkflowAppService wps = services.get(WorkflowAppService.class); + String wfDef = wps.readDefinition("file://" + getTestCaseDir(), System.getProperty("user.name"), "group", + "authToken"); + assertNotNull(reader.toString(), wfDef); + } + finally { + services.destroy(); + } + } + + public void testNoAppPath() throws Exception { + Services services = new Services(); + services.init(); + WorkflowAppService wps = services.get(WorkflowAppService.class); + try { + assertNotNull(wps.parseDef(new XConfiguration(), "authToken")); + fail(); + } + catch (Exception ex) { + //nop + } + services.destroy(); + } + + public void testSchema() throws Exception { + Services services = new Services(); + try { + services.init(); + + Reader reader = IOUtils.getResourceAsReader("wf-schema-valid.xml", -1); + Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); + IOUtils.copyCharStream(reader, writer); + + WorkflowAppService wps = services.get(WorkflowAppService.class); + + Configuration jobConf = new XConfiguration(); + jobConf.set(WorkflowClient.APP_PATH, "file://" + getTestCaseDir()); + jobConf.set(WorkflowClient.USER_NAME, System.getProperty("user.name")); + jobConf.set(WorkflowClient.GROUP_NAME, "group"); + + WorkflowApp app = wps.parseDef(jobConf, "authToken"); + assertNotNull(app); + assertEquals("test-wf", app.getName()); + + reader = IOUtils.getResourceAsReader("wf-schema-invalid.xml", -1); + writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); + IOUtils.copyCharStream(reader, writer); + + try { + wps.parseDef(jobConf, "authToken"); + fail(); + } + catch (WorkflowException ex) { + //nop + } + } + finally { + services.destroy(); + } + } + + public void testExtSchema() throws Exception { + setSystemProperty(WorkflowSchemaService.CONF_EXT_SCHEMAS, "wf-ext-schema.xsd"); + Services services = new Services(); + try { + services.init(); + + Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1); + Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); + IOUtils.copyCharStream(reader, writer); + + WorkflowAppService wps = services.get(WorkflowAppService.class); + + Configuration jobConf = new XConfiguration(); + jobConf.set(WorkflowClient.APP_PATH, "file://" + getTestCaseDir()); + jobConf.set(WorkflowClient.USER_NAME, System.getProperty("user.name")); + jobConf.set(WorkflowClient.GROUP_NAME, "group"); + + LiteWorkflowApp app = (LiteWorkflowApp) wps.parseDef(jobConf, "authToken"); + assertNotNull(app); + assertEquals("test-wf", app.getName()); + + reader = IOUtils.getResourceAsReader("wf-ext-schema-invalid.xml", -1); + writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); + IOUtils.copyCharStream(reader, writer); + + try { + wps.parseDef(jobConf, "authToken"); + fail(); + } + catch (WorkflowException ex) { + //nop + } + } + finally { + services.destroy(); + } + } + + public void testParsing() throws Exception { + Services services = new Services(); + try { + services.init(); + WorkflowAppService wps = services.get(WorkflowAppService.class); + + Reader reader = IOUtils.getResourceAsReader("wf-schema-valid.xml", -1); + Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); + IOUtils.copyCharStream(reader, writer); + + Configuration jobConf = new XConfiguration(); + jobConf.set(WorkflowClient.APP_PATH, "file://" + getTestCaseDir()); + jobConf.set(WorkflowClient.USER_NAME, System.getProperty("user.name")); + jobConf.set(WorkflowClient.GROUP_NAME, "group"); + + LiteWorkflowApp app = (LiteWorkflowApp) wps.parseDef(jobConf, "authToken"); + assertNotNull(app); + assertEquals("test-wf", app.getName()); + assertNotNull(app.getNode("::start::")); + assertEquals("a", app.getNode("::start::").getTransitions().get(0)); + assertEquals("b", app.getNode("a").getTransitions().get(0)); + assertEquals("c", app.getNode("a").getTransitions().get(1)); + assertEquals("d", app.getNode("a").getTransitions().get(2)); + assertTrue(app.getNode("b").getConf().contains("kill")); + assertEquals("d", app.getNode("c").getTransitions().get(0)); + assertEquals("e", app.getNode("c").getTransitions().get(1)); + assertEquals(2, app.getNode("c").getTransitions().size()); + + assertEquals("e", app.getNode("d").getTransitions().get(0)); + assertEquals("b", app.getNode("d").getTransitions().get(1)); + assertTrue(app.getNode("d").getConf().startsWith("" + + "" + + ""; + + private static final String APP2 = "" + + "" + + "" + + "" + + "a" + + "b" + + "c" + + "d" + + "d" + + "e" + + "" + + "" + + "" + + "" + + "" + + ""; + + protected void setUp() throws Exception { + super.setUp(); + new Services().init(); + } + + protected void tearDown() throws Exception { + Services.get().destroy(); + super.tearDown(); + } + + public void testService() throws Exception { + assertNotNull(Services.get().get(WorkflowSchemaService.class)); + } + + public void testOozieSchema() throws Exception { + WorkflowSchemaService wss = Services.get().get(WorkflowSchemaService.class); + Validator validator = wss.getSchema().newValidator(); + validator.validate(new StreamSource(new StringReader(APP1))); + } + + public void testExtSchema() throws Exception { + Services.get().destroy(); + setSystemProperty(WorkflowSchemaService.CONF_EXT_SCHEMAS, "wf-ext-schema.xsd"); + new Services().init(); + WorkflowSchemaService wss = Services.get().get(WorkflowSchemaService.class); + Validator validator = wss.getSchema().newValidator(); + validator.validate(new StreamSource(new StringReader(APP2))); + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/dag/service/TestActionRecoveryService.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/service/TestActionRecoveryService.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/service/TestActionRecoveryService.java (revision 0) @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.service; + +import java.io.FileWriter; +import java.io.Reader; +import java.io.Writer; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.oozie.client.Workflow; +import org.apache.oozie.client.WorkflowClient; +import org.apache.oozie.dag.ActionBean; +import org.apache.oozie.dag.DagEngine; +import org.apache.oozie.dag.ForTestingActionExecutor; +import org.apache.oozie.dag.service.ActionRecoveryService.ActionRecoveryRunnable; +import org.apache.oozie.dag.store.WorkflowStore; +import org.apache.oozie.service.Services; +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.IOUtils; +import org.apache.oozie.util.XConfiguration; + +public class TestActionRecoveryService extends XTestCase { + private Services services; + + @Override + public void setUp() throws Exception { + super.setUp(); + setSystemProperty(WorkflowSchemaService.CONF_EXT_SCHEMAS, "wf-ext-schema.xsd"); + services = new Services(); + services.init(); + services.get(ActionService.class).register(ForTestingActionExecutor.class); + } + + public void tearDown() { + services.destroy(); + } + + /** + * Tests functionality of the Recovery Service Runnable command. + *

+ * Starts an action which behaves like an Async Action (Action and Job state + * set to Running). Changes the action configuration to run in sync mode and + * updates the store. Runs the recovery runnable, and ensures the state of + * the action and job have not changed. + *

+ * Changes the state of the action from RUNNING to PREP and updates the + * store. Again, runs the recovery runnable and ensures the state changes to + * OK and the job completes successfully. + * + * @throws Exception + */ + public void testRecoveryService() throws Exception { + Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1); + Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); + IOUtils.copyCharStream(reader, writer); + + final DagEngine engine = new DagEngine("u", "g", "a"); + Configuration conf = new XConfiguration(); + conf.set(WorkflowClient.APP_PATH, getTestCaseDir()); + conf.set(WorkflowClient.USER_NAME, "u"); + conf.set(WorkflowClient.GROUP_NAME, "g"); + conf.set(WorkflowClient.LOG_TOKEN, "t"); + + conf.set("external-status", "ok"); + conf.set("signal-value", "based_on_action_status"); + conf.set("running-mode", "async"); + + final String jobId = engine.submitJob(conf, true); + Thread.sleep(200); + + waitFor(5000, new Predicate() { + public boolean evaluate() throws Exception { + return (engine.getJob(jobId).getStatus() == Workflow.Status.RUNNING); + } + }); + + Thread.sleep(100); + final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create(); + List actions = store.getActionsForWorkflow(jobId, false); + ActionBean action = actions.get(0); + assertEquals(ActionBean.Status.RUNNING, action.getStatus()); + + String actionConf = action.getConf(); + String fixedActionConf = actionConf.replaceAll("async", "sync"); + action.setConf(fixedActionConf); + store.updateAction(action); + store.commit(); + store.close(); + + Runnable recoveryRunnable = new ActionRecoveryRunnable(0); + + recoveryRunnable.run(); + Thread.sleep(3000); + + final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create(); + assertEquals(Workflow.Status.RUNNING, engine.getJob(jobId).getStatus()); + List actions2 = store2.getActionsForWorkflow(jobId, false); + ActionBean action2 = actions2.get(0); + assertEquals(ActionBean.Status.RUNNING, action2.getStatus()); + action.setStatus(ActionBean.Status.PREP); + store2.updateAction(action); + store2.commit(); + store2.close(); + + recoveryRunnable.run(); + Thread.sleep(3000); + + final WorkflowStore store3 = Services.get().get(WorkflowStoreService.class).create(); + assertEquals(Workflow.Status.SUCCEEDED, engine.getJob(jobId).getStatus()); + List actions3 = store3.getActionsForWorkflow(jobId, false); + ActionBean action3 = actions3.get(0); + assertEquals(ActionBean.Status.OK, action3.getStatus()); + store3.close(); + } +} Index: oozie/core/src/test/java/org/apache/oozie/dag/service/TestCallbackService.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/service/TestCallbackService.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/service/TestCallbackService.java (revision 0) @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.service; + +import org.apache.oozie.service.Services; +import org.apache.oozie.test.XTestCase; + +public class TestCallbackService extends XTestCase { + + public void testService() throws Exception { + Services services = new Services(); + services.init(); + CallbackService cs = services.get(CallbackService.class); + assertNotNull(cs); + services.destroy(); + } + + public void testCallbacks() throws Exception { + Services services = new Services(); + services.init(); + CallbackService cs = services.get(CallbackService.class); + assertNotNull(cs); + String callback = cs.createCallBackUrl("a", "@STATUS"); + assertTrue(callback.contains("http://")); + assertTrue(callback.contains("id=a")); + assertTrue(callback.contains("status=@STATUS")); + callback = callback.replace("@STATUS", "OK"); + assertEquals("a", cs.getActionId(callback)); + assertEquals("OK", cs.getExternalStatus(callback)); + services.destroy(); + } + +} Index: oozie/core/src/test/java/org/apache/oozie/dag/service/TestActionService.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/service/TestActionService.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/service/TestActionService.java (revision 0) @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.service; + +import org.apache.oozie.service.Services; +import org.apache.oozie.test.XTestCase; + +public class TestActionService extends XTestCase { + + public void testService() throws Exception { + Services services = new Services(); + services.init(); + assertNotNull(services.get(ActionService.class)); + services.destroy(); + } + + public void testActions() throws Exception { + Services services = new Services(); + services.init(); + ActionService as = services.get(ActionService.class); + assertNotNull(as.getExecutor("switch")); + services.destroy(); + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/dag/TestWorkflowBean.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/TestWorkflowBean.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/TestWorkflowBean.java (revision 0) @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag; + +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.dag.workflow.WorkflowInstance; +import org.apache.oozie.dag.workflow.WorkflowApp; +import org.apache.oozie.dag.workflow.WorkflowException; +import org.apache.hadoop.conf.Configuration; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.util.Map; + +public class TestWorkflowBean extends XTestCase { + + private static class MyWorkflowInstance implements WorkflowInstance { + public Configuration getConf() { + return null; + } + + public String getId() { + return null; + } + + public WorkflowApp getApp() { + return null; + } + + public boolean start() throws WorkflowException { + return false; + } + + public boolean signal(String path, String signaValue) throws WorkflowException { + return false; + } + + public void fail(String nodeName) throws WorkflowException { + } + + public void kill() throws WorkflowException { + } + + public void suspend() throws WorkflowException { + } + + public void resume() throws WorkflowException { + } + + public Status getStatus() { + return null; + } + + public void setVar(String name, String value) { + } + + public String getVar(String name) { + return null; + } + + public Map getAllVars() { + return null; + } + + public void setAllVars(Map varMap) { + } + + public void setTransientVar(String name, Object value) { + } + + public Object getTransientVar(String name) { + return null; + } + + public String getTransition(String node) { + return null; + } + } + + public void testWorkflow() { + WorkflowBean workflow = new WorkflowBean(); + workflow.setAuthToken("authToken"); + workflow.setLogToken("logToken"); + workflow.setWorkflowInstance(new MyWorkflowInstance()); + workflow.setProtoActionConf("proto"); + assertEquals("authToken", workflow.getAuthToken()); + assertEquals("logToken", workflow.getLogToken()); + assertNotNull(workflow.getWorkflowInstance()); + assertEquals("proto", workflow.getProtoActionConf()); + } + + public void testEmptyWriteRead() throws Exception { + WorkflowBean workflow = new WorkflowBean(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + workflow.write(dos); + dos.close(); + DataInputStream dis = new DataInputStream(new ByteArrayInputStream(baos.toByteArray())); + workflow = new WorkflowBean(); + workflow.readFields(dis); + + } + + public void testFullWriteRead() throws Exception { + //TODO + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/dag/ForTestingActionExecutor.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/ForTestingActionExecutor.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/ForTestingActionExecutor.java (revision 0) @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag; + +import org.apache.oozie.client.Action; +import org.apache.oozie.dag.action.ActionExecutor; +import org.apache.oozie.dag.action.ActionExecutorException; +import org.apache.oozie.util.XmlUtils; +import org.jdom.Element; +import org.jdom.JDOMException; +import org.jdom.Namespace; + +public class ForTestingActionExecutor extends ActionExecutor { + public final static String TEST_ERROR = "TEST_ERROR"; + + protected ForTestingActionExecutor() { + super("test"); + } + + public void initActionType() { + } + + private Element getConfiguration(String strConf) throws ActionExecutorException { + try { + return XmlUtils.parseXml(strConf); + } + catch (JDOMException ex) { + throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, TEST_ERROR, ex.getMessage(), ex); + } + } + + public void start(Context context, Action action) throws ActionExecutorException { + Element eConf = getConfiguration(action.getConf()); + Namespace ns = eConf.getNamespace(); + String error = eConf.getChild("error", ns).getText().trim(); + + if ("start.transient".equals(error)) { + throw new ActionExecutorException(ActionExecutorException.ErrorType.TRANSIENT, TEST_ERROR, "start"); + } + if ("start.non-transient".equals(error)) { + throw new ActionExecutorException(ActionExecutorException.ErrorType.NON_TRANSIENT, TEST_ERROR, "start"); + } + if ("start.error".equals(error)) { + throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, TEST_ERROR, "start"); + } + String externalStatus = eConf.getChild("external-status", ns).getText().trim(); + + String runningMode = "sync"; + Element runningModeElement = eConf.getChild("running-mode", ns); + if (null != runningModeElement) { + if (runningModeElement.getText().trim().equals("async")) { + runningMode = "async"; + } + } + if (runningMode.equals("async")) { + context.setStartData("blah", "blah", "blah"); + return; + } + + boolean callSetExecutionData = true; + Element setStartData = eConf.getChild("avoid-set-execution-data", ns); + if (null != setStartData) { + if (setStartData.getText().trim().equals("true")) { + callSetExecutionData = false; + } + } + if (callSetExecutionData) { + context.setExecutionData(externalStatus, null); + } + } + + public void end(Context context, Action action) throws ActionExecutorException { + Element eConf = getConfiguration(action.getConf()); + Namespace ns = eConf.getNamespace(); + String error = eConf.getChild("error", ns).getText().trim(); + if ("end.transient".equals(error)) { + throw new ActionExecutorException(ActionExecutorException.ErrorType.TRANSIENT, TEST_ERROR, "end"); + } + if ("end.non-transient".equals(error)) { + throw new ActionExecutorException(ActionExecutorException.ErrorType.NON_TRANSIENT, TEST_ERROR, "end"); + } + if ("end.error".equals(error)) { + throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, TEST_ERROR, "end"); + } + String signalValue = eConf.getChild("signal-value", ns).getText().trim(); + String externalStatus = action.getExternalStatus(); + Action.Status status = null; + if (externalStatus.equals("ok")) { + status = Action.Status.OK; + } + else { + status = Action.Status.ERROR; + } + if (signalValue.equals("based_on_action_status")) { + signalValue = status.toString(); + } + + boolean callSetEndData = true; + Element setEndData = eConf.getChild("avoid-set-end-data", ns); + if (null != setEndData) { + if (setEndData.getText().trim().equals("true")) { + callSetEndData = false; + } + } + if (callSetEndData) { + context.setEndData(status, signalValue); + } + } + + public void check(Context context, Action action) throws ActionExecutorException { + context.setExecutionData("ok", null); + } + + public void kill(Context context, Action action) throws ActionExecutorException { + } + + public boolean isCompleted(String externalStatus) { + return false; + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/dag/TestDagEngine.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/TestDagEngine.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/TestDagEngine.java (revision 0) @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag; + +import org.apache.hadoop.conf.Configuration; +import org.apache.oozie.client.Workflow; +import org.apache.oozie.client.WorkflowClient; +import org.apache.oozie.dag.service.ActionService; +import org.apache.oozie.dag.service.WorkflowStoreService; +import org.apache.oozie.dag.service.WorkflowSchemaService; +import org.apache.oozie.service.Services; +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.IOUtils; +import org.apache.oozie.util.XConfiguration; + +import java.io.FileWriter; +import java.io.Reader; +import java.io.Writer; +import java.io.StringReader; +import java.io.OutputStream; +import java.io.FileOutputStream; +import java.util.List; + +public class TestDagEngine extends XTestCase { + private Services services; + + public void setUp() throws Exception { + super.setUp(); + setSystemProperty(WorkflowSchemaService.CONF_EXT_SCHEMAS, "wf-ext-schema.xsd"); + services = new Services(); + cleanUpDB(services.getConf()); + services.init(); + services.get(ActionService.class).register(ForTestingActionExecutor.class); + } + + public void tearDown() { + services.destroy(); + } + + public void testSubmit() throws Exception { + Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1); + Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); + IOUtils.copyCharStream(reader, writer); + + OutputStream os = new FileOutputStream(getTestCaseDir() + "/config-default.xml"); + XConfiguration defaultConf = new XConfiguration(); + defaultConf.set("a", "AA"); + defaultConf.set("b", "BB"); + defaultConf.writeXml(os); + os.close(); + + final DagEngine engine = new DagEngine("u", "g", "a"); + Configuration conf = new XConfiguration(); + conf.set(WorkflowClient.APP_PATH, getTestCaseDir()); + conf.set(WorkflowClient.USER_NAME, "u"); + conf.set(WorkflowClient.GROUP_NAME, "g"); + conf.set(WorkflowClient.LOG_TOKEN, "t"); + conf.set("signal-value", "OK"); + conf.set("external-status", "ok"); + conf.set("error", "end.error"); + conf.set("b", "B"); + + final String jobId1 = engine.submitJob(conf, true); + + Workflow wf = engine.getJob(jobId1); + XConfiguration wfConf = new XConfiguration(new StringReader(wf.getConf())); + assertEquals("AA", wfConf.get("a")); + assertEquals("B", wfConf.get("b")); + + waitFor(5000, new Predicate() { + public boolean evaluate() throws Exception { + WorkflowBean bean = Services.get().get(WorkflowStoreService.class).create().getWorkflow(jobId1, false); + return bean.getWorkflowInstance().getStatus().isEndState(); + } + }); + assertEquals(Workflow.Status.KILLED, engine.getJob(jobId1).getStatus()); + } + + public void testJobDefinition() throws Exception { + Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1); + Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); + IOUtils.copyCharStream(reader, writer); + + final DagEngine engine = new DagEngine("u", "g", "a"); + Configuration conf = new XConfiguration(); + conf.set(WorkflowClient.APP_PATH, getTestCaseDir()); + conf.set(WorkflowClient.USER_NAME, "u"); + conf.set(WorkflowClient.GROUP_NAME, "g"); + conf.set(WorkflowClient.LOG_TOKEN, "t"); + conf.set("signal-value", "OK"); + conf.set("external-status", "ok"); + conf.set("error", "end.error"); + + String jobId1 = engine.submitJob(conf, false); + + String def = engine.getDefinition(jobId1); + assertNotNull(def); + } + + public void testGetJobs() throws Exception { + Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1); + Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); + IOUtils.copyCharStream(reader, writer); + + final DagEngine engine = new DagEngine("u", "g", "a"); + Configuration conf = new XConfiguration(); + conf.set(WorkflowClient.APP_PATH, getTestCaseDir()); + conf.set(WorkflowClient.USER_NAME, "u"); + conf.set(WorkflowClient.GROUP_NAME, "g"); + conf.set(WorkflowClient.LOG_TOKEN, "t"); + conf.set("signal-value", "OK"); + conf.set("external-status", "ok"); + conf.set("error", "end.error"); + + final String jobId1 = engine.submitJob(conf, true); + String jobId2 = engine.submitJob(conf, false); + + WorkflowsInfo wfInfo = engine.getJobs("group=g", 1, 1); + List workflows = wfInfo.getWorkflows(); + assertEquals(1, workflows.size()); + assertEquals("g", workflows.get(0).getGroup()); + assertEquals(jobId1, workflows.get(0).getId()); + + wfInfo = engine.getJobs("group=g", 1, 5); + workflows = wfInfo.getWorkflows(); + assertEquals(2, workflows.size()); + assertEquals("g", workflows.get(0).getGroup()); + assertEquals(jobId1, workflows.get(0).getId()); + assertEquals(jobId2, workflows.get(1).getId()); + + wfInfo = engine.getJobs("user=u", 1, 1); + workflows = wfInfo.getWorkflows(); + assertEquals(1, workflows.size()); + assertEquals("u", workflows.get(0).getUser()); + assertEquals(jobId1, workflows.get(0).getId()); + + wfInfo = engine.getJobs("user=u", 2, 5); + workflows = wfInfo.getWorkflows(); + assertEquals(1, workflows.size()); + assertEquals("u", workflows.get(0).getUser()); + assertEquals(jobId2, workflows.get(0).getId()); + + waitFor(5000, new Predicate() { + public boolean evaluate() throws Exception { + WorkflowBean bean = Services.get().get(WorkflowStoreService.class).create().getWorkflow(jobId1, false); + return bean.getWorkflowInstance().getStatus().isEndState(); + } + }); + + wfInfo = engine.getJobs("status=PREP", 1, 5); + workflows = wfInfo.getWorkflows(); + assertEquals(1, workflows.size()); + assertEquals(jobId2, workflows.get(0).getId()); + + wfInfo = engine.getJobs("name=test-wf", 1, 5); + workflows = wfInfo.getWorkflows(); + assertEquals(2, workflows.size()); + assertEquals(jobId1, workflows.get(0).getId()); + assertEquals(jobId2, workflows.get(1).getId()); + + wfInfo = engine.getJobs("name=test-wf;status=PREP", 1, 5); + workflows = wfInfo.getWorkflows(); + assertEquals(1, workflows.size()); + assertEquals(jobId2, workflows.get(0).getId()); + } +} Index: oozie/core/src/test/java/org/apache/oozie/dag/TestActionBean.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/TestActionBean.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/TestActionBean.java (revision 0) @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag; + +import org.apache.oozie.client.Action; +import org.apache.oozie.test.XTestCase; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.util.Date; + +public class TestActionBean extends XTestCase { + + public void testAction() { + ActionBean action = new ActionBean(); + action.setJobId("id"); + action.setExecutionPath("executionPath"); + action.setPending(); + action.setPendingAge(new Date()); + action.setSignalValue("signal"); + action.setLogToken("logToken"); + assertEquals("id", action.getJobId()); + assertEquals("executionPath", action.getExecutionPath()); + assertTrue(action.isPending()); + assertNotNull(action.getPendingAge()); + assertEquals("signal", action.getSignalValue()); + assertEquals("logToken", action.getLogToken()); + + action.setExecutionData("externalStatus", System.getProperties()); + assertEquals("externalStatus", action.getExternalStatus()); + assertNotNull(action.getData()); + + action.setEndData(Action.Status.OK, "signal"); + assertEquals(Action.Status.OK, action.getStatus()); + assertEquals("externalStatus", action.getExternalStatus()); + + action.setStartData("externalId", "trackerUri", "consoleUrl"); + assertEquals("externalId", action.getExternalId()); + assertEquals("trackerUri", action.getTrackerUri()); + assertEquals("consoleUrl", action.getConsoleUrl()); + } + + public void testEmptyWriteRead() throws Exception { + ActionBean action = new ActionBean(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + action.write(dos); + dos.close(); + DataInputStream dis = new DataInputStream(new ByteArrayInputStream(baos.toByteArray())); + action = new ActionBean(); + action.readFields(dis); + } + + public void testFullWriteRead() throws Exception { + //TODO + } +} Index: oozie/core/src/test/java/org/apache/oozie/dag/servlet/TestJobServlet.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/servlet/TestJobServlet.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/servlet/TestJobServlet.java (revision 0) @@ -0,0 +1,186 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.servlet; + +import org.apache.hadoop.conf.Configuration; +import org.apache.oozie.client.WorkflowClient; +import org.apache.oozie.client.rest.RestConstants; +import org.apache.oozie.client.rest.JsonTags; +import org.apache.oozie.util.IOUtils; +import org.apache.oozie.util.XConfiguration; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; + +import javax.servlet.http.HttpServletResponse; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Callable; + +public class TestJobServlet extends DagServletTestCase { + + static { + new JobServlet(); + } + private static final boolean IS_SECURITY_ENABLED = false; + + public void setUp() throws Exception { + super.setUp(); + } + + private void _testAction(final String action, final Configuration conf) throws Exception { + runTest("/job/*", JobServlet.class, IS_SECURITY_ENABLED, new Callable() { + public Void call() throws Exception { + MockDagEngineService.reset(); + Map params = new HashMap(); + params.put(RestConstants.ACTION_PARAM, action); + URL url = createURL(MockDagEngineService.JOB_ID + 1, params); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("PUT"); + conn.setRequestProperty("content-type", RestConstants.XML_CONTENT_TYPE); + conn.setDoOutput(true); + if (conf != null) { + conf.writeXml(conn.getOutputStream()); + } + if (conf == null || conf.get(WorkflowClient.USER_NAME) != null) { + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + assertEquals(action, MockDagEngineService.did); + } + else { + assertEquals(HttpServletResponse.SC_UNAUTHORIZED, conn.getResponseCode()); + } + + MockDagEngineService.reset(); + params = new HashMap(); + params.put(RestConstants.ACTION_PARAM, action); + url = createURL(MockDagEngineService.JOB_ID+(MockDagEngineService.workflows.size()+1), params); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("PUT"); + conn.setRequestProperty("content-type", RestConstants.XML_CONTENT_TYPE); + conn.setDoOutput(true); + if (conf != null) { + conf.writeXml(conn.getOutputStream()); + } + if (conf == null || conf.get(WorkflowClient.USER_NAME) != null) { + assertEquals(HttpServletResponse.SC_BAD_REQUEST, conn.getResponseCode()); + assertEquals(action, MockDagEngineService.did); + } + else { + assertEquals(HttpServletResponse.SC_UNAUTHORIZED, conn.getResponseCode()); + } + return null; + } + }); + } + + public void testStart() throws Exception { + _testAction(RestConstants.JOB_ACTION_START, null); + } + + public void testSuspend() throws Exception { + _testAction(RestConstants.JOB_ACTION_SUSPEND, null); + } + + public void testResume() throws Exception { + _testAction(RestConstants.JOB_ACTION_RESUME, null); + } + + public void testKill() throws Exception { + _testAction(RestConstants.JOB_ACTION_KILL, null); + } + + public void testReRun() throws Exception { + Configuration conf = new XConfiguration(); + conf.set(WorkflowClient.USER_NAME, "user"); + _testAction(RestConstants.JOB_ACTION_RERUN, conf); + } + + public void testInvalidReRunConfigurations() throws Exception { + Configuration conf = new XConfiguration(); + _testAction(RestConstants.JOB_ACTION_RERUN, conf); + } + + private void _testNonJsonResponses(final String show, final String contentType, final String response) + throws Exception { + runTest("/job/*", JobServlet.class, IS_SECURITY_ENABLED, new Callable() { + public Void call() throws Exception { + MockDagEngineService.reset(); + Map params = new HashMap(); + params.put(RestConstants.JOB_SHOW_PARAM, show); + URL url = createURL(MockDagEngineService.JOB_ID+1, params); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + assertTrue(conn.getHeaderField("content-type").startsWith(contentType)); + String output = IOUtils.getReaderAsString(new InputStreamReader(conn.getInputStream()), 1000); + assertEquals(response, output); + assertEquals(show, MockDagEngineService.did); + + MockDagEngineService.reset(); + params = new HashMap(); + params.put(RestConstants.JOB_SHOW_PARAM, show); + url = createURL(MockDagEngineService.JOB_ID+(MockDagEngineService.workflows.size()+1), params); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, conn.getResponseCode()); + assertEquals(show, MockDagEngineService.did); + return null; + } + }); + } + + public void testJobDef() throws Exception { + _testNonJsonResponses(RestConstants.JOB_SHOW_DEFINITION, RestConstants.XML_CONTENT_TYPE, + MockDagEngineService.WORKFLOW_APP); + } + + public void testJobLog() throws Exception { + _testNonJsonResponses(RestConstants.JOB_SHOW_LOG, RestConstants.TEXT_CONTENT_TYPE, + MockDagEngineService.LOG); + } + + public void testJobInfo() throws Exception { + runTest("/job/*", JobServlet.class, IS_SECURITY_ENABLED, new Callable() { + public Void call() throws Exception { + MockDagEngineService.reset(); + Map params = new HashMap(); + params.put(RestConstants.JOB_SHOW_PARAM, RestConstants.JOB_SHOW_INFO); + URL url = createURL(MockDagEngineService.JOB_ID+1, params); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE)); + JSONObject obj = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); + assertEquals(MockDagEngineService.JOB_ID+1, obj.get(JsonTags.WORKFLOW_ID)); + assertEquals(RestConstants.JOB_SHOW_INFO, MockDagEngineService.did); + + MockDagEngineService.reset(); + params = new HashMap(); + params.put(RestConstants.JOB_SHOW_PARAM, RestConstants.JOB_SHOW_INFO); + url = createURL(MockDagEngineService.JOB_ID+(MockDagEngineService.workflows.size()+1), params); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, conn.getResponseCode()); + assertEquals(RestConstants.JOB_SHOW_INFO, MockDagEngineService.did); + return null; + } + }); + } +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/dag/servlet/TestCallbackServlet.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/servlet/TestCallbackServlet.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/servlet/TestCallbackServlet.java (revision 0) @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.servlet; + +import org.apache.oozie.client.rest.RestConstants; +import org.apache.oozie.dag.service.CallbackService; +import org.apache.oozie.service.Services; + +import javax.servlet.http.HttpServletResponse; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.Callable; + +public class TestCallbackServlet extends DagServletTestCase { + + static { + new CallbackServlet(); + } + + @SuppressWarnings("unchecked") + public void testCallbackGet() throws Exception { + runTest("/callback", CallbackServlet.class, true, new Callable() { + public Void call() throws Exception { + URL url = createURL("", Collections.EMPTY_MAP); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, conn.getResponseCode()); + + Map params = new HashMap(); + params.put("id", "error"); + params.put("status", "error"); + url = createURL("", params); + conn = (HttpURLConnection) url.openConnection(); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, conn.getResponseCode()); + + params = new HashMap(); + params.put("id", "ok"); + params.put("status", "ok"); + url = createURL("", params); + conn = (HttpURLConnection) url.openConnection(); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + + return null; + } + }); + } + + @SuppressWarnings("unchecked") + public void testCallbackPost() throws Exception { + runTest("/callback", CallbackServlet.class, true, new Callable() { + public Void call() throws Exception { + MockDagEngineService.reset(); + URL url = createURL("", Collections.EMPTY_MAP); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, conn.getResponseCode()); + + MockDagEngineService.reset(); + Map params = new HashMap(); + params.put("id", "error"); + params.put("status", "error"); + url = createURL("", params); + conn = (HttpURLConnection) url.openConnection(); + conn.setDoOutput(true); + conn.setRequestMethod("POST"); + conn.setRequestProperty("content-type", RestConstants.TEXT_CONTENT_TYPE); + Properties props = new Properties(); + props.setProperty("a", "A"); + props.store(conn.getOutputStream(), "UTF-8"); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, conn.getResponseCode()); + + MockDagEngineService.reset(); + params = new HashMap(); + params.put("id", "ok"); + params.put("status", "ok"); + url = createURL("", params); + conn = (HttpURLConnection) url.openConnection(); + conn.setDoOutput(true); + conn.setRequestMethod("POST"); + conn.setRequestProperty("content-type", RestConstants.TEXT_CONTENT_TYPE); + props = new Properties(); + props.setProperty("a", "A"); + props.store(conn.getOutputStream(), "UTF-8"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + assertEquals(props, MockDagEngineService.properties); + return null; + } + }); + } + +} Index: oozie/core/src/test/java/org/apache/oozie/dag/servlet/TestJobsServlet.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/servlet/TestJobsServlet.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/servlet/TestJobsServlet.java (revision 0) @@ -0,0 +1,166 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.servlet; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.oozie.client.WorkflowClient; +import org.apache.oozie.client.rest.JsonTags; +import org.apache.oozie.client.rest.RestConstants; +import org.apache.oozie.util.XConfiguration; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; + +import javax.servlet.http.HttpServletResponse; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Callable; + +public class TestJobsServlet extends DagServletTestCase { + + static { + new JobsServlet(); + } + private static final boolean IS_SECURITY_ENABLED = false; + + public void setUp() throws Exception { + super.setUp(); + } + + public void testSubmit() throws Exception { + runTest("/jobs", JobsServlet.class, IS_SECURITY_ENABLED, new Callable() { + public Void call() throws Exception { + MockDagEngineService.reset(); + + String appPath = getTestCaseDir() + "/app"; + Configuration conf = new Configuration(); + + FileSystem fs = FileSystem.get(new URI(appPath), conf); + Path jobXmlPath = new Path(appPath, "workflow.xml"); + fs.create(jobXmlPath); + + int wfCount = MockDagEngineService.workflows.size(); + Configuration jobConf = new XConfiguration(); + jobConf.set(WorkflowClient.USER_NAME, System.getProperty("user.name")); + jobConf.set(WorkflowClient.GROUP_NAME, "others"); + jobConf.set(WorkflowClient.APP_PATH, appPath); + Map params = new HashMap(); + URL url = createURL("", params); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("POST"); + conn.setRequestProperty("content-type", RestConstants.XML_CONTENT_TYPE); + conn.setDoOutput(true); + jobConf.writeXml(conn.getOutputStream()); + assertEquals(HttpServletResponse.SC_CREATED, conn.getResponseCode()); + JSONObject obj = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); + assertEquals(MockDagEngineService.JOB_ID+wfCount, obj.get(JsonTags.JOB_ID)); + assertFalse(MockDagEngineService.started.get(wfCount)); + wfCount++; + + jobConf = new XConfiguration(); + jobConf.set(WorkflowClient.USER_NAME, System.getProperty("user.name")); + jobConf.set(WorkflowClient.GROUP_NAME, "others"); + jobConf.set(WorkflowClient.APP_PATH, appPath); + params = new HashMap(); + params.put(RestConstants.ACTION_PARAM, RestConstants.JOB_ACTION_START); + url = createURL("", params); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("POST"); + conn.setRequestProperty("content-type", RestConstants.XML_CONTENT_TYPE); + conn.setDoOutput(true); + jobConf.writeXml(conn.getOutputStream()); + assertEquals(HttpServletResponse.SC_CREATED, conn.getResponseCode()); + obj = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); + assertEquals(MockDagEngineService.JOB_ID+wfCount, obj.get(JsonTags.JOB_ID)); + assertTrue(MockDagEngineService.started.get(wfCount)); + return null; + } + }); + } + + public void testJobs() throws Exception { + runTest("/jobs", JobsServlet.class, IS_SECURITY_ENABLED, new Callable() { + public Void call() throws Exception { + MockDagEngineService.reset(); + + int wfCount = MockDagEngineService.workflows.size(); + Map params = new HashMap(); + params.put(RestConstants.JOBS_FILTER_PARAM, "name=x"); + URL url = createURL("", params); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE)); + JSONObject json = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); + JSONArray array = (JSONArray) json.get(JsonTags.WORKFLOWS_JOBS); + assertEquals(MockDagEngineService.INIT_WF_COUNT, array.size()); + for (int i = 0; i < MockDagEngineService.INIT_WF_COUNT; i++) { + assertEquals(MockDagEngineService.JOB_ID + i, ((JSONObject)array.get(i)).get(JsonTags.WORKFLOW_ID)); + assertNotNull(((JSONObject) array.get(i)).get(JsonTags.WORKFLOW_APP_PATH)); + } + + params = new HashMap(); + params.put(RestConstants.JOBS_FILTER_PARAM, "name=x"); + params.put(RestConstants.OFFSET_PARAM, "2"); + params.put(RestConstants.LEN_PARAM, "100"); + url = createURL("", params); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE)); + json = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); + array = (JSONArray) json.get(JsonTags.WORKFLOWS_JOBS); + + assertEquals(MockDagEngineService.INIT_WF_COUNT, array.size()); + for (int i = 0; i < MockDagEngineService.INIT_WF_COUNT; i++) { + assertEquals(MockDagEngineService.JOB_ID + i, ((JSONObject)array.get(i)).get(JsonTags.WORKFLOW_ID)); + assertNotNull(((JSONObject) array.get(i)).get(JsonTags.WORKFLOW_APP_PATH)); + } + + params = new HashMap(); + params.put(RestConstants.JOBS_EXTERNAL_ID_PARAM, "external-valid"); + url = createURL("", params); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE)); + JSONObject obj = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); + assertEquals("id-valid", obj.get(JsonTags.JOB_ID)); + + params = new HashMap(); + params.put(RestConstants.JOBS_EXTERNAL_ID_PARAM, "external-invalid"); + url = createURL("", params); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE)); + obj = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); + assertNull(obj.get(JsonTags.JOB_ID)); + + return null; + } + }); + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/dag/servlet/TestVersionServlet.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/servlet/TestVersionServlet.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/servlet/TestVersionServlet.java (revision 0) @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.servlet; + +import org.apache.oozie.client.rest.RestConstants; +import org.apache.oozie.client.WorkflowClient; +import org.json.simple.JSONArray; +import org.json.simple.JSONValue; + +import javax.servlet.http.HttpServletResponse; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Callable; + +public class TestVersionServlet extends DagServletTestCase { + + static { + new VersionServlet(); + } + + public void testVersion() throws Exception { + runTest("/version", VersionServlet.class, true, new Callable() { + public Void call() throws Exception { + Map params = new HashMap(); + URL url = createURL("", params); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE)); + JSONArray array = (JSONArray) JSONValue.parse(new InputStreamReader(conn.getInputStream())); + assertEquals(1, array.size()); + assertEquals(WorkflowClient.WS_PROTOCOL_VERSION, array.get(0)); + return null; + } + }); + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/dag/servlet/DagServletTestCase.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/servlet/DagServletTestCase.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/servlet/DagServletTestCase.java (revision 0) @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.servlet; + +import org.apache.oozie.service.AuthorizationService; + +import org.apache.oozie.service.Services; +import org.apache.oozie.test.EmbeddedServletContainer; +import org.apache.oozie.test.XTestCase; + +import java.net.URL; +import java.net.URLEncoder; +import java.util.Map; +import java.util.concurrent.Callable; + +public abstract class DagServletTestCase extends XTestCase { + private EmbeddedServletContainer container; + private String servletPath; + + protected String getContextURL() { + return container.getContextURL(); + } + + protected URL createURL(String servletPath, String resource, Map parameters) throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append(container.getServletURL(servletPath)); + if (resource != null && resource.length() > 0) { + sb.append("/").append(resource); + } + if (parameters.size() > 0) { + String separator = "?"; + for (Map.Entry param : parameters.entrySet()) { + sb.append(separator).append(URLEncoder.encode(param.getKey(), "UTF-8")).append("=") + .append(URLEncoder.encode(param.getValue(), "UTF-8")); + separator = "&"; + } + } + return new URL(sb.toString()); + } + + protected URL createURL(String resource, Map parameters) throws Exception { + return createURL(servletPath, resource, parameters); + } + + @SuppressWarnings("unchecked") + protected void runTest(String servletPath, Class servletClass, boolean securityEnabled, Callable assertions) + throws Exception { + runTest(new String[]{servletPath}, new Class[]{servletClass}, securityEnabled, assertions); + } + + protected void runTest(String[] servletPath, Class[] servletClass, boolean securityEnabled, + Callable assertions) throws Exception { + Services services = new Services(); + this.servletPath = servletPath[0]; + try { + services.init(); + services.getConf().setBoolean(AuthorizationService.CONF_SECURITY_ENABLED, securityEnabled); + services.get(AuthorizationService.class).init(services); + Services.get().setService(MockDagEngineService.class); + container = new EmbeddedServletContainer("oozie"); + for (int i = 0; i < servletPath.length; i++) { + container.addServletEndpoint(servletPath[i], servletClass[i]); + } + container.start(); + assertions.call(); + } + finally { + this.servletPath = null; + if (container != null) { + container.stop(); + } + services.destroy(); + container = null; + } + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/dag/servlet/MockDagEngineService.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/servlet/MockDagEngineService.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/servlet/MockDagEngineService.java (revision 0) @@ -0,0 +1,245 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.servlet; + +import org.apache.hadoop.conf.Configuration; +import org.apache.oozie.client.Workflow; +import org.apache.oozie.client.rest.JsonAction; +import org.apache.oozie.client.rest.JsonWorkflow; +import org.apache.oozie.client.rest.RestConstants; +import org.apache.oozie.dag.DagEngine; +import org.apache.oozie.dag.DagEngineException; +import org.apache.oozie.dag.WorkflowsInfo; +import org.apache.oozie.dag.WorkflowBean; +import org.apache.oozie.dag.service.DagEngineService; + +import java.io.IOException; +import java.io.Writer; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Properties; + +public class MockDagEngineService extends DagEngineService { + public static final String JOB_ID = "job-"; + public static final String ACTION_ID = "action-"; + public static final String EXT_ID = "ext-"; + public static final String WORKFLOW_APP = ""; + public static final String CONFIGURATION = ""; + public static final String GROUP = "group"; + public static final String USER = "user"; + + public static final String LOG = "log"; + + public static String did = null; + public static Properties properties; + public static List workflows; + public static List started; + public static final int INIT_WF_COUNT = 3; + + static { + reset(); + } + + public static void reset() { + did = null; + properties = null; + workflows = new ArrayList(); + started = new ArrayList(); + for(int i=0; i) (List) workflows, start, len, workflows.size()); + } + + public String getJobIdForExternalId(String externalId) throws DagEngineException { + did = RestConstants.JOBS_EXTERNAL_ID_PARAM; + return (externalId.equals("external-valid")) ? "id-valid" : null; + } + + private int validateWorkflowIdx(String jobId) throws DagEngineException { + int idx = -1; + try { + idx = Integer.parseInt(jobId.replace(JOB_ID, "")); + } + catch (Exception e) { + throw new DagEngineException(DagEngineException.ErrorCode.ETEST, jobId); + } + + if (idx >= workflows.size()) { + throw new DagEngineException(DagEngineException.ErrorCode.ETEST, jobId); + } + + return idx; + } + } + + private static Workflow createDummyWorkflow(int idx) { + JsonWorkflow workflow = new JsonWorkflow(); + workflow.setId(JOB_ID + idx); + workflow.setAppPath("hdfs://blah/blah/" + idx + "-blah"); + workflow.setStatus((idx % 2) == 0 ? Workflow.Status.RUNNING : Workflow.Status.SUCCEEDED); + workflow.setRun(idx); + workflow.setCreatedTime(new Date()); + workflow.setStartTime(new Date()); + workflow.setEndTime((idx % 2) == 0 ? null : (new Date())); + workflow.setConf(CONFIGURATION); + workflow.setAppName("workflow-" + idx); + workflow.setGroup(GROUP); + workflow.setUser(USER); + + List actions = new ArrayList(); + for (int i = 0; i < idx; i++) { + actions.add(createDummyAction(i)); + } + + workflow.setActions(actions); + return workflow; + } + + private static JsonAction createDummyAction(int idx) { + JsonAction action = new JsonAction(); + int mod = idx % 5; + action.setId(ACTION_ID + idx); + action.setExternalId(EXT_ID + idx); + action.setConsoleUrl("http://blah:blah/blah/" + idx); + action.setConf(""); + action.setData(null); + action.setStartTime(new Date()); + action.setEndTime(new Date()); + action.setErrorInfo(null, null); + action.setExternalStatus((idx % 2) == 0 ? "RUNNING" : "OK"); + action.setName(ACTION_ID + idx); + action.setRetries(idx); + action.setTrackerUri("http://trackerhost:blah/blah/" + idx); + action.setTransition("OK"); + switch (mod) { + case 0: { + action.setType("hadoop"); + break; + } + case 1: { + action.setType("fs"); + break; + } + case 2: { + action.setType("pig"); + break; + } + case 3: { + action.setType("ssh"); + break; + } + case 4: { + action.setType("decision"); + break; + } + } + + return action; + } +} Index: oozie/core/src/test/java/org/apache/oozie/dag/servlet/TestAdminServlet.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/servlet/TestAdminServlet.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/servlet/TestAdminServlet.java (revision 0) @@ -0,0 +1,220 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.servlet; + +import org.apache.oozie.client.rest.JsonTags; +import org.apache.oozie.client.rest.RestConstants; +import org.apache.oozie.service.Services; +import org.apache.oozie.BuildInfo; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; + +import javax.servlet.http.HttpServletResponse; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Callable; + +public class TestAdminServlet extends DagServletTestCase { + + static { + new AdminServlet(); + new JobServlet(); + } + private static final boolean IS_SECURITY_ENABLED = false; + + public void setUp() throws Exception { + super.setUp(); + } + + public void testStatus() throws Exception { + runTest("/admin/*", AdminServlet.class, IS_SECURITY_ENABLED, new Callable() { + public Void call() throws Exception { + URL url = createURL(RestConstants.ADMIN_STATUS_RESOURCE, Collections.EMPTY_MAP); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE)); + JSONObject json = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); + assertEquals(false, json.get(JsonTags.SYSTEM_SAFE_MODE)); + return null; + } + }); + } + + public void testOsEnv() throws Exception { + runTest("/admin/*", AdminServlet.class, IS_SECURITY_ENABLED, new Callable() { + public Void call() throws Exception { + URL url = createURL(RestConstants.ADMIN_OS_ENV_RESOURCE, Collections.EMPTY_MAP); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE)); + JSONObject json = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); + assertTrue(json.containsKey("USER")); + return null; + } + }); + } + + public void testJavaSysProps() throws Exception { + runTest("/admin/*", AdminServlet.class, IS_SECURITY_ENABLED, new Callable() { + public Void call() throws Exception { + URL url = createURL(RestConstants.ADMIN_JAVA_SYS_PROPS_RESOURCE, Collections.EMPTY_MAP); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE)); + JSONObject json = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); + assertTrue(json.containsKey("java.version")); + return null; + } + }); + } + + public void testConfiguration() throws Exception { + runTest("/admin/*", AdminServlet.class, IS_SECURITY_ENABLED, new Callable() { + public Void call() throws Exception { + URL url = createURL(RestConstants.ADMIN_CONFIG_RESOURCE, Collections.EMPTY_MAP); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE)); + JSONObject json = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); + assertTrue(json.containsKey(Services.CONF_SERVICE_CLASSES)); + return null; + } + }); + } + + public void testInstrumentation() throws Exception { + runTest("/admin/*", AdminServlet.class, IS_SECURITY_ENABLED, new Callable() { + public Void call() throws Exception { + URL url = createURL(RestConstants.ADMIN_INSTRUMENTATION_RESOURCE, Collections.EMPTY_MAP); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE)); + JSONObject json = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); + assertTrue(json.containsKey(JsonTags.INSTR_VARIABLES)); + return null; + } + }); + } + + public void testSafeMode() throws Exception { + runTest(new String[]{"/admin/*", "/job/*"}, new Class[]{AdminServlet.class, JobServlet.class}, + IS_SECURITY_ENABLED, new Callable() { + public Void call() throws Exception { + + MockDagEngineService.reset(); + Map params = new HashMap(); + params.put(RestConstants.ACTION_PARAM, RestConstants.JOB_ACTION_START); + URL url = createURL("/job/*", MockDagEngineService.JOB_ID+1, params); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("PUT"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + + MockDagEngineService.reset(); + url = createURL("/admin/*", RestConstants.ADMIN_STATUS_RESOURCE, Collections.EMPTY_MAP); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE)); + JSONObject json = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); + assertTrue(json.containsKey(JsonTags.SYSTEM_SAFE_MODE)); + assertFalse((Boolean)json.get(JsonTags.SYSTEM_SAFE_MODE)); + + + MockDagEngineService.reset(); + params = new HashMap(); + params.put(RestConstants.ADMIN_SAFE_MODE_PARAM, "true"); + url = createURL("/admin/*", RestConstants.ADMIN_STATUS_RESOURCE, params); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("PUT"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + + MockDagEngineService.reset(); + url = createURL("/admin/*", RestConstants.ADMIN_STATUS_RESOURCE, Collections.EMPTY_MAP); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE)); + json = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); + assertTrue(json.containsKey(JsonTags.SYSTEM_SAFE_MODE)); + assertTrue((Boolean)json.get(JsonTags.SYSTEM_SAFE_MODE)); + + MockDagEngineService.reset(); + params = new HashMap(); + params.put(RestConstants.ACTION_PARAM, RestConstants.JOB_ACTION_START); + url = createURL("/job/*", MockDagEngineService.JOB_ID+1, params); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("PUT"); + assertEquals(HttpServletResponse.SC_SERVICE_UNAVAILABLE, conn.getResponseCode()); + + MockDagEngineService.reset(); + params = new HashMap(); + params.put(RestConstants.ADMIN_SAFE_MODE_PARAM, "false"); + url = createURL("/admin/*", RestConstants.ADMIN_STATUS_RESOURCE, params); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("PUT"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + + MockDagEngineService.reset(); + url = createURL("/admin/*", RestConstants.ADMIN_STATUS_RESOURCE, Collections.EMPTY_MAP); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE)); + json = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); + assertTrue(json.containsKey(JsonTags.SYSTEM_SAFE_MODE)); + assertFalse((Boolean)json.get(JsonTags.SYSTEM_SAFE_MODE)); + + MockDagEngineService.reset(); + params = new HashMap(); + params.put(RestConstants.ACTION_PARAM, RestConstants.JOB_ACTION_START); + url = createURL("/job/*", MockDagEngineService.JOB_ID+1, params); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("PUT"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + + return null; + } + }); + } + + public void testVersion() throws Exception { + runTest("/admin/*", AdminServlet.class, IS_SECURITY_ENABLED, new Callable() { + public Void call() throws Exception { + URL url = createURL(RestConstants.ADMIN_BUILD_VERSION_RESOURCE, Collections.EMPTY_MAP); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode()); + assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE)); + JSONObject json = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream())); + assertEquals(BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION), + json.get(JsonTags.BUILD_VERSION)); + return null; + } + }); + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/dag/TestDagELFunctions.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/TestDagELFunctions.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/TestDagELFunctions.java (revision 0) @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag; + +import org.apache.oozie.client.WorkflowClient; +import org.apache.oozie.client.Action; +import org.apache.oozie.dag.workflow.lite.EndNodeDef; +import org.apache.oozie.dag.workflow.lite.LiteWorkflowApp; +import org.apache.oozie.dag.workflow.lite.LiteWorkflowInstance; +import org.apache.oozie.dag.workflow.lite.StartNodeDef; +import org.apache.oozie.service.ELService; +import org.apache.oozie.service.Services; +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.ELEvaluator; +import org.apache.oozie.util.XConfiguration; + +public class TestDagELFunctions extends XTestCase { + + public void testFunctions() throws Exception { + Services services = new Services(); + services.init(); + + XConfiguration conf = new XConfiguration(); + conf.set(WorkflowClient.APP_PATH, "appPath"); + conf.set(WorkflowClient.USER_NAME, "user"); + conf.set(WorkflowClient.GROUP_NAME, "group"); + conf.set("a", "A"); + LiteWorkflowApp def = + new LiteWorkflowApp("name", "", new StartNodeDef("end")).addNode(new EndNodeDef("end")); + LiteWorkflowInstance job = new LiteWorkflowInstance(def, conf, "wfId"); + + WorkflowBean wf = new WorkflowBean(); + wf.setId(job.getId()); + wf.setAppName("name"); + wf.setAppPath("appPath"); + wf.setUser("user"); + wf.setGroup("group"); + wf.setWorkflowInstance(job); + wf.setRun(2); + wf.setProtoActionConf(conf.toXmlString()); + + ActionBean action = new ActionBean(); + action.setId("actionId"); + action.setName("actionName"); + action.setErrorInfo("ec", "em"); + action.setData("b=B"); + action.setExternalId("ext"); + action.setTrackerUri("tracker"); + action.setExternalStatus("externalStatus"); + + ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(); + DagELFunctions.configureEvaluator(eval, wf, action); + + assertEquals("wfId", eval.evaluate("${wf:id()}", String.class)); + assertEquals("name", eval.evaluate("${wf:name()}", String.class)); + assertEquals("appPath", eval.evaluate("${wf:appPath()}", String.class)); + assertEquals("A", eval.evaluate("${wf:conf('a')}", String.class)); + assertEquals("A", eval.evaluate("${a}", String.class)); + assertEquals("user", eval.evaluate("${wf:user()}", String.class)); + assertEquals("group", eval.evaluate("${wf:group()}", String.class)); + assertTrue(eval.evaluate("${wf:callback('XX')}", String.class).contains("id=actionId")); + assertTrue(eval.evaluate("${wf:callback('XX')}", String.class).contains("status=XX")); + assertTrue(eval.evaluate("${wf:callback('XX')}", String.class).contains("status=XX")); + assertEquals(2, (int) eval.evaluate("${wf:run()}", Integer.class)); + + action.setStatus(Action.Status.ERROR); + DagELFunctions.setActionInfo(wf.getWorkflowInstance(), action); + + assertEquals("actionName", eval.evaluate("${wf:lastErrorNode()}", String.class)); + assertEquals("ec", eval.evaluate("${wf:errorCode('actionName')}", String.class)); + assertEquals("em", eval.evaluate("${wf:errorMessage('actionName')}", String.class)); + + assertEquals("B", eval.evaluate("${wf:actionData('actionName')['b']}", String.class)); + + assertEquals("ext", eval.evaluate("${wf:actionExternalId('actionName')}", String.class)); + assertEquals("tracker", eval.evaluate("${wf:actionTrackerUri('actionName')}", String.class)); + assertEquals("externalStatus", eval.evaluate("${wf:actionExternalStatus('actionName')}", String.class)); + + services.destroy(); + } +} Index: oozie/core/src/test/java/org/apache/oozie/dag/workflow/lite/TestLiteWorkflowLib.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/workflow/lite/TestLiteWorkflowLib.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/workflow/lite/TestLiteWorkflowLib.java (revision 0) @@ -0,0 +1,773 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.workflow.lite; + + +import org.apache.oozie.dag.workflow.WorkflowException; +import org.apache.oozie.dag.workflow.WorkflowInstance; +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.WritableUtils; +import org.apache.oozie.util.XConfiguration; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class TestLiteWorkflowLib extends XTestCase { + + static Map enters = new HashMap(); + static Map exits = new HashMap(); + static Map kills = new HashMap(); + static Map fails = new HashMap(); + + static int enterCounter = 0; + static int exitCounter = 0; + static int killCounter = 0; + static int failCounter = 0; + + + public static abstract class BaseNodeHandler extends NodeHandler { + private boolean synch; + + protected BaseNodeHandler(Boolean synch) { + this.synch = synch; + } + + @Override + public boolean enter(Context context) { + enters.put(context.getNodeDef().getName(), enterCounter++); + return synch; + } + + @Override + public String exit(Context context) { + exits.put(context.getNodeDef().getName(), exitCounter++); + return context.getNodeDef().getTransitions().get(0); + } + + @Override + public void kill(Context context) { + kills.put(context.getNodeDef().getName(), killCounter++); + } + + @Override + public void fail(Context context) { + fails.put(context.getNodeDef().getName(), failCounter++); + } + } + + public static class AsynchNodeHandler extends BaseNodeHandler { + + public AsynchNodeHandler() { + super(false); + } + } + + public static class SynchNodeHandler extends BaseNodeHandler { + + public SynchNodeHandler() { + super(true); + } + } + + public static class TestActionNodeHandler extends ActionNodeHandler { + + @Override + public void start(Context context) { + } + + @Override + public void end(Context context) { + } + } + + public static class TestDecisionNodeHandler extends DecisionNodeHandler { + @Override + public void start(Context context) { + enters.put(context.getNodeDef().getName(), enterCounter++); + } + + @Override + public void end(Context context) { + exits.put(context.getNodeDef().getName(), exitCounter++); + } + } + + public static class TestRootContextHandler extends SynchNodeHandler { + + @Override + public boolean enter(Context context) { + assertNotNull(context.getNodeDef()); + assertNotNull(context.getSignalValue()); + assertNotNull(context.getProcessInstance()); + assertEquals("/", context.getExecutionPath()); + assertEquals(null, context.getParentExecutionPath("/")); + assertEquals("A", context.getVar("a")); + assertEquals("AA", context.getTransientVar("ta")); + context.setVar("b", "B"); + context.setTransientVar("tb", "BB"); + return super.enter(context); + } + + @Override + public String exit(Context context) { + assertEquals("A", context.getVar("a")); + assertEquals("AA", context.getTransientVar("ta")); + context.setVar("b", "B"); + context.setTransientVar("tb", "BB"); + return super.exit(context); + } + } + + public static class TestForkedContextHandler extends SynchNodeHandler { + + @Override + public boolean enter(Context context) { + assertNotNull(context.getNodeDef()); + assertNotNull(context.getSignalValue()); + assertNotNull(context.getProcessInstance()); + assertEquals("/a/", context.getExecutionPath()); + assertEquals("/", context.getParentExecutionPath("/a/")); + return super.enter(context); + } + + } + + @Override + protected void setUp() throws Exception { + super.setUp(); + enters.clear(); + exits.clear(); + kills.clear(); + fails.clear(); + enterCounter = 0; + exitCounter = 0; + killCounter = 0; + failCounter = 0; + } + + public void testEmptyWorkflow() throws WorkflowException { + LiteWorkflowApp def = new LiteWorkflowApp("wf", "", new StartNodeDef("end")) + .addNode(new EndNodeDef("end")); + + LiteWorkflowInstance job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + assertEquals(WorkflowInstance.Status.PREP, job.getStatus()); + job.start(); + assertEquals(WorkflowInstance.Status.SUCCEEDED, job.getStatus()); + } + + public void testKillWorkflow() throws WorkflowException { + LiteWorkflowApp def = new LiteWorkflowApp("wf", "", new StartNodeDef("kill")) + .addNode(new KillNodeDef("kill", "killed")) + .addNode(new EndNodeDef("end")); + + LiteWorkflowInstance job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + assertEquals(WorkflowInstance.Status.PREP, job.getStatus()); + job.start(); + assertEquals(WorkflowInstance.Status.KILLED, job.getStatus()); + } + + public void testWorkflowStates() throws WorkflowException { + LiteWorkflowApp def = new LiteWorkflowApp("wf", "", new StartNodeDef("one")) + .addNode(new NodeDef("one", null, AsynchNodeHandler.class, Arrays.asList(new String[]{"end"}))) + .addNode(new EndNodeDef("end")); + + LiteWorkflowInstance job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + assertEquals(WorkflowInstance.Status.PREP, job.getStatus()); + + job.kill(); + assertEquals(WorkflowInstance.Status.KILLED, job.getStatus()); + + job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + job.fail("one"); + assertEquals(WorkflowInstance.Status.FAILED, job.getStatus()); + + job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + + try { + job.suspend(); + fail(); + } + catch (WorkflowException ex) { + //nop + } + + try { + job.resume(); + fail(); + } + catch (WorkflowException ex) { + //nop + } + + job.start(); + assertEquals(WorkflowInstance.Status.RUNNING, job.getStatus()); + + try { + job.resume(); + fail(); + } + catch (WorkflowException ex) { + //nop + } + + try { + job.start(); + fail(); + } + catch (WorkflowException ex) { + //nop + } + + job.suspend(); + assertEquals(WorkflowInstance.Status.SUSPENDED, job.getStatus()); + + try { + job.suspend(); + fail(); + } + catch (WorkflowException ex) { + //nop + } + + try { + job.start(); + fail(); + } + catch (WorkflowException ex) { + //nop + } + + job.resume(); + assertEquals(WorkflowInstance.Status.RUNNING, job.getStatus()); + + try { + job.resume(); + fail(); + } + catch (WorkflowException ex) { + //nop + } + + try { + job.start(); + fail(); + } + catch (WorkflowException ex) { + //nop + } + + job.kill(); + assertEquals(WorkflowInstance.Status.KILLED, job.getStatus()); + + try { + job.kill(); + fail(); + } + catch (WorkflowException ex) { + //nop + } + + try { + job.suspend(); + fail(); + } + catch (WorkflowException ex) { + //nop + } + + try { + job.resume(); + fail(); + } + catch (WorkflowException ex) { + //nop + } + + try { + job.start(); + fail(); + } + catch (WorkflowException ex) { + //nop + } + } + + public void testSynchSimple() throws WorkflowException { + LiteWorkflowApp def = new LiteWorkflowApp("wf", "", new StartNodeDef("one")) + .addNode(new NodeDef("one", null, SynchNodeHandler.class, Arrays.asList(new String[]{"end"}))) + .addNode(new EndNodeDef("end")); + + LiteWorkflowInstance job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + job.start(); + + assertEquals(WorkflowInstance.Status.SUCCEEDED, job.getStatus()); + assertEquals(1, enters.size()); + assertEquals(1, exits.size()); + assertEquals(0, kills.size()); + assertEquals(0, fails.size()); + } + + public void testNodeContext() throws WorkflowException { + LiteWorkflowApp def = new LiteWorkflowApp("wf", "", new StartNodeDef("one")) + .addNode(new NodeDef("one", null, TestRootContextHandler.class, Arrays.asList(new String[]{"end"}))) + .addNode(new EndNodeDef("end")); + + LiteWorkflowInstance job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + job.setVar("a", "A"); + job.setTransientVar("ta", "AA"); + job.start(); + + assertEquals(WorkflowInstance.Status.SUCCEEDED, job.getStatus()); + assertEquals("B", job.getVar("b")); + assertEquals("BB", job.getTransientVar("tb")); + assertEquals(1, enters.size()); + assertEquals(1, exits.size()); + assertEquals(0, kills.size()); + assertEquals(0, fails.size()); + } + + public void testSynchDouble() throws WorkflowException { + LiteWorkflowApp def = new LiteWorkflowApp("wf", "", new StartNodeDef("one")) + .addNode(new NodeDef("one", null, SynchNodeHandler.class, Arrays.asList(new String[]{"two"}))) + .addNode(new NodeDef("two", null, SynchNodeHandler.class, Arrays.asList(new String[]{"end"}))) + .addNode(new EndNodeDef("end")); + + LiteWorkflowInstance job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + job.start(); + + assertEquals(WorkflowInstance.Status.SUCCEEDED, job.getStatus()); + assertEquals(2, enters.size()); + assertEquals(2, exits.size()); + assertEquals(0, kills.size()); + assertEquals(0, fails.size()); + } + + public void testAsynchSimple() throws WorkflowException { + LiteWorkflowApp def = new LiteWorkflowApp("wf", "", new StartNodeDef("one")) + .addNode(new NodeDef("one", null, AsynchNodeHandler.class, Arrays.asList(new String[]{"end"}))) + .addNode(new EndNodeDef("end")); + + LiteWorkflowInstance job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + job.start(); + + assertEquals(WorkflowInstance.Status.RUNNING, job.getStatus()); + + job.signal("/", ""); + + assertEquals(WorkflowInstance.Status.SUCCEEDED, job.getStatus()); + assertEquals(1, enters.size()); + assertEquals(1, exits.size()); + assertEquals(0, kills.size()); + assertEquals(0, fails.size()); + } + + public void testInvalidExecutionPath() throws WorkflowException { + LiteWorkflowApp def = new LiteWorkflowApp("wf", "", new StartNodeDef("one")) + .addNode(new NodeDef("one", null, AsynchNodeHandler.class, Arrays.asList(new String[]{"end"}))) + .addNode(new EndNodeDef("end")); + + LiteWorkflowInstance job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + job.start(); + + assertEquals(WorkflowInstance.Status.RUNNING, job.getStatus()); + + job.signal("/a/", ""); + assertEquals(WorkflowInstance.Status.FAILED, job.getStatus()); + } + + public void testSimpleFork() throws WorkflowException { + + LiteWorkflowApp def = new LiteWorkflowApp("wf", "", new StartNodeDef("one")) + .addNode(new NodeDef("one", null, SynchNodeHandler.class, Arrays.asList(new String[]{"f"}))) + .addNode(new ForkNodeDef("f", Arrays.asList(new String[]{"two", "three"}))) + .addNode(new NodeDef("two", null, SynchNodeHandler.class, Arrays.asList(new String[]{"j"}))) + .addNode(new NodeDef("three", null, SynchNodeHandler.class, Arrays.asList(new String[]{"j"}))) + .addNode(new JoinNodeDef("j", "four")) + .addNode(new NodeDef("four", null, SynchNodeHandler.class, Arrays.asList(new String[]{"end"}))) + .addNode(new EndNodeDef("end")); + + LiteWorkflowInstance job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + job.start(); + + assertEquals(WorkflowInstance.Status.SUCCEEDED, job.getStatus()); + assertEquals(4, enters.size()); + assertEquals(4, exits.size()); + assertEquals(0, kills.size()); + assertEquals(0, fails.size()); + + assertTrue(enters.get("one") < enters.get("two")); + assertTrue(enters.get("one") < enters.get("three")); + assertTrue(enters.get("three") < enters.get("four")); + assertTrue(enters.get("two") < enters.get("four")); + } + + public void testForkedContext() throws WorkflowException { + + LiteWorkflowApp def = new LiteWorkflowApp("wf", "", new StartNodeDef("f")) + .addNode(new ForkNodeDef("f", Arrays.asList(new String[]{"a", "b"}))) + .addNode(new NodeDef("a", null, TestForkedContextHandler.class, Arrays.asList(new String[]{"j"}))) + .addNode(new NodeDef("b", null, SynchNodeHandler.class, Arrays.asList(new String[]{"j"}))) + .addNode(new JoinNodeDef("j", "end")) + .addNode(new EndNodeDef("end")); + + LiteWorkflowInstance job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + job.start(); + + assertEquals(WorkflowInstance.Status.SUCCEEDED, job.getStatus()); + } + + + public void testNestedFork() throws WorkflowException { + + LiteWorkflowApp def = new LiteWorkflowApp("testWf", "", new StartNodeDef("one")) + .addNode(new NodeDef("one", null, SynchNodeHandler.class, Arrays.asList(new String[]{"f"}))) + .addNode(new ForkNodeDef("f", Arrays.asList(new String[]{"two", "three"}))) + .addNode(new NodeDef("two", null, SynchNodeHandler.class, Arrays.asList(new String[]{"f2"}))) + .addNode(new NodeDef("three", null, SynchNodeHandler.class, Arrays.asList(new String[]{"j"}))) + .addNode(new ForkNodeDef("f2", Arrays.asList(new String[]{"four", "five", "six"}))) + .addNode(new NodeDef("four", null, SynchNodeHandler.class, Arrays.asList(new String[]{"j2"}))) + .addNode(new NodeDef("five", null, SynchNodeHandler.class, Arrays.asList(new String[]{"j2"}))) + .addNode(new NodeDef("six", null, SynchNodeHandler.class, Arrays.asList(new String[]{"j2"}))) + .addNode(new JoinNodeDef("j2", "seven")) + .addNode(new NodeDef("seven", null, SynchNodeHandler.class, Arrays.asList(new String[]{"j"}))) + .addNode(new JoinNodeDef("j", "end")) + .addNode(new EndNodeDef("end")); + + LiteWorkflowInstance job = new LiteWorkflowInstance(def, new XConfiguration(), "abcde"); + job.start(); + + assertEquals(WorkflowInstance.Status.SUCCEEDED, job.getStatus()); + assertEquals(7, enters.size()); + assertEquals(7, exits.size()); + assertEquals(0, kills.size()); + assertEquals(0, fails.size()); + + assertTrue(enters.get("one") < enters.get("two")); + assertTrue(enters.get("one") < enters.get("three")); + assertTrue(enters.get("two") < enters.get("four")); + + assertTrue(enters.get("four") < enters.get("seven")); + assertTrue(enters.get("five") < enters.get("seven")); + assertTrue(enters.get("six") < enters.get("seven")); + } + + + public void testKillWithRunningNodes() throws WorkflowException { + + LiteWorkflowApp def = new LiteWorkflowApp("wf", "", new StartNodeDef("f")) + .addNode(new ForkNodeDef("f", Arrays.asList(new String[]{"a", "b"}))) + .addNode(new NodeDef("a", null, SynchNodeHandler.class, Arrays.asList(new String[]{"j"}))) + .addNode(new NodeDef("b", null, AsynchNodeHandler.class, Arrays.asList(new String[]{"j"}))) + .addNode(new JoinNodeDef("j", "end")) + .addNode(new EndNodeDef("end")); + + LiteWorkflowInstance job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + job.start(); + assertEquals(WorkflowInstance.Status.RUNNING, job.getStatus()); + job.kill(); + assertEquals(2, enters.size()); + assertEquals(1, kills.size()); + assertEquals(1, exits.size()); + assertEquals(0, fails.size()); + } + + public void testFailWithRunningNodes() throws WorkflowException { + + LiteWorkflowApp def = new LiteWorkflowApp("wf", "", new StartNodeDef("f")) + .addNode(new ForkNodeDef("f", Arrays.asList(new String[] { "a", "b" }))) + .addNode(new NodeDef("a", null, SynchNodeHandler.class, Arrays.asList(new String[] { "j" }))) + .addNode(new NodeDef("b", null, AsynchNodeHandler.class, Arrays.asList(new String[] { "j" }))) + .addNode(new JoinNodeDef("j", "end")).addNode(new EndNodeDef("end")); + + + LiteWorkflowInstance job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + job.start(); + assertEquals(WorkflowInstance.Status.RUNNING, job.getStatus()); + job.fail("b"); + assertEquals(2, enters.size()); + assertEquals(0, kills.size()); + assertEquals(1, exits.size()); + assertEquals(1, fails.size()); + } + + + public void testDoneWithRunningNodes() throws WorkflowException { + + LiteWorkflowApp def = new LiteWorkflowApp("wf", "", new StartNodeDef("f")) + .addNode(new ForkNodeDef("f", Arrays.asList(new String[]{"a", "b"}))) + .addNode(new NodeDef("a", null, AsynchNodeHandler.class, Arrays.asList(new String[]{"j"}))) + .addNode(new NodeDef("b", null, AsynchNodeHandler.class, Arrays.asList(new String[]{"end"}))) + .addNode(new JoinNodeDef("j", "end")) + .addNode(new EndNodeDef("end")); + + LiteWorkflowInstance job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + job.start(); + assertEquals(WorkflowInstance.Status.RUNNING, job.getStatus()); + job.signal("/b/", ""); + assertEquals(2, enters.size()); + assertEquals(1, kills.size()); + assertEquals(1, exits.size()); + assertEquals(0, fails.size()); + } + + public void testWFKillWithRunningNodes() throws WorkflowException { + + LiteWorkflowApp def = new LiteWorkflowApp("wf", "", new StartNodeDef("f")) + .addNode(new ForkNodeDef("f", Arrays.asList(new String[]{"a", "b"}))) + .addNode(new NodeDef("a", null, AsynchNodeHandler.class, Arrays.asList(new String[]{"j"}))) + .addNode(new NodeDef("b", null, AsynchNodeHandler.class, Arrays.asList(new String[]{"kill"}))) + .addNode(new JoinNodeDef("j", "end")) + .addNode(new KillNodeDef("kill", "killed")) + .addNode(new EndNodeDef("end")); + + LiteWorkflowInstance job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + job.start(); + assertEquals(WorkflowInstance.Status.RUNNING, job.getStatus()); + job.signal("/b/", ""); + assertEquals(2, enters.size()); + assertEquals(1, kills.size()); + assertEquals(1, exits.size()); + assertEquals(0, fails.size()); + } + + public void testWfFailWithRunningNodes() throws WorkflowException { + + LiteWorkflowApp def = new LiteWorkflowApp("wf", "", new StartNodeDef("f")) + .addNode(new ForkNodeDef("f", Arrays.asList(new String[]{"a", "b"}))) + .addNode(new NodeDef("a", null, AsynchNodeHandler.class, Arrays.asList(new String[]{"j"}))) + .addNode(new NodeDef("b", null, AsynchNodeHandler.class, Arrays.asList(new String[]{"x"}))) + .addNode(new JoinNodeDef("j", "end")) + .addNode(new EndNodeDef("end")); + + LiteWorkflowInstance job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + try { + job.start(); + job.signal("/b/", ""); + } + catch (WorkflowException ex) { + } + assertEquals(WorkflowInstance.Status.FAILED, job.getStatus()); + assertEquals(2, enters.size()); + assertEquals(1, fails.size()); + //assertEquals(1, kills.size()); + assertEquals(1, exits.size()); + assertEquals(1, fails.size()); + } + + public void testDecision() throws WorkflowException { + List decTrans = new ArrayList(); + decTrans.add("one"); + decTrans.add("two"); + decTrans.add("three"); + LiteWorkflowApp def = new LiteWorkflowApp("testWf", "", new StartNodeDef("d")) + .addNode(new DecisionNodeDef("d", "", TestDecisionNodeHandler.class, decTrans)) + .addNode(new NodeDef("one", null, SynchNodeHandler.class, Arrays.asList(new String[]{"end"}))) + .addNode(new NodeDef("two", null, SynchNodeHandler.class, Arrays.asList(new String[]{"end"}))) + .addNode(new NodeDef("three", null, SynchNodeHandler.class, Arrays.asList(new String[]{"end"}))) + .addNode(new EndNodeDef("end")); + + LiteWorkflowInstance job = new LiteWorkflowInstance(def, new XConfiguration(), "abcde"); + job.start(); + + assertEquals(WorkflowInstance.Status.RUNNING, job.getStatus()); + job.signal("/", "one"); + + assertEquals(WorkflowInstance.Status.SUCCEEDED, job.getStatus()); + assertEquals(2, enters.size()); + assertEquals(2, exits.size()); + assertTrue(enters.containsKey("one")); + assertTrue(!enters.containsKey("two")); + assertTrue(!enters.containsKey("three")); + + enters.clear(); + job = new LiteWorkflowInstance(def, new XConfiguration(), "abcde"); + job.start(); + + assertEquals(WorkflowInstance.Status.RUNNING, job.getStatus()); + job.signal("/", "two"); + + assertEquals(WorkflowInstance.Status.SUCCEEDED, job.getStatus()); + assertTrue(!enters.containsKey("one")); + assertTrue(enters.containsKey("two")); + assertTrue(!enters.containsKey("three")); + + enters.clear(); + job = new LiteWorkflowInstance(def, new XConfiguration(), "abcde"); + job.start(); + + assertEquals(WorkflowInstance.Status.RUNNING, job.getStatus()); + job.signal("/", "three"); + + assertEquals(WorkflowInstance.Status.SUCCEEDED, job.getStatus()); + assertTrue(!enters.containsKey("one")); + assertTrue(!enters.containsKey("two")); + assertTrue(enters.containsKey("three")); + + enters.clear(); + job = new LiteWorkflowInstance(def, new XConfiguration(), "abcde"); + job.start(); + + assertEquals(WorkflowInstance.Status.RUNNING, job.getStatus()); + try { + job.signal("/", "bla"); + fail(); + } + catch (Exception e) { + } + assertEquals(WorkflowInstance.Status.FAILED, job.getStatus()); + + } + + + public void testActionOKError() throws WorkflowException { + LiteWorkflowApp def = new LiteWorkflowApp("wf", "", new StartNodeDef("a")) + .addNode(new ActionNodeDef("a", "", TestActionNodeHandler.class, "b", "c")) + .addNode(new NodeDef("b", null, SynchNodeHandler.class, Arrays.asList(new String[]{"end"}))) + .addNode(new NodeDef("c", null, SynchNodeHandler.class, Arrays.asList(new String[]{"end"}))) + .addNode(new EndNodeDef("end")); + + LiteWorkflowInstance job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + + job.start(); + assertEquals(WorkflowInstance.Status.RUNNING, job.getStatus()); + + job.signal("/", "OK"); + assertEquals(WorkflowInstance.Status.SUCCEEDED, job.getStatus()); + assertTrue(enters.containsKey("b")); + assertTrue(!enters.containsKey("c")); + + enters.clear(); + job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + + job.start(); + assertEquals(WorkflowInstance.Status.RUNNING, job.getStatus()); + + job.signal("/", "ERROR"); + assertEquals(WorkflowInstance.Status.SUCCEEDED, job.getStatus()); + assertTrue(!enters.containsKey("b")); + assertTrue(enters.containsKey("c")); + + } + + public void testJobPersistance() throws WorkflowException { + LiteWorkflowApp def = new LiteWorkflowApp("wf", "", new StartNodeDef("one")) + .addNode(new NodeDef("one", null, AsynchNodeHandler.class, Arrays.asList(new String[]{"end"}))) + .addNode(new EndNodeDef("end")); + + LiteWorkflowInstance job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + job.setVar("a", "A"); + job.setTransientVar("b", "B"); + assertEquals(WorkflowInstance.Status.PREP, job.getStatus()); + assertEquals("A", job.getVar("a")); + assertEquals("B", job.getTransientVar("b")); + assertEquals("1", job.getId()); + + byte[] array = WritableUtils.toByteArray(job); + job = WritableUtils.fromByteArray(array, LiteWorkflowInstance.class); + assertEquals(WorkflowInstance.Status.PREP, job.getStatus()); + assertEquals("A", job.getVar("a")); + assertEquals(null, job.getTransientVar("b")); + assertEquals("1", job.getId()); + + job.start(); + assertEquals(WorkflowInstance.Status.RUNNING, job.getStatus()); + array = WritableUtils.toByteArray(job); + job = WritableUtils.fromByteArray(array, LiteWorkflowInstance.class); + assertEquals(WorkflowInstance.Status.RUNNING, job.getStatus()); + job.signal("/", ""); + assertEquals(WorkflowInstance.Status.SUCCEEDED, job.getStatus()); + } + + + public void testImmediateError() throws WorkflowException { + LiteWorkflowApp workflowDef = new LiteWorkflowApp("testWf", "", new StartNodeDef("one")) + .addNode(new NodeDef("one", null, SynchNodeHandler.class, Arrays.asList(new String[]{"two"}))) + .addNode(new NodeDef("two", null, SynchNodeHandler.class, Arrays.asList(new String[]{"four"}))) + .addNode(new NodeDef("three", null, SynchNodeHandler.class, Arrays.asList(new String[]{"end"}))) + .addNode(new EndNodeDef("end")); + + LiteWorkflowInstance workflowJob = new LiteWorkflowInstance(workflowDef, new XConfiguration(), "abcde"); + try { + workflowJob.start(); + } + catch (WorkflowException e) { + } + + assertEquals(WorkflowInstance.Status.FAILED, workflowJob.getStatus()); + assertEquals(2, enters.size()); + assertEquals(2, exits.size()); + assertEquals(0, kills.size()); + assertEquals(0, fails.size()); + + } + + public void testSelfTransition() throws WorkflowException { + try { + new LiteWorkflowApp("wf", "", new StartNodeDef("one")) + .addNode(new NodeDef("one", null, SynchNodeHandler.class, Arrays.asList(new String[]{"one"}))) + .addNode(new EndNodeDef("end")); + fail(); + } + catch (WorkflowException ex) { + assertEquals(WorkflowException.ErrorCode.E1306, ex.getErrorCode()); + } + } + + public void testLoopSimple() throws WorkflowException { + LiteWorkflowApp def = new LiteWorkflowApp("wf", "", new StartNodeDef("one")) + .addNode(new NodeDef("one", null, SynchNodeHandler.class, Arrays.asList(new String[]{"two"}))) + .addNode(new NodeDef("two", null, SynchNodeHandler.class, Arrays.asList(new String[]{"one"}))) + .addNode(new EndNodeDef("end")); + + LiteWorkflowInstance job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + try { + job.start(); + fail(); + } + catch (WorkflowException ex) { + assertEquals(WorkflowException.ErrorCode.E1309, ex.getErrorCode()); + } + assertEquals(WorkflowInstance.Status.FAILED, job.getStatus()); + } + + public void testLoopFork() throws WorkflowException { + + LiteWorkflowApp def = new LiteWorkflowApp("wf", "", new StartNodeDef("one")) + .addNode(new NodeDef("one", null, SynchNodeHandler.class, Arrays.asList(new String[]{"f"}))) + .addNode(new ForkNodeDef("f", Arrays.asList(new String[]{"two", "three"}))) + .addNode(new NodeDef("two", null, SynchNodeHandler.class, Arrays.asList(new String[]{"j"}))) + .addNode(new NodeDef("three", null, SynchNodeHandler.class, Arrays.asList(new String[]{"j"}))) + .addNode(new JoinNodeDef("j", "four")) + .addNode(new NodeDef("four", null, SynchNodeHandler.class, Arrays.asList(new String[]{"f"}))) + .addNode(new EndNodeDef("end")); + + LiteWorkflowInstance job = new LiteWorkflowInstance(def, new XConfiguration(), "1"); + try { + job.start(); + fail(); + } + catch (WorkflowException ex) { + assertEquals(WorkflowException.ErrorCode.E1309, ex.getErrorCode()); + } + assertEquals(WorkflowInstance.Status.FAILED, job.getStatus()); + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/dag/workflow/lite/TestLiteWorkflowAppParser.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/workflow/lite/TestLiteWorkflowAppParser.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/workflow/lite/TestLiteWorkflowAppParser.java (revision 0) @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.workflow.lite; + +import org.apache.oozie.dag.service.LiteWorkflowStoreService; +import org.apache.oozie.dag.workflow.WorkflowException; +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.IOUtils; + +public class TestLiteWorkflowAppParser extends XTestCase { + + public void testParser() throws Exception { + LiteWorkflowAppParser parser = new LiteWorkflowAppParser(null, + LiteWorkflowStoreService.LiteDecisionHandler.class, + LiteWorkflowStoreService.LiteActionHandler.class); + + parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid.xml", -1)); + + try { + parser.validateAndParse(IOUtils.getResourceAsReader("wf-loop1-invalid.xml", -1)); + fail(); + } + catch (WorkflowException ex) { + assertEquals(WorkflowException.ErrorCode.E1307, ex.getErrorCode()); + } + catch (Exception ex) { + fail(); + } + + try { + parser.validateAndParse(IOUtils.getResourceAsReader("wf-loop2-invalid.xml", -1)); + fail(); + } + catch (WorkflowException ex) { + assertEquals(WorkflowException.ErrorCode.E1306, ex.getErrorCode()); + } + catch (Exception ex) { + fail(); + } + + try { + parser.validateAndParse(IOUtils.getResourceAsReader("wf-transition-invalid.xml", -1)); + fail(); + } + catch (WorkflowException ex) { + assertEquals(WorkflowException.ErrorCode.E1308, ex.getErrorCode()); + } + catch (Exception ex) { + fail(); + } + } + +} Index: oozie/core/src/test/java/org/apache/oozie/dag/store/TestDBWorkflowStore.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/store/TestDBWorkflowStore.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/store/TestDBWorkflowStore.java (revision 0) @@ -0,0 +1,488 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.store; + +import static org.apache.oozie.dag.store.OozieSchema.OozieColumn.ACTIONS_id; +import static org.apache.oozie.dag.store.OozieSchema.OozieColumn.ACTIONS_wfId; +import static org.apache.oozie.dag.store.OozieSchema.OozieColumn.WF_id; +import static org.apache.oozie.util.db.SqlStatement.getCount; +import static org.apache.oozie.util.db.SqlStatement.isEqual; + +import org.apache.oozie.util.XLog; + +import org.apache.oozie.util.db.Schema; +import org.apache.oozie.util.db.Schema.DBType; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Date; +import java.util.List; +import java.util.HashMap; +import java.util.Map; +import java.util.Arrays; + +import org.apache.hadoop.conf.Configuration; +import org.apache.oozie.client.Action; +import org.apache.oozie.client.Workflow; +import org.apache.oozie.client.WorkflowClient; +import org.apache.oozie.dag.ActionBean; +import org.apache.oozie.dag.WorkflowBean; +import org.apache.oozie.dag.WorkflowsInfo; +import org.apache.oozie.dag.service.WorkflowStoreService; +import org.apache.oozie.dag.service.WorkflowAppService; +import org.apache.oozie.dag.store.OozieSchema.OozieTable; +import org.apache.oozie.dag.workflow.WorkflowApp; +import org.apache.oozie.dag.workflow.WorkflowLib; +import org.apache.oozie.dag.workflow.WorkflowInstance; +import org.apache.oozie.dag.workflow.lite.EndNodeDef; +import org.apache.oozie.dag.workflow.lite.LiteWorkflowApp; +import org.apache.oozie.dag.workflow.lite.StartNodeDef; +import org.apache.oozie.service.DataSourceService; +import org.apache.oozie.service.Services; +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.db.SqlStatement; +import org.apache.oozie.util.XmlUtils; + +public class TestDBWorkflowStore extends XTestCase { + Connection conn; + WorkflowLib wfLib; + WorkflowStore store; + WorkflowBean wfBean1; + WorkflowBean wfBean2; + String dbName; + Services services; + + @Override + protected void setUp() throws Exception { + super.setUp(); + services = new Services(); + cleanUpDB(services.getConf()); + services.init(); + store = Services.get().get(WorkflowStoreService.class).create(); + conn = Services.get().get(DataSourceService.class).getRawConnection(); + conn.setAutoCommit(false); + conn.setTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + dropSchema(dbName, conn); + services.destroy(); + } + + public void testDBWorkflowStore() throws Exception { + _testInsertWF(); + _testGetWF(); + _testUpdateWF(); + _testWaitWriteLock(); + _testGetWFIDWithExtID(); + _testSaveAction(); + _testLoadAction(); + _testUpdateAction(); + _testDeleteAction(); + _testGetActionsForWF(); + _testGetPendingActions(); + _testGetWFInfo(); + _testGetWFInfos(); + _testPurge(); + } + + private WorkflowBean createWorkflow(WorkflowApp app, Configuration conf, String authToken) throws Exception { + WorkflowAppService wps = Services.get().get(WorkflowAppService.class); + Configuration protoActionConf = wps.createProtoActionConf(conf, authToken); + WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB(); + WorkflowInstance wfInstance; + wfInstance = workflowLib.createInstance(app, conf); + WorkflowBean workflow = new WorkflowBean(); + workflow.setId(wfInstance.getId()); + workflow.setAppName(app.getName()); + workflow.setAppPath(conf.get(WorkflowClient.APP_PATH)); + workflow.setConf(XmlUtils.prettyPrint(conf).toString()); + workflow.setProtoActionConf(XmlUtils.prettyPrint(protoActionConf).toString()); + workflow.setCreatedTime(new Date()); + workflow.setLogToken(conf.get(WorkflowClient.LOG_TOKEN, "")); + workflow.setStatus(Workflow.Status.PREP); + workflow.setRun(0); + workflow.setUser(conf.get(WorkflowClient.USER_NAME)); + workflow.setGroup(conf.get(WorkflowClient.GROUP_NAME)); + workflow.setAuthToken(authToken); + workflow.setWorkflowInstance(wfInstance); + return workflow; + } + + private void _testInsertWF() throws Exception { + WorkflowApp app = new LiteWorkflowApp("testApp", "", new StartNodeDef("end")) + .addNode(new EndNodeDef("end")); + Configuration conf1 = new Configuration(); + + conf1.set(WorkflowClient.APP_PATH, "testPath"); + conf1.set(WorkflowClient.LOG_TOKEN, "testToken"); + conf1.set(WorkflowClient.USER_NAME, "testUser1"); + conf1.set(WorkflowClient.GROUP_NAME, "testGroup1"); + wfBean1 = createWorkflow(app, conf1, "auth"); + + Configuration conf2 = new Configuration(); + conf2.set(WorkflowClient.APP_PATH, "testPath"); + conf2.set(WorkflowClient.LOG_TOKEN, "testToken"); + conf2.set(WorkflowClient.USER_NAME, "testUser2"); + conf2.set(WorkflowClient.GROUP_NAME, "testGroup2"); + wfBean2 = createWorkflow(app, conf2, "auth"); + + store.insertWorkflow(wfBean1); + store.insertWorkflow(wfBean2); + store.commit(); + + SqlStatement s = getCount(OozieTable.WORKFLOWS); + ResultSet rs = s.prepareAndSetValues(conn).executeQuery(); + rs.next(); + assertEquals(2, rs.getInt(1)); + rs.close(); + + s = getCount(OozieTable.WORKFLOWS).where(isEqual(WF_id, wfBean1.getId())); + rs = s.prepareAndSetValues(conn).executeQuery(); + rs.next(); + assertEquals(1, rs.getInt(1)); + rs.close(); + + s = getCount(OozieTable.WORKFLOWS).where(isEqual(WF_id, wfBean2.getId())); + rs = s.prepareAndSetValues(conn).executeQuery(); + rs.next(); + assertEquals(1, rs.getInt(1)); + rs.close(); + + } + + private void _testGetWF() throws StoreException { + WorkflowBean wfBean = store.getWorkflow(wfBean1.getId(), false); + assertEquals(wfBean.getId(), wfBean1.getId()); + assertEquals(wfBean.getStatus(), Workflow.Status.PREP); + assertEquals(wfBean.getWorkflowInstance().getId(), wfBean1.getId()); + } + + private void _testUpdateWF() throws StoreException { + wfBean1.setStatus(Workflow.Status.SUCCEEDED); + wfBean1.getWorkflowInstance().setVar("test", "hello"); + wfBean1.setExternalId("testExtId"); + store.getWorkflow(wfBean1.getId(), true); + store.updateWorkflow(wfBean1); + store.commit(); + WorkflowBean wfBean = store.getWorkflow(wfBean1.getId(), false); + assertEquals("hello", wfBean.getWorkflowInstance().getVar("test")); + assertEquals(wfBean.getStatus(), Workflow.Status.SUCCEEDED); + } + + public class Locker implements Runnable { + protected String id; + private String nameIndex; + private StringBuffer sb; + protected long timeout; + + public Locker(String id, String nameIndex, StringBuffer buffer) { + this.id = id; + this.nameIndex = id + ":" + nameIndex; + this.sb = buffer; + } + + public void run() { + XLog log = XLog.getLog(getClass()); + try { + WorkflowStore store = Services.get().get(WorkflowStoreService.class).create(); + log.info("Get [{0}]", nameIndex); + store.getWorkflow(id, true); + log.info("Got [{0}]", nameIndex); + sb.append(nameIndex + "-L "); + synchronized (this) { + wait(); + } + sb.append(nameIndex + "-U "); + store.close(); + log.info("Release [{0}]", nameIndex); + } + catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + public void finish() { + synchronized (this) { + notify(); + } + } + } + + public void _testWaitWriteLock() throws Exception { + StringBuffer sb = new StringBuffer(""); + String id = wfBean1.getId(); + Locker l1 = new Locker(id, "1", sb); + Locker l2 = new Locker(id, "2", sb); + + new Thread(l1).start(); + Thread.sleep(300); + new Thread(l2).start(); + Thread.sleep(300); + l1.finish(); + Thread.sleep(1000); + l2.finish(); + Thread.sleep(1000); + assertEquals(id + ":1-L " + id + ":1-U " + id + ":2-L " + id + ":2-U", sb.toString().trim()); + } + + private void _testGetWFIDWithExtID() throws StoreException { + String id = store.getWorkflowIdForExternalId("testExtId"); + assertEquals(wfBean1.getId(), id); + } + + private void _testSaveAction() throws StoreException, SQLException { + ActionBean a11 = new ActionBean(); + a11.setId("11"); + a11.setJobId(wfBean1.getId()); + a11.setStatus(Action.Status.PREP); + + ActionBean a12 = new ActionBean(); + a12.setId("12"); + a12.setJobId(wfBean1.getId()); + a12.setStatus(Action.Status.PREP); + + ActionBean a21 = new ActionBean(); + a21.setId("21"); + a21.setJobId(wfBean2.getId()); + a21.setStatus(Action.Status.PREP); + + ActionBean a22 = new ActionBean(); + a22.setId("22"); + a22.setJobId(wfBean2.getId()); + a22.setStatus(Action.Status.PREP); + + store.insertAction(a11); + store.insertAction(a12); + store.insertAction(a21); + store.insertAction(a22); + store.commit(); + + SqlStatement s = getCount(OozieTable.ACTIONS); + ResultSet rs = s.prepareAndSetValues(conn).executeQuery(); + rs.next(); + assertEquals(4, rs.getInt(1)); + rs.close(); + + s = getCount(OozieTable.ACTIONS).where(isEqual(ACTIONS_wfId, wfBean1.getId())); + rs = s.prepareAndSetValues(conn).executeQuery(); + rs.next(); + assertEquals(2, rs.getInt(1)); + rs.close(); + + s = getCount(OozieTable.ACTIONS).where(isEqual(ACTIONS_wfId, wfBean2.getId())); + rs = s.prepareAndSetValues(conn).executeQuery(); + rs.next(); + assertEquals(2, rs.getInt(1)); + rs.close(); + + s = getCount(OozieTable.ACTIONS).where(isEqual(ACTIONS_id, "11")); + rs = s.prepareAndSetValues(conn).executeQuery(); + rs.next(); + assertEquals(1, rs.getInt(1)); + rs.close(); + + s = getCount(OozieTable.ACTIONS).where(isEqual(ACTIONS_id, "12")); + rs = s.prepareAndSetValues(conn).executeQuery(); + rs.next(); + assertEquals(1, rs.getInt(1)); + rs.close(); + } + + private void _testLoadAction() throws StoreException { + ActionBean a11 = store.getAction("11", false); + assertEquals(a11.getId(), "11"); + assertEquals(a11.getJobId(), wfBean1.getId()); + assertEquals(a11.getStatus(), Action.Status.PREP); + } + + private void _testUpdateAction() throws StoreException { + ActionBean a11 = store.getAction("11", false); + a11.setStatus(Action.Status.OK); + a11.setPending(); + a11.setPendingAge(new Date(System.currentTimeMillis() - 10000)); + store.updateAction(a11); + store.commit(); + ActionBean a = store.getAction(a11.getId(), false); + assertEquals(a.getId(), a11.getId()); + assertEquals(a.getStatus(), Action.Status.OK); + } + + private void _testDeleteAction() throws StoreException { + store.deleteAction("12"); + store.commit(); + boolean actionDeleted = false; + try { + store.getAction("12", false); + } + catch (StoreException e) { + if (StoreException.ErrorCode.E1205.equals(e.getErrorCode())) { + actionDeleted = true; + } + } + assertEquals(true, actionDeleted); + } + + private void _testGetActionsForWF() throws StoreException { + List actions1 = store.getActionsForWorkflow(wfBean1.getId(), false); + assertEquals(actions1.size(), 1); + List actions2 = store.getActionsForWorkflow(wfBean2.getId(), false); + assertEquals(actions2.size(), 2); + } + + private void _testGetPendingActions() throws StoreException { + List pActions = store.getPendingActions(5); + assertEquals(1, pActions.size()); + assertEquals("11", pActions.get(0).getId()); + } + + private void _testGetWFInfo() throws StoreException { + WorkflowBean wfBean = store.getWorkflowInfo(wfBean1.getId()); + assertEquals(wfBean.getId(), wfBean1.getId()); + assertEquals(wfBean.getStatus(), wfBean1.getStatus()); + assertEquals(wfBean.getActions().size(), 1); + assertEquals(wfBean.getActions().get(0).getId(), "11"); + } + + private void _testGetWFInfos() throws StoreException { + Map> filter = new HashMap>(); + WorkflowsInfo wfInfo = store.getWorkflowsInfo(filter, 1, 1); + List wfBeans = wfInfo.getWorkflows(); + + assertEquals(1, wfBeans.size()); + + filter = new HashMap>(); + wfInfo = store.getWorkflowsInfo(filter, 1, 2); + wfBeans = wfInfo.getWorkflows(); + assertEquals(2, wfBeans.size()); + + filter = new HashMap>(); + filter.put("user", Arrays.asList("testUser1")); + wfInfo = store.getWorkflowsInfo(filter, 1, 2); + wfBeans = wfInfo.getWorkflows(); + assertEquals(1, wfBeans.size()); + + filter = new HashMap>(); + filter.put("user", Arrays.asList("testUser1", "testUser2")); + wfInfo = store.getWorkflowsInfo(filter, 1, 2); + wfBeans = wfInfo.getWorkflows(); + assertEquals(2, wfBeans.size()); + + filter = new HashMap>(); + filter.put("user", Arrays.asList("testUser1")); + filter.put("status", Arrays.asList("succeeded")); + + wfInfo = store.getWorkflowsInfo(filter, 1, 2); + wfBeans = wfInfo.getWorkflows(); + assertEquals(1, wfBeans.size()); + + filter = new HashMap>(); + filter.put("user", Arrays.asList("testUser1", "testUser2")); + filter.put("name", Arrays.asList("testApp")); + wfInfo = store.getWorkflowsInfo(filter, 1, 2); + wfBeans = wfInfo.getWorkflows(); + assertEquals(2, wfBeans.size()); + assertEquals(2, wfInfo.getTotal()); + assertEquals(1, wfInfo.getStart()); + assertEquals(2, wfInfo.getLen()); + + filter = new HashMap>(); + filter.put("user", Arrays.asList("testUser1", "testUser2")); + filter.put("name", Arrays.asList("testApp")); + wfInfo = store.getWorkflowsInfo(filter, 1, 1); + wfBeans = wfInfo.getWorkflows(); + assertEquals(1, wfBeans.size()); + assertEquals(2, wfInfo.getTotal()); + assertEquals(1, wfInfo.getStart()); + assertEquals(1, wfInfo.getLen()); + } + + private void _testPurge() throws Exception { + wfBean1.setEndTime(new Date(System.currentTimeMillis() - (31 * 24 * 60 * 60 * 1000l))); + wfBean2.setEndTime(new Date(System.currentTimeMillis() - (31 * 24 * 60 * 60 * 1000l))); + WorkflowApp app = new LiteWorkflowApp("testApp", "", new StartNodeDef("end")) + .addNode(new EndNodeDef("end")); + Configuration conf2 = new Configuration(); + conf2.set(WorkflowClient.APP_PATH, "testPath"); + conf2.set(WorkflowClient.LOG_TOKEN, "testToken"); + conf2.set(WorkflowClient.USER_NAME, "testUser2"); + conf2.set(WorkflowClient.GROUP_NAME, "testGroup2"); + WorkflowBean wfBean3 = createWorkflow(app, conf2, "auth"); + + store.insertWorkflow(wfBean3); + ActionBean a31 = new ActionBean(); + a31.setId("31"); + a31.setJobId(wfBean3.getId()); + a31.setStatus(Action.Status.PREP); + store.insertAction(a31); + store.updateWorkflow(wfBean1); + store.updateWorkflow(wfBean2); + store.commit(); + + SqlStatement s = getCount(OozieTable.WORKFLOWS); + ResultSet rs = s.prepareAndSetValues(conn).executeQuery(); + rs.next(); + assertEquals(3, rs.getInt(1)); + rs.close(); + + s = getCount(OozieTable.ACTIONS); + rs = s.prepareAndSetValues(conn).executeQuery(); + rs.next(); + assertEquals(4, rs.getInt(1)); + rs.close(); + + store.purge(30); + store.commit(); + + s = getCount(OozieTable.WORKFLOWS); + rs = s.prepareAndSetValues(conn).executeQuery(); + rs.next(); + assertEquals(1, rs.getInt(1)); + rs.close(); + + WorkflowBean tmp = store.getWorkflow(wfBean3.getId(), false); + assertEquals(tmp.getId(), wfBean3.getId()); + + s = getCount(OozieTable.ACTIONS); + rs = s.prepareAndSetValues(conn).executeQuery(); + rs.next(); + assertEquals(1, rs.getInt(1)); + rs.close(); + + ActionBean tmpa = store.getAction("31", false); + assertEquals("31", tmpa.getId()); + } + + private static void dropSchema(String dbName, Connection conn) { + try { + DBType type = DBType.MySQL; + if (Schema.isHsqlConnection(conn)) { + type = DBType.HSQL; + } + conn.prepareStatement( + "DROP " + (type.equals(DBType.MySQL) ? "DATABASE " : "SCHEMA ") + dbName + + (type.equals(DBType.HSQL) ? " CASCADE" : "")).execute(); + } + catch (SQLException e) { + + } + } +} Index: oozie/core/src/test/java/org/apache/oozie/dag/action/oozie/TestSubWorkflowActionExecutor.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/action/oozie/TestSubWorkflowActionExecutor.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/action/oozie/TestSubWorkflowActionExecutor.java (revision 0) @@ -0,0 +1,206 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.action.oozie; + +import org.apache.oozie.dag.action.hadoop.HadoopActionExecutorTestCase; +import org.apache.oozie.dag.WorkflowBean; +import org.apache.oozie.dag.ActionBean; +import org.apache.oozie.util.XConfiguration; +import org.apache.oozie.client.WorkflowClient; +import org.apache.oozie.client.Action; +import org.apache.oozie.client.Workflow; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FileSystem; + +import java.io.StringReader; +import java.io.Writer; +import java.io.OutputStreamWriter; +import java.util.List; + +public class TestSubWorkflowActionExecutor extends HadoopActionExecutorTestCase { + private static final int JOB_TIMEOUT = 100 * 1000; + + public void testType() { + SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor(); + assertEquals(SubWorkflowActionExecutor.ACTION_TYPE, subWorkflow.getType()); + } + + public void testSubWorkflowConfCreation() throws Exception { + SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor(); + + XConfiguration protoConf = getBaseProtoConf(); + WorkflowBean workflow = createBaseWorkflow(protoConf, "W"); + + ActionBean action = (ActionBean) workflow.getActions().get(0); + action.setConf("" + + " hdfs://foo:9000/user/bar" + + " " + + " " + + " a" + + " A" + + " " + + " " + + ""); + + WorkflowClient workflowClient = subWorkflow.getWorkflowClient(new Context(workflow, action), + SubWorkflowActionExecutor.LOCAL); + assertNotNull(workflowClient); + + workflowClient = subWorkflow.getWorkflowClient(new Context(workflow, action), "http://localhost:8080/oozie"); + + assertNotNull(workflowClient); + } + + private static final String APP1 = "" + + "" + + "" + + ""; + + public void testSubWorkflowStart() throws Exception { + Path subWorkflowAppPath = getFsTestCaseDir(); + FileSystem fs = getFileSystem(); + Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml"))); + writer.write(APP1); + writer.close(); + + XConfiguration protoConf = getBaseProtoConf(); + WorkflowBean workflow = createBaseWorkflow(protoConf, "W"); + + final ActionBean action = (ActionBean) workflow.getActions().get(0); + action.setConf("" + + " " + subWorkflowAppPath + "" + + " " + + " " + + " a" + + " A" + + " " + + " " + + ""); + + SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor(); + subWorkflow.start(new Context(workflow, action), action); + + final WorkflowClient workflowClient = subWorkflow.getWorkflowClient(new Context(workflow, action), + SubWorkflowActionExecutor.LOCAL); + waitFor(JOB_TIMEOUT, new Predicate() { + public boolean evaluate() throws Exception { + return workflowClient.getJobInfo(action.getExternalId()).getStatus() == Workflow.Status.SUCCEEDED; + } + }); + + assertEquals(Workflow.Status.SUCCEEDED, workflowClient.getJobInfo(action.getExternalId()).getStatus()); + + subWorkflow.check(new Context(workflow, action), action); + + assertEquals(Action.Status.DONE, action.getStatus()); + + subWorkflow.end(new Context(workflow, action), action); + + assertEquals(Action.Status.OK, action.getStatus()); + + } + + public void testConfigPropagation() throws Exception { + Path subWorkflowAppPath = getFsTestCaseDir(); + FileSystem fs = getFileSystem(); + Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml"))); + writer.write(APP1); + writer.close(); + + XConfiguration protoConf = getBaseProtoConf(); + WorkflowBean workflow = createBaseWorkflow(protoConf, "W"); + String defaultConf = workflow.getConf(); + XConfiguration newConf = new XConfiguration(new StringReader(defaultConf)); + newConf.set("abc", "xyz"); + workflow.setConf(newConf.toXmlString()); + + final ActionBean action = (ActionBean) workflow.getActions().get(0); + action.setConf("" + + " " + subWorkflowAppPath + "" + + " " + + " " + + " " + + " a" + + " A" + + " " + + " " + + ""); + + SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor(); + subWorkflow.start(new Context(workflow, action), action); + + final WorkflowClient workflowClient = subWorkflow.getWorkflowClient(new Context(workflow, action), + SubWorkflowActionExecutor.LOCAL); + waitFor(JOB_TIMEOUT, new Predicate() { + public boolean evaluate() throws Exception { + return workflowClient.getJobInfo(action.getExternalId()).getStatus() == Workflow.Status.SUCCEEDED; + } + }); + + assertEquals(Workflow.Status.SUCCEEDED, workflowClient.getJobInfo(action.getExternalId()).getStatus()); + + subWorkflow.check(new Context(workflow, action), action); + + assertEquals(Action.Status.DONE, action.getStatus()); + + subWorkflow.end(new Context(workflow, action), action); + + assertEquals(Action.Status.OK, action.getStatus()); + + List workflows = workflowClient.getJobsInfo(""); + Configuration childConf = new XConfiguration(new StringReader(workflows.get(0).getConf())); + assertEquals("xyz", childConf.get("abc")); + + final ActionBean action1 = (ActionBean) workflow.getActions().get(0); + action1.setConf("" + + " " + subWorkflowAppPath + "" + + " " + + " " + + " a" + + " A" + + " " + + " " + + ""); + + subWorkflow = new SubWorkflowActionExecutor(); + subWorkflow.start(new Context(workflow, action1), action1); + + final WorkflowClient workflowClient1 = subWorkflow.getWorkflowClient(new Context(workflow, action1), + SubWorkflowActionExecutor.LOCAL); + waitFor(JOB_TIMEOUT, new Predicate() { + public boolean evaluate() throws Exception { + return workflowClient1.getJobInfo(action1.getExternalId()).getStatus() == Workflow.Status.SUCCEEDED; + } + }); + + assertEquals(Workflow.Status.SUCCEEDED, workflowClient1.getJobInfo(action1.getExternalId()).getStatus()); + + subWorkflow.check(new Context(workflow, action1), action1); + + assertEquals(Action.Status.DONE, action1.getStatus()); + + subWorkflow.end(new Context(workflow, action1), action1); + + assertEquals(Action.Status.OK, action1.getStatus()); + + workflows = workflowClient1.getJobsInfo(""); + childConf = new XConfiguration(new StringReader(workflows.get(0).getConf())); + assertNull(childConf.get("abc")); + } +} Index: oozie/core/src/test/java/org/apache/oozie/dag/action/hadoop/TestFsELFunctions.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/action/hadoop/TestFsELFunctions.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/action/hadoop/TestFsELFunctions.java (revision 0) @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.action.hadoop; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.oozie.client.WorkflowClient; +import org.apache.oozie.dag.ActionBean; +import org.apache.oozie.dag.DagELFunctions; +import org.apache.oozie.dag.WorkflowBean; +import org.apache.oozie.dag.workflow.lite.EndNodeDef; +import org.apache.oozie.dag.workflow.lite.LiteWorkflowApp; +import org.apache.oozie.dag.workflow.lite.LiteWorkflowInstance; +import org.apache.oozie.dag.workflow.lite.StartNodeDef; +import org.apache.oozie.service.ELService; +import org.apache.oozie.service.Services; +import org.apache.oozie.test.XFsTestCase; +import org.apache.oozie.util.ELEvaluator; +import org.apache.oozie.util.XConfiguration; + +public class TestFsELFunctions extends XFsTestCase { + + public void testFunctions() throws Exception { + Services services = new Services(); + services.init(); + + String file1 = new Path(getFsTestCaseDir(), "file1").toString(); + String file2 = new Path(getFsTestCaseDir(), "file2").toString(); + String dir = new Path(getFsTestCaseDir(), "dir").toString(); + Configuration protoConf = new Configuration(); + protoConf.set(WorkflowClient.USER_NAME, System.getProperty("user.name")); + protoConf.set(WorkflowClient.GROUP_NAME, "group"); + protoConf.set("hadoop.job.ugi", System.getProperty("user.name") + "," + "group"); + + FileSystem fs = getFileSystem(); + fs.mkdirs(new Path(dir)); + fs.create(new Path(file1)).close(); + OutputStream os = fs.create(new Path(dir, "a")); + byte[] arr = new byte[1]; + os.write(arr); + os.close(); + os = fs.create(new Path(dir, "b")); + arr = new byte[2]; + os.write(arr); + os.close(); + + Configuration conf = new XConfiguration(); + conf.set(WorkflowClient.APP_PATH, "appPath"); + conf.set(WorkflowClient.USER_NAME, System.getProperty("user.name")); + conf.set(WorkflowClient.GROUP_NAME, "group"); + conf.set("test.dir", getTestCaseDir()); + conf.set("file1", file1); + conf.set("file2", file2); + conf.set("file3", "${file2}"); + conf.set("dir", dir); + + LiteWorkflowApp def = + new LiteWorkflowApp("name", "", new StartNodeDef("end")).addNode(new EndNodeDef("end")); + LiteWorkflowInstance job = new LiteWorkflowInstance(def, conf, "wfId"); + + WorkflowBean wf = new WorkflowBean(); + wf.setId(job.getId()); + wf.setAppName("name"); + wf.setAppPath("appPath"); + wf.setUser(System.getProperty("user.name")); + wf.setGroup("group"); + wf.setWorkflowInstance(job); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + protoConf.writeXml(baos); + wf.setProtoActionConf(baos.toString()); + + ActionBean action = new ActionBean(); + action.setId("actionId"); + action.setName("actionName"); + + ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(); + DagELFunctions.configureEvaluator(eval, wf, action); + + assertEquals(true, (boolean) eval.evaluate("${fs:exists(wf:conf('file1'))}", Boolean.class)); + assertEquals(false, (boolean) eval.evaluate("${fs:exists(wf:conf('file2'))}", Boolean.class)); + assertEquals(true, (boolean) eval.evaluate("${fs:exists(wf:conf('dir'))}", Boolean.class)); + assertEquals(false, (boolean) eval.evaluate("${fs:isDir(wf:conf('file1'))}", Boolean.class)); + assertEquals(0, (int) eval.evaluate("${fs:fileSize(wf:conf('file1'))}", Integer.class)); + assertEquals(-1, (int) eval.evaluate("${fs:fileSize(wf:conf('file2'))}", Integer.class)); + assertEquals(3, (int) eval.evaluate("${fs:dirSize(wf:conf('dir'))}", Integer.class)); + assertEquals(-1, (int) eval.evaluate("${fs:blockSize(wf:conf('file2'))}", Integer.class)); + assertTrue(eval.evaluate("${fs:blockSize(wf:conf('file1'))}", Integer.class) > 0); + } + +} Index: oozie/core/src/test/java/org/apache/oozie/dag/action/hadoop/TestMapReduceActionExecutor.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/action/hadoop/TestMapReduceActionExecutor.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/action/hadoop/TestMapReduceActionExecutor.java (revision 0) @@ -0,0 +1,783 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.action.hadoop; + +import org.apache.hadoop.filecache.DistributedCache; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobID; +import org.apache.hadoop.streaming.StreamJob; +import org.apache.oozie.dag.ActionBean; +import org.apache.oozie.dag.WorkflowBean; +import org.apache.oozie.dag.service.WorkflowAppService; +import org.apache.oozie.util.XConfiguration; +import org.apache.oozie.util.IOUtils; +import org.apache.oozie.util.ClassUtils; +import org.apache.oozie.service.Services; +import org.apache.oozie.service.UUIDService; +import org.apache.oozie.client.Action; + +import java.io.File; +import java.io.Writer; +import java.io.InputStream; +import java.io.FileInputStream; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.net.URI; + +public class TestMapReduceActionExecutor extends HadoopActionExecutorTestCase { + private static final int JOB_TIMEOUT = 100 * 1000; + + public void testType() { + MapReduceActionExecutor mapReduce = new MapReduceActionExecutor(); + assertEquals(MapReduceActionExecutor.ACTION_TYPE, mapReduce.getType()); + } + + public void testJobConfCreation() throws Exception { + URI appUri = getAppPath().toUri(); + String appPath = appUri.getPath(); + MapReduceActionExecutor mapReduce = new MapReduceActionExecutor(); + + XConfiguration protoConf = getBaseProtoConf(); + protoConf.setStrings(WorkflowAppService.APP_LIB_JAR_PATH_LIST, "lib/x.jar"); + protoConf.setStrings(WorkflowAppService.APP_LIB_SO_PATH_LIST, "lib/y.so"); + WorkflowBean workflow = createBaseWorkflow(protoConf, "H"); + + ActionBean action = (ActionBean) workflow.getActions().get(0); + action.setConf("" + + " " + getJobTrackerUri() + "" + + " " + getNameNodeUri() + "" + + " " + + " " + + " mapred.mapper.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + " mapred.reducer.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + " a" + + " A" + + " " + + " " + + " b" + + " B" + + " " + + " " + + ""); + + JobConf jobConf = mapReduce.createMapRedJobConf(new Context(workflow, action), action); + + assertEquals(System.getProperty("user.name"), jobConf.getUser()); + assertEquals(System.getProperty("user.name") + ",other", jobConf.get("hadoop.job.ugi")); + assertEquals("A", jobConf.get("a")); + assertEquals("B", jobConf.get("b")); + + assertTrue(DistributedCache.getSymlink(jobConf)); + assertEquals(2, DistributedCache.getFileClassPaths(jobConf).length); + assertTrue(DistributedCache.getFileClassPaths(jobConf)[0].toString().endsWith("lib/x.jar")); + assertTrue(DistributedCache.getFileClassPaths(jobConf)[1].toString().endsWith(HadoopActionExecutor.JAR_NAME)); + assertEquals(5, DistributedCache.getCacheFiles(jobConf).length); + assertTrue(DistributedCache.getCacheFiles(jobConf)[0].toString().endsWith("lib/x.jar")); + assertTrue(DistributedCache.getCacheFiles(jobConf)[1].toString().endsWith("lib/x.jar")); + assertTrue(DistributedCache.getCacheFiles(jobConf)[2].toString().endsWith("lib/y.so#y.so")); + assertTrue(DistributedCache.getCacheFiles(jobConf)[3].toString().endsWith(HadoopActionExecutor.JAR_NAME)); + assertTrue(DistributedCache.getCacheFiles(jobConf)[4].toString().endsWith(HadoopActionExecutor.JAR_NAME)); + + + FileSystem fs = getFileSystem(); + Writer writer = new OutputStreamWriter(fs.create(new Path(appPath, "job.xml"))); + writer.write(" " + + " " + + " mapred.mapper.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + " mapred.reducer.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + " b" + + " BB" + + " " + + " " + + " c" + + " CC" + + " " + + " "); + writer.close(); + + action.setConf("" + + " " + getJobTrackerUri() + "" + + " " + getNameNodeUri() + "" + + " job.xml" + + " " + + " " + + " mapred.mapper.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + " mapred.reducer.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + " a" + + " A" + + " " + + " " + + " b" + + " B" + + " " + + " " + + ""); + + jobConf = mapReduce.createMapRedJobConf(new Context(workflow, action), action); + assertEquals(System.getProperty("user.name"), jobConf.getUser()); + assertEquals(System.getProperty("user.name") + ",other", jobConf.get("hadoop.job.ugi")); + assertEquals("A", jobConf.get("a")); + assertEquals("B", jobConf.get("b")); + assertEquals("CC", jobConf.get("c")); + + + action.setConf("" + + " " + getJobTrackerUri() + "" + + " " + getNameNodeUri() + "" + + " job.xml" + + " " + + " " + + " a" + + " A" + + " " + + " " + + " b" + + " B" + + " " + + " " + + " file\n" + + " file1.so\n" + + " file2.so.3\n" + + " /file3.so\n" + + " /test1.jar\n" + + " archive\n" + + " /abs-archive\n" + + ""); + + jobConf = mapReduce.createMapRedJobConf(new Context(workflow, action), action); + assertEquals(System.getProperty("user.name"), jobConf.getUser()); + assertEquals(System.getProperty("user.name") + ",other", jobConf.get("hadoop.job.ugi")); + assertEquals("A", jobConf.get("a")); + assertEquals("B", jobConf.get("b")); + assertEquals("CC", jobConf.get("c")); + + assertTrue(DistributedCache.getSymlink(jobConf)); + assertEquals(4, DistributedCache.getFileClassPaths(jobConf).length); + assertTrue(DistributedCache.getFileClassPaths(jobConf)[0].toString().endsWith(appPath + "/lib/x.jar")); + assertTrue(DistributedCache.getFileClassPaths(jobConf)[1].toString().endsWith(appPath + "/file")); + assertTrue(DistributedCache.getFileClassPaths(jobConf)[2].toString().equals("/test1.jar")); + assertEquals(12, DistributedCache.getCacheFiles(jobConf).length); + assertTrue(DistributedCache.getCacheFiles(jobConf)[0].toString().endsWith(appPath + "/lib/x.jar")); + assertTrue(DistributedCache.getCacheFiles(jobConf)[1].toString().endsWith(appPath + "/lib/x.jar")); + assertTrue(DistributedCache.getCacheFiles(jobConf)[2].toString().endsWith(appPath + "/lib/y.so#y.so")); + assertTrue(DistributedCache.getCacheFiles(jobConf)[3].toString().endsWith(appPath + "/file")); + assertTrue(DistributedCache.getCacheFiles(jobConf)[4].toString().endsWith(appPath + "/file")); + assertTrue(DistributedCache.getCacheFiles(jobConf)[5].toString().endsWith(appPath + "/file1.so#file1.so")); + assertTrue(DistributedCache.getCacheFiles(jobConf)[6].toString().endsWith(appPath + "/file2.so.3#file2.so")); + assertTrue(DistributedCache.getCacheFiles(jobConf)[7].toString().equals("/file3.so#file3.so")); + assertTrue(DistributedCache.getCacheFiles(jobConf)[8].toString().endsWith("/test1.jar")); + assertTrue(DistributedCache.getCacheFiles(jobConf)[9].toString().equals("/test1.jar")); + assertEquals(2, DistributedCache.getCacheArchives(jobConf).length); + assertTrue(DistributedCache.getCacheArchives(jobConf)[0].toString().equals(appPath + "/archive")); + assertTrue(DistributedCache.getCacheArchives(jobConf)[1].toString().equals("/abs-archive")); + + + action.setConf("" + + " " + getJobTrackerUri() + "" + + " " + getNameNodeUri() + "" + + " \n" + + " map.sh\n" + + " reduce.sh\n" + + " x=X" + + " y=Y" + + " RR" + + " r=R" + + " " + + " " + + " " + + " a" + + " A" + + " " + + " " + + " b" + + " B" + + " " + + " " + + " mapred.reducer.class" + + " test.Reducer" + + " " + + " " + + ""); + + jobConf = mapReduce.createMapRedJobConf(new Context(workflow, action), action); + assertEquals(System.getProperty("user.name"), jobConf.getUser()); + assertEquals(System.getProperty("user.name") + ",other", jobConf.get("hadoop.job.ugi")); + assertEquals("A", jobConf.get("a")); + assertEquals("B", jobConf.get("b")); + + assertEquals("org.apache.hadoop.streaming.PipeMapper", jobConf.get("oozie.mapper.class")); + assertEquals("test.Reducer", jobConf.get("mapred.reducer.class")); + + assertEquals("x=X y=Y", jobConf.get("stream.addenvironment")); + assertEquals("RR", jobConf.get("stream.recordreader.class")); + assertEquals("R", jobConf.get("stream.recordreader.r")); + } + + public void testJobStartUsingAppJar() throws Exception { + URI appUri = getAppPath().toUri(); + String appPath = appUri.getPath(); + + FileSystem fs = getFileSystem(); + fs.mkdirs(new Path(appPath, "lib")); + File jarFile = IOUtils.createJar(new File(getTestCaseDir()), "test.jar", MapperReducerForTest.class); + InputStream is = new FileInputStream(jarFile); + OutputStream os = fs.create(new Path(getAppPath(), "lib/test.jar")); + IOUtils.copyStream(is, os); + + XConfiguration protoConf = getBaseProtoConf(); + protoConf.setStrings(WorkflowAppService.APP_LIB_JAR_PATH_LIST, "lib/test.jar"); + WorkflowBean workflow = createBaseWorkflow(protoConf, "H"); + + Path input = new Path(getFsTestCaseDir(), "input"); + Path output = new Path(getFsTestCaseDir(), "output"); + fs.mkdirs(input); + Writer writer = new OutputStreamWriter(fs.create(new Path(input, "test.txt"))); + writer.write("hello"); + writer.close(); + + final ActionBean action = (ActionBean) workflow.getActions().get(0); + action.setConf("" + + " " + getJobTrackerUri() + "" + + " " + getNameNodeUri() + "" + + " " + + " " + + " mapred.mapper.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + " mapred.reducer.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + " mapred.input.dir" + + " " + input + "" + + " " + + " " + + " mapred.output.dir" + + " " + output + "" + + " " + + " " + + ""); + + MapReduceActionExecutor mapReduce = new MapReduceActionExecutor(); + + JobConf jobConf = mapReduce.createMapRedJobConf(new Context(workflow, action), action); + + + mapReduce.start(new Context(workflow, action), action); + + final JobClient jobClient = new JobClient(jobConf); + waitFor(JOB_TIMEOUT, new Predicate() { + public boolean evaluate() throws Exception { + return jobClient.getJob(JobID.forName(action.getExternalId())).isComplete(); + } + }); + + assertTrue(jobClient.getJob(JobID.forName(action.getExternalId())).isComplete()); + assertTrue(jobClient.getJob(JobID.forName(action.getExternalId())).isSuccessful()); + } + + public void testJobStartWithPrepare() throws Exception { + URI appUri = getAppPath().toUri(); + String appPath = appUri.getPath(); + + FileSystem fs = getFileSystem(); + fs.mkdirs(new Path(appPath, "lib")); + File jarFile = IOUtils.createJar(new File(getTestCaseDir()), "test.jar", MapperReducerForTest.class); + InputStream is = new FileInputStream(jarFile); + OutputStream os = fs.create(new Path(getAppPath(), "lib/test.jar")); + IOUtils.copyStream(is, os); + + XConfiguration protoConf = getBaseProtoConf(); + protoConf.setStrings(WorkflowAppService.APP_LIB_JAR_PATH_LIST, "lib/test.jar"); + WorkflowBean workflow = createBaseWorkflow(protoConf, "H"); + + Path input = new Path(getFsTestCaseDir(), "input"); + Path output = new Path(getFsTestCaseDir(), "output"); + fs.mkdirs(input); + Writer writer = new OutputStreamWriter(fs.create(new Path(input, "test.txt"))); + writer.write("hello"); + writer.close(); + + Path deletePath = new Path(getFsTestCaseDir(), "deleteDir"); + Path mkdirPath = new Path(getFsTestCaseDir(), "mkdirDir"); + fs.mkdirs(deletePath); + + final ActionBean action = (ActionBean) workflow.getActions().get(0); + action.setConf("" + + " " + getJobTrackerUri() + "" + + " " + getNameNodeUri() + "" + + " " + + " " + + " " + + " " + + " " + + " " + + " mapred.mapper.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + " mapred.reducer.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + " mapred.input.dir" + + " " + input + "" + + " " + + " " + + " mapred.output.dir" + + " " + output + "" + + " " + + " " + + ""); + + MapReduceActionExecutor mapReduce = new MapReduceActionExecutor(); + + JobConf jobConf = mapReduce.createMapRedJobConf(new Context(workflow, action), action); + + + mapReduce.start(new Context(workflow, action), action); + + final JobClient jobClient = new JobClient(jobConf); + waitFor(JOB_TIMEOUT, new Predicate() { + public boolean evaluate() throws Exception { + return jobClient.getJob(JobID.forName(action.getExternalId())).isComplete(); + } + }); + + assertTrue(jobClient.getJob(JobID.forName(action.getExternalId())).isComplete()); + assertTrue(jobClient.getJob(JobID.forName(action.getExternalId())).isSuccessful()); + assertTrue(fs.exists(mkdirPath)); + assertFalse(fs.exists(deletePath)); + } + + public void testJobStartUsingAbsJar() throws Exception { + FileSystem fs = getFileSystem(); + fs.mkdirs(new Path(getFsTestCaseDir(), "xxx")); + + Path jarFsPath = new Path(getFsTestCaseDir(), "xxx/test.jar"); + + File jarFile = IOUtils.createJar(new File(getTestCaseDir()), "test.jar", MapperReducerForTest.class); + InputStream is = new FileInputStream(jarFile); + OutputStream os = fs.create(jarFsPath); + IOUtils.copyStream(is, os); + + XConfiguration protoConf = getBaseProtoConf(); + WorkflowBean workflow = createBaseWorkflow(protoConf, "H"); + + Path input = new Path(getFsTestCaseDir(), "input"); + Path output = new Path(getFsTestCaseDir(), "output"); + fs.mkdirs(input); + Writer writer = new OutputStreamWriter(fs.create(new Path(input, "test.txt"))); + writer.write("hello"); + writer.close(); + + final ActionBean action = (ActionBean) workflow.getActions().get(0); + action.setConf("" + + " " + getJobTrackerUri() + "" + + " " + getNameNodeUri() + "" + + " " + + " " + + " mapred.mapper.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + " mapred.reducer.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + " mapred.input.dir" + + " " + input + "" + + " " + + " " + + " mapred.output.dir" + + " " + output + "" + + " " + + " " + + " " + jarFsPath.toString() + "\n" + + ""); + + MapReduceActionExecutor mapReduce = new MapReduceActionExecutor(); + + JobConf jobConf = mapReduce.createMapRedJobConf(new Context(workflow, action), action); + final JobClient jobClient = new JobClient(jobConf); + + mapReduce.start(new Context(workflow, action), action); + + waitFor(JOB_TIMEOUT, new Predicate() { + public boolean evaluate() throws Exception { + return jobClient.getJob(JobID.forName(action.getExternalId())).isComplete(); + } + }); + + assertTrue(jobClient.getJob(JobID.forName(action.getExternalId())).isComplete()); + assertTrue(jobClient.getJob(JobID.forName(action.getExternalId())).isSuccessful()); + } + + + public void testJobStartStreaming() throws Exception { + FileSystem fs = getFileSystem(); + fs.mkdirs(new Path(getAppPath(), "lib")); + + InputStream is = new FileInputStream(ClassUtils.findContainingJar(StreamJob.class)); + OutputStream os = fs.create(new Path(getAppPath(), "lib/hadoop-streaming.jar")); + IOUtils.copyStream(is, os); + + XConfiguration protoConf = getBaseProtoConf(); + protoConf.setStrings(WorkflowAppService.APP_LIB_JAR_PATH_LIST, "lib/hadoop-streaming.jar"); + WorkflowBean workflow = createBaseWorkflow(protoConf, "H"); + + Path input = new Path(getFsTestCaseDir(), "input"); + Path output = new Path(getFsTestCaseDir(), "output"); + fs.mkdirs(input); + Writer writer = new OutputStreamWriter(fs.create(new Path(input, "test.txt"))); + writer.write("hello"); + writer.close(); + + final ActionBean action = (ActionBean) workflow.getActions().get(0); + action.setConf("" + + " " + getJobTrackerUri() + "" + + " " + getNameNodeUri() + "" + + " " + + " /bin/cat" + + " /usr/bin/wc" + + " " + + " " + + " " + + " mapred.input.dir" + + " " + input + "" + + " " + + " " + + " mapred.output.dir" + + " " + output + "" + + " " + + " " + + ""); + + MapReduceActionExecutor mapReduce = new MapReduceActionExecutor(); + + JobConf jobConf = mapReduce.createMapRedJobConf(new Context(workflow, action), action); + final JobClient jobClient = new JobClient(jobConf); + + mapReduce.start(new Context(workflow, action), action); + + waitFor(JOB_TIMEOUT, new Predicate() { + public boolean evaluate() throws Exception { + return jobClient.getJob(JobID.forName(action.getExternalId())).isComplete(); + } + }); + + assertTrue(jobClient.getJob(JobID.forName(action.getExternalId())).isComplete()); + assertTrue(jobClient.getJob(JobID.forName(action.getExternalId())).isSuccessful()); + } + + public void testJobRecovery() throws Exception { + URI appUri = getAppPath().toUri(); + String appPath = appUri.getPath(); + + FileSystem fs = getFileSystem(); + fs.mkdirs(new Path(appPath, "lib")); + File jarFile = IOUtils.createJar(new File(getTestCaseDir()), "test.jar", MapperReducerForTest.class); + InputStream is = new FileInputStream(jarFile); + OutputStream os = fs.create(new Path(getAppPath(), "lib/test.jar")); + IOUtils.copyStream(is, os); + + XConfiguration protoConf = getBaseProtoConf(); + protoConf.setStrings(WorkflowAppService.APP_LIB_JAR_PATH_LIST, "lib/test.jar"); + WorkflowBean workflow = createBaseWorkflow(protoConf, "H"); + + Path input = new Path(getFsTestCaseDir(), "input"); + Path output = new Path(getFsTestCaseDir(), "output"); + fs.mkdirs(input); + Writer writer = new OutputStreamWriter(fs.create(new Path(input, "test.txt"))); + writer.write("hello"); + writer.close(); + + final ActionBean action = (ActionBean) workflow.getActions().get(0); + action.setConf("" + + " " + getJobTrackerUri() + "" + + " " + getNameNodeUri() + "" + + " " + + " " + + " mapred.mapper.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + " mapred.reducer.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + " mapred.input.dir" + + " " + input + "" + + " " + + " " + + " mapred.output.dir" + + " " + output + "" + + " " + + " " + + ""); + + String wfId = "recovery-id_" + System.currentTimeMillis(); + workflow.setId(wfId); + + action.setId(Services.get().get(UUIDService.class).generateChildId(workflow.getId(), "H")); + + MapReduceActionExecutor mapReduce = new MapReduceActionExecutor(); + + JobConf jobConf = mapReduce.createMapRedJobConf(new Context(workflow, action), action); + final JobClient jobClient = new JobClient(jobConf); + + mapReduce.start(new Context(workflow, action), action); + String jobId = action.getExternalId(); + + waitFor(JOB_TIMEOUT, new Predicate() { + public boolean evaluate() throws Exception { + return jobClient.getJob(JobID.forName(action.getExternalId())).mapProgress() != 0; + } + }); + + mapReduce.start(new Context(workflow, action), action); + String jobIdRecovery = action.getExternalId(); + + assertEquals(jobId, jobIdRecovery); + + waitFor(JOB_TIMEOUT, new Predicate() { + public boolean evaluate() throws Exception { + return jobClient.getJob(JobID.forName(action.getExternalId())).isComplete(); + } + }); + + assertTrue(jobClient.getJob(JobID.forName(action.getExternalId())).isComplete()); + assertTrue(jobClient.getJob(JobID.forName(action.getExternalId())).isSuccessful()); + fs.delete(mapReduce.getActionDir(wfId, action, "map-reduce", false), true); + } + + + public void testOK() throws Exception { + URI appUri = getAppPath().toUri(); + String appPath = appUri.getPath(); + + FileSystem fs = getFileSystem(); + fs.mkdirs(new Path(appPath, "lib")); + File jarFile = IOUtils.createJar(new File(getTestCaseDir()), "test.jar", MapperReducerForTest.class); + InputStream is = new FileInputStream(jarFile); + OutputStream os = fs.create(new Path(getAppPath(), "lib/test.jar")); + IOUtils.copyStream(is, os); + + XConfiguration protoConf = getBaseProtoConf(); + protoConf.setStrings(WorkflowAppService.APP_LIB_JAR_PATH_LIST, "lib/test.jar"); + WorkflowBean workflow = createBaseWorkflow(protoConf, "H"); + + Path input = new Path(getFsTestCaseDir(), "input"); + Path output = new Path(getFsTestCaseDir(), "output"); + fs.mkdirs(input); + Writer writer = new OutputStreamWriter(fs.create(new Path(input, "test.txt"))); + writer.write("hello"); + writer.close(); + + final ActionBean action = (ActionBean) workflow.getActions().get(0); + action.setConf("" + + " " + getJobTrackerUri() + "" + + " " + getNameNodeUri() + "" + + " " + + " " + + " mapred.mapper.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + " mapred.reducer.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + " mapred.input.dir" + + " " + input + "" + + " " + + " " + + " mapred.output.dir" + + " " + output + "" + + " " + + " " + + ""); + + MapReduceActionExecutor mapReduce = new MapReduceActionExecutor(); + + JobConf jobConf = mapReduce.createMapRedJobConf(new Context(workflow, action), action); + + mapReduce.start(new Context(workflow, action), action); + mapReduce.check(new Context(workflow, action), action); + assertEquals("RUNNING", action.getExternalStatus()); + assertEquals(Action.Status.RUNNING, action.getStatus()); + + final JobClient jobClient = new JobClient(jobConf); + waitFor(JOB_TIMEOUT, new Predicate() { + public boolean evaluate() throws Exception { + return jobClient.getJob(JobID.forName(action.getExternalId())).isComplete(); + } + }); + + mapReduce.check(new Context(workflow, action), action); + assertEquals("SUCCEEDED", action.getExternalStatus()); + assertEquals(Action.Status.DONE, action.getStatus()); + mapReduce.end(new Context(workflow, action), action); + assertEquals("SUCCEEDED", action.getExternalStatus()); + assertEquals(Action.Status.OK, action.getStatus()); + assertEquals("OK", action.getSignalValue()); + } + + public void testKilled() throws Exception { + URI appUri = getAppPath().toUri(); + String appPath = appUri.getPath(); + + FileSystem fs = getFileSystem(); + fs.mkdirs(new Path(appPath, "lib")); + File jarFile = IOUtils.createJar(new File(getTestCaseDir()), "test.jar", MapperReducerForTest.class); + InputStream is = new FileInputStream(jarFile); + OutputStream os = fs.create(new Path(getAppPath(), "lib/test.jar")); + IOUtils.copyStream(is, os); + + XConfiguration protoConf = getBaseProtoConf(); + protoConf.setStrings(WorkflowAppService.APP_LIB_JAR_PATH_LIST, "lib/test.jar"); + WorkflowBean workflow = createBaseWorkflow(protoConf, "H"); + + Path input = new Path(getFsTestCaseDir(), "input"); + Path output = new Path(getFsTestCaseDir(), "output"); + fs.mkdirs(input); + Writer writer = new OutputStreamWriter(fs.create(new Path(input, "test.txt"))); + writer.write("hello"); + writer.close(); + + final ActionBean action = (ActionBean) workflow.getActions().get(0); + action.setConf("" + + " " + getJobTrackerUri() + "" + + " " + getNameNodeUri() + "" + + " " + + " " + + " mapred.mapper.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + " mapred.reducer.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + " mapred.input.dir" + + " " + input + "" + + " " + + " " + + " mapred.output.dir" + + " " + output + "" + + " " + + " " + + ""); + + MapReduceActionExecutor mapReduce = new MapReduceActionExecutor(); + + JobConf jobConf = mapReduce.createMapRedJobConf(new Context(workflow, action), action); + + mapReduce.start(new Context(workflow, action), action); + mapReduce.check(new Context(workflow, action), action); + assertEquals("RUNNING", action.getExternalStatus()); + assertEquals(Action.Status.RUNNING, action.getStatus()); + final JobClient jobClient = new JobClient(jobConf); + jobClient.getJob(JobID.forName(action.getExternalId())).killJob(); + waitFor(JOB_TIMEOUT, new Predicate() { + public boolean evaluate() throws Exception { + return jobClient.getJob(JobID.forName(action.getExternalId())).isComplete(); + } + }); + if(!jobClient.getJob(JobID.forName(action.getExternalId())).isComplete()) { + System.err.println("TRACKER URI: " + + jobClient.getJob(JobID.forName(action.getExternalId())).getTrackingURL()); + } + mapReduce.check(new Context(workflow, action), action); + assertEquals("FAILED/KILLED", action.getExternalStatus()); + assertEquals(Action.Status.DONE, action.getStatus()); + mapReduce.end(new Context(workflow, action), action); + assertEquals("FAILED/KILLED", action.getExternalStatus()); + assertEquals(Action.Status.ERROR, action.getStatus()); + assertEquals("ERROR", action.getSignalValue()); + + } + + private void _testCleanUp(boolean cleanUp) throws Exception { + Services.get().destroy(); + setSystemProperty(HadoopActionExecutor.CONF_CLEAN_UP_TEMP_DIR, Boolean.toString(cleanUp)); + new Services().init(); + + XConfiguration protoConf = getBaseProtoConf(); + WorkflowBean workflow = createBaseWorkflow(protoConf, "H"); + + final ActionBean action = (ActionBean) workflow.getActions().get(0); + action.setConf("" + + " " + getJobTrackerUri() + "" + + " " + getNameNodeUri() + "" + + " " + + " " + + " mapred.mapper.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + " mapred.reducer.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + ""); + + MapReduceActionExecutor mapReduce = new MapReduceActionExecutor(); + + mapReduce.createMapRedJobConf(new Context(workflow, action), action); + + Path path = mapReduce.getActionDir(workflow.getId(), action, HadoopActionExecutor.ACTION_TYPE, false); + FileSystem fs = getFileSystem(); + assertTrue(fs.exists(path)); + + action.setExecutionData("SUCCEEDED", null); + mapReduce.end(new Context(workflow, action), action); + + assertEquals(!cleanUp, fs.exists(path)); + } + + public void testWithCleanUp() throws Exception { + _testCleanUp(true); + } + + public void testWithNoCleanUp() throws Exception { + _testCleanUp(false); + } +} Index: oozie/core/src/test/java/org/apache/oozie/dag/action/hadoop/HadoopActionExecutorTestCase.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/action/hadoop/HadoopActionExecutorTestCase.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/action/hadoop/HadoopActionExecutorTestCase.java (revision 0) @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.action.hadoop; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.oozie.client.Action; +import org.apache.oozie.client.Workflow; +import org.apache.oozie.client.WorkflowClient; +import org.apache.oozie.dag.ActionBean; +import org.apache.oozie.dag.WorkflowBean; +import org.apache.oozie.dag.action.ActionExecutor; +import org.apache.oozie.dag.service.WorkflowAppService; +import org.apache.oozie.dag.service.CallbackService; +import org.apache.oozie.service.Services; +import org.apache.oozie.service.UUIDService; +import org.apache.oozie.test.XFsTestCase; +import org.apache.oozie.util.ELEvaluator; +import org.apache.oozie.util.XConfiguration; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Properties; + +public abstract class HadoopActionExecutorTestCase extends XFsTestCase { + + protected void setUp() throws Exception { + super.setUp(); + new Services().init(); + } + + protected void tearDown() throws Exception { + Services.get().destroy(); + super.tearDown(); + } + + protected static class Context implements ActionExecutor.Context { + private ActionBean action; + private WorkflowBean workflow; + boolean started; + boolean executed; + boolean ended; + + public Context(WorkflowBean workflow, ActionBean action) { + this.workflow = workflow; + this.action = action; + } + + public String getCallbackUrl(String externalStatusVar) { + return Services.get().get(CallbackService.class).createCallBackUrl(action.getId(), externalStatusVar); + } + + public Configuration getProtoActionConf() { + String s = workflow.getProtoActionConf(); + try { + return new XConfiguration(new StringReader(s)); + } + catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + public Workflow getWorkflow() { + return workflow; + } + + public Action getAction() { + return action; + } + + public ELEvaluator getELEvaluator() { + throw new UnsupportedOperationException(); + } + + public void setVar(String name, String value) { + throw new UnsupportedOperationException(); + } + + public String getVar(String name) { + throw new UnsupportedOperationException(); + } + + public void setStartData(String externalId, String trackerUri, String consoleUrl) { + action.setStartData(externalId, trackerUri, consoleUrl); + started = true; + } + + public void setExecutionData(String externalStatus, Properties actionData) { + action.setExecutionData(externalStatus, actionData); + executed = true; + } + + public void setEndData(Action.Status status, String signalValue) { + action.setEndData(status, signalValue); + ended = true; + } + + public boolean isRetry() { + throw new UnsupportedOperationException(); + } + + public boolean isStarted() { + return started; + } + + public boolean isExecuted() { + return executed; + } + + public boolean isEnded() { + return ended; + } + + public void setExternalStatus(String externalStatus) { + action.setExternalStatus(externalStatus); + } + } + + protected Path getAppPath() { + Path baseDir = getFsTestCaseDir(); + return new Path(baseDir, "app"); + } + + protected XConfiguration getBaseProtoConf() { + XConfiguration protoConf = new XConfiguration(); + protoConf.set(WorkflowAppService.HADOOP_USER, System.getProperty("user.name")); + protoConf.set(WorkflowAppService.HADOOP_UGI, System.getProperty("user.name") + ",other"); + protoConf.set(WorkflowClient.GROUP_NAME, "other"); + return protoConf; + } + + // it contains one action with no configuration. + protected WorkflowBean createBaseWorkflow(XConfiguration protoConf, String actionName) { + Path appUri = getAppPath(); + XConfiguration wfConf = new XConfiguration(); + wfConf.set(WorkflowClient.USER_NAME, protoConf.get(WorkflowClient.USER_NAME)); + wfConf.set(WorkflowClient.GROUP_NAME, "other"); + wfConf.set(WorkflowClient.APP_PATH, appUri.toString()); + + WorkflowBean workflow = new WorkflowBean(); + workflow.setAppPath(appUri.toString()); + workflow.setId(Services.get().get(UUIDService.class).generateId()); + workflow.setConf(wfConf.toXmlString()); + workflow.setUser(wfConf.get(WorkflowClient.USER_NAME)); + workflow.setGroup(wfConf.get(WorkflowClient.GROUP_NAME)); + workflow.setAuthToken("authToken"); + + workflow.setProtoActionConf(protoConf.toXmlString()); + + ActionBean action = new ActionBean(); + action.setName(actionName); + action.setId(Services.get().get(UUIDService.class).generateChildId(workflow.getId(), actionName)); + workflow.getActions().add(action); + return workflow; + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/dag/action/hadoop/TestHadoopELFunctions.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/action/hadoop/TestHadoopELFunctions.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/action/hadoop/TestHadoopELFunctions.java (revision 0) @@ -0,0 +1,226 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.action.hadoop; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobID; +import org.apache.oozie.client.Action; +import org.apache.oozie.client.Workflow; +import org.apache.oozie.client.WorkflowClient; +import org.apache.oozie.dag.ActionBean; +import org.apache.oozie.dag.WorkflowBean; +import org.apache.oozie.dag.DagELFunctions; +import org.apache.oozie.dag.workflow.WorkflowInstance; +import org.apache.oozie.dag.workflow.lite.LiteWorkflowInstance; +import org.apache.oozie.dag.workflow.lite.StartNodeDef; +import org.apache.oozie.dag.workflow.lite.LiteWorkflowApp; +import org.apache.oozie.dag.workflow.lite.EndNodeDef; +import org.apache.oozie.dag.action.ActionExecutor; +import org.apache.oozie.dag.service.WorkflowAppService; +import org.apache.oozie.dag.service.CallbackService; +import org.apache.oozie.util.ELEvaluator; +import org.apache.oozie.util.XConfiguration; +import org.apache.oozie.util.IOUtils; +import org.apache.oozie.service.Services; +import org.apache.oozie.service.ELService; +import org.apache.oozie.service.UUIDService; + +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.io.Writer; +import java.io.OutputStreamWriter; +import java.util.Properties; + +public class TestHadoopELFunctions extends HadoopActionExecutorTestCase { + private static final int JOB_TIMEOUT = 200 * 1000; + + private static class Context implements ActionExecutor.Context { + private ActionBean action; + private WorkflowBean workflow; + + public Context(WorkflowBean workflow, ActionBean action) { + this.workflow = workflow; + this.action = action; + } + + public String getCallbackUrl(String externalStatusVar) { + return Services.get().get(CallbackService.class).createCallBackUrl(action.getId(), externalStatusVar); + } + + public Configuration getProtoActionConf() { + String s = workflow.getProtoActionConf(); + try { + return new XConfiguration(new StringReader(s)); + } + catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + public Workflow getWorkflow() { + return workflow; + } + + public ELEvaluator getELEvaluator() { + throw new UnsupportedOperationException(); + } + + public void setVar(String name, String value) { + throw new UnsupportedOperationException(); + } + + public String getVar(String name) { + throw new UnsupportedOperationException(); + } + + public void setStartData(String externalId, String trackerUri, String consoleUrl) { + action.setStartData(externalId, trackerUri, consoleUrl); + } + + public void setExecutionData(String externalStatus, Properties actionData) { + action.setExecutionData(externalStatus, actionData); + } + + public void setEndData(Action.Status status, String signalValue) { + action.setEndData(status, signalValue); + } + + public boolean isRetry() { + throw new UnsupportedOperationException(); + } + + public Action getAction() { + return null; + } + + public boolean isStarted() { + throw new UnsupportedOperationException(); + } + + public boolean isExecuted() { + throw new UnsupportedOperationException(); + } + + public boolean isEnded() { + throw new UnsupportedOperationException(); + } + + public void setExternalStatus(String externalStatus) { + action.setExternalStatus(externalStatus); + } + } + + public void testCountersEL() throws Exception { + FileSystem fs = getFileSystem(); + Path appPath = getAppPath(); + + XConfiguration protoConf = new XConfiguration(); + protoConf.setStrings(WorkflowAppService.HADOOP_USER, System.getProperty("user.name")); + protoConf.setStrings(WorkflowAppService.HADOOP_UGI, System.getProperty("user.name") + ",other"); + protoConf.setStrings(WorkflowAppService.APP_LIB_JAR_PATH_LIST, "lib/test.jar"); + + fs.mkdirs(new Path(appPath, "lib")); + + File jarFile = IOUtils.createJar(new File(getTestCaseDir()), "test.jar", MapperReducerForTest.class); + fs.copyFromLocalFile(new Path(jarFile.getAbsolutePath()), new Path(appPath, "lib/test.jar")); + + XConfiguration wfConf = new XConfiguration(); + wfConf.set(WorkflowClient.APP_PATH, appPath.toString()); + + WorkflowBean workflow = new WorkflowBean(); + workflow.setConf(wfConf.toXmlString()); + workflow.setAppPath(wfConf.get(WorkflowClient.APP_PATH)); + workflow.setProtoActionConf(protoConf.toXmlString()); + + LiteWorkflowApp wfApp = new LiteWorkflowApp("x", "", new StartNodeDef("a")); + wfApp.addNode(new EndNodeDef("a")); + WorkflowInstance wi = new LiteWorkflowInstance(wfApp, new XConfiguration(), "1"); + workflow.setWorkflowInstance(wi); + workflow.setId(Services.get().get(UUIDService.class).generateId()); + + Path input = new Path(getFsTestCaseDir(), "input"); + Path output = new Path(getFsTestCaseDir(), "output"); + fs.mkdirs(input); + Writer writer = new OutputStreamWriter(fs.create(new Path(input, "test.txt"))); + writer.write("hello"); + writer.close(); + + final ActionBean action = new ActionBean(); + action.setName("H"); + action.setId(Services.get().get(UUIDService.class).generateChildId(workflow.getId(), "H")); + action.setConf("" + + " " + getJobTrackerUri() + "" + + " " + getNameNodeUri() + "" + + " " + + " " + + " mapred.mapper.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + " mapred.reducer.class" + + " " + MapperReducerForTest.class.getName() + "" + + " " + + " " + + " mapred.input.dir" + + " " + input.toString() + "" + + " " + + " " + + " mapred.output.dir" + + " " + output.toString() + "" + + " " + + " " + + ""); + + MapReduceActionExecutor hadoop = new MapReduceActionExecutor(); + + JobConf jobConf = hadoop.createMapRedJobConf(new Context(workflow, action), action); + final JobClient jobClient = new JobClient(jobConf); + + hadoop.start(new Context(workflow, action), action); + + waitFor(JOB_TIMEOUT, new Predicate() { + public boolean evaluate() throws Exception { + return jobClient.getJob(JobID.forName(action.getExternalId())).isComplete(); + } + }); + + assertTrue(jobClient.getJob(JobID.forName(action.getExternalId())).isComplete()); + assertTrue(jobClient.getJob(JobID.forName(action.getExternalId())).isSuccessful()); + + DagELFunctions.setActionInfo(wi, action); + + ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(); + DagELFunctions.configureEvaluator(eval, workflow, action); + + String group = MapperReducerForTest.GROUP; + String name = MapperReducerForTest.NAME; + assertEquals(new Long(5), eval.evaluate("${hadoop:counters('H')['" + group + "']['" + name +"']}", Long.class)); + + assertEquals(new Long(1), eval.evaluate("${hadoop:counters('H')[RECORDS][GROUPS]}", Long.class)); + assertEquals(new Long(1), eval.evaluate("${hadoop:counters('H')[RECORDS][REDUCE_IN]}", Long.class)); + assertEquals(new Long(1), eval.evaluate("${hadoop:counters('H')[RECORDS][REDUCE_OUT]}", Long.class)); + assertEquals(new Long(1), eval.evaluate("${hadoop:counters('H')[RECORDS][MAP_IN]}", Long.class)); + assertEquals(new Long(1), eval.evaluate("${hadoop:counters('H')[RECORDS][MAP_OUT]}", Long.class)); + } + +} Index: oozie/core/src/test/java/org/apache/oozie/dag/action/hadoop/MapperReducerForTest.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/action/hadoop/MapperReducerForTest.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/action/hadoop/MapperReducerForTest.java (revision 0) @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.action.hadoop; + +import org.apache.hadoop.mapred.Mapper; +import org.apache.hadoop.mapred.OutputCollector; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.Reducer; + +import java.io.IOException; +import java.util.Iterator; + +public class MapperReducerForTest implements Mapper, Reducer { + public static final String GROUP = "g"; + public static final String NAME = "c"; + + public static void main(String[] args) { + System.out.println("hello!"); + } + + public void configure(JobConf jobConf) { + } + + public void close() throws IOException { + } + + @SuppressWarnings("unchecked") + public void map(Object key, Object value, OutputCollector collector, Reporter reporter) throws IOException { + collector.collect(key, value); + reporter.incrCounter(GROUP, NAME, 5l); + } + + @SuppressWarnings("unchecked") + public void reduce(Object key, Iterator values, OutputCollector collector, Reporter reporter) + throws IOException { + while (values.hasNext()) { + collector.collect(key, values.next()); + } + } +} Index: oozie/core/src/test/java/org/apache/oozie/dag/action/hadoop/TestFsActionExecutor.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/action/hadoop/TestFsActionExecutor.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/action/hadoop/TestFsActionExecutor.java (revision 0) @@ -0,0 +1,341 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.action.hadoop; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.oozie.client.Action; +import org.apache.oozie.client.Workflow; +import org.apache.oozie.client.WorkflowClient; +import org.apache.oozie.dag.ActionBean; +import org.apache.oozie.dag.action.ActionExecutor; +import org.apache.oozie.dag.action.ActionExecutorException; +import org.apache.oozie.dag.service.ActionService; +import org.apache.oozie.dag.service.CallbackService; +import org.apache.oozie.service.Services; +import org.apache.oozie.util.ELEvaluator; +import org.apache.oozie.util.XLog; + +import java.util.Properties; + +public class TestFsActionExecutor extends HadoopActionExecutorTestCase { + + public static class Context implements ActionExecutor.Context { + private ActionBean action; + boolean started; + boolean executed; + boolean ended; + private String user; + + public Context(ActionBean action) { + this(action, System.getProperty("user.name")); + } + + public Context(ActionBean action, String user) { + this.action = action; + this.user = user; + } + + public String getCallbackUrl(String externalStatusVar) { + return Services.get().get(CallbackService.class).createCallBackUrl(action.getId(), externalStatusVar); + } + + public Configuration getProtoActionConf() { + Configuration conf = new Configuration(); + conf.set(WorkflowClient.USER_NAME, user); + conf.set(WorkflowClient.GROUP_NAME, "group"); + conf.set("hadoop.job.ugi", user + "," + "group"); + return conf; + } + + public Workflow getWorkflow() { + throw new UnsupportedOperationException(); + } + + public Action getAction() { + throw new UnsupportedOperationException(); + } + + public ELEvaluator getELEvaluator() { + throw new UnsupportedOperationException(); + } + + public void setVar(String name, String value) { + throw new UnsupportedOperationException(); + } + + public String getVar(String name) { + throw new UnsupportedOperationException(); + } + + public void setStartData(String externalId, String trackerUri, String consoleUrl) { + action.setStartData(externalId, trackerUri, consoleUrl); + started = true; + } + + + public void setExecutionData(String externalStatus, Properties actionData) { + action.setExecutionData(externalStatus, actionData); + executed = true; + } + + public void setEndData(Action.Status status, String signalValue) { + action.setEndData(status, signalValue); + ended = true; + } + + public boolean isRetry() { + throw new UnsupportedOperationException(); + } + + public boolean isStarted() { + return started; + } + + public boolean isExecuted() { + return executed; + } + + public boolean isEnded() { + return ended; + } + + public void setExternalStatus(String externalStatus) { + action.setExternalStatus(externalStatus); + } + } + + public void testFs() throws Exception { + ActionService as = Services.get().get(ActionService.class); + ActionExecutor fsExecutor = as.getExecutor(FsActionExecutor.ACTION_TYPE); + + assertEquals(FsActionExecutor.ACTION_TYPE, fsExecutor.getType()); + + ActionBean action = new ActionBean(); + + action.setConf(XLog.format(" ", getFsTestCaseDir())); + fsExecutor.start(new Context(action), action); + assertEquals(Action.Status.DONE, action.getStatus()); + fsExecutor.end(new Context(action), action); + assertEquals(Action.Status.OK, action.getStatus()); + + action.setConf(XLog.format(" ", getFsTestCaseDir())); + fsExecutor.start(new Context(action), action); + assertEquals(Action.Status.DONE, action.getStatus()); + fsExecutor.end(new Context(action), action); + assertEquals(Action.Status.OK, action.getStatus()); + + action.setConf(XLog.format(" ", getFsTestCaseDir())); + fsExecutor.start(new Context(action), action); + assertEquals(Action.Status.DONE, action.getStatus()); + fsExecutor.end(new Context(action), action); + assertEquals(Action.Status.OK, action.getStatus()); + + String source = new Path(getFsTestCaseDir(), "fs-test").toString(); + String target = new Path(getFsTestCaseDir(), "fs-test1").toString(); + action.setConf(XLog.format(" ", source, target)); + fsExecutor.start(new Context(action), action); + assertEquals(Action.Status.DONE, action.getStatus()); + fsExecutor.end(new Context(action), action); + assertEquals(Action.Status.OK, action.getStatus()); + } + + public void testSourceNotFoundException() throws ActionExecutorException { + ActionService as = Services.get().get(ActionService.class); + ActionExecutor fsExecutor = as.getExecutor(FsActionExecutor.ACTION_TYPE); + + ActionBean action = new ActionBean(); + String source = getNameNodeUri() + getTestCaseDir() + "/fs-test"; + String target = getNameNodeUri() + getTestCaseDir() + "/fs-test1"; + action.setConf(XLog.format(" ", source, target)); + try { + fsExecutor.start(new Context(action), action); + fail(); + } + catch (ActionExecutorException e) { + assertEquals(ActionExecutorException.ErrorType.ERROR, e.getErrorType()); + assertEquals("SOURCE_NOT_FOUND", e.getErrorCode()); + } + } + + public void testTargetExistException() throws ActionExecutorException { + ActionService as = Services.get().get(ActionService.class); + ActionExecutor fsExecutor = as.getExecutor(FsActionExecutor.ACTION_TYPE); + + ActionBean action = new ActionBean(); + String fsUri = getFsTestCaseDir().toString(); + action.setConf(XLog.format(" " + " ", + fsUri, getFsTestCaseDir())); + fsExecutor.start(new Context(action), action); + assertEquals(Action.Status.DONE, action.getStatus()); + fsExecutor.end(new Context(action), action); + assertEquals(Action.Status.OK, action.getStatus()); + + String source = new Path(getFsTestCaseDir(), "fs-test").toString(); + String target = new Path(getFsTestCaseDir(), "fs-test1").toString(); + action.setConf(XLog.format(" ", source, target)); + try { + fsExecutor.start(new Context(action), action); + fail(); + } + catch (ActionExecutorException e) { + assertEquals(ActionExecutorException.ErrorType.ERROR, e.getErrorType()); + assertEquals("TARGET_EXISTS", e.getErrorCode()); + } + } + + public void testMkdirTargetExistException() throws ActionExecutorException { + ActionService as = Services.get().get(ActionService.class); + ActionExecutor fsExecutor = as.getExecutor(FsActionExecutor.ACTION_TYPE); + + ActionBean action = new ActionBean(); + Path dir = new Path(getFsTestCaseDir(), "fs-test"); + action.setConf(XLog.format(" ", dir)); + fsExecutor.start(new Context(action), action); + assertEquals(Action.Status.DONE, action.getStatus()); + fsExecutor.end(new Context(action), action); + assertEquals(Action.Status.OK, action.getStatus()); + + action.setConf(XLog.format(" ", dir)); + try { + fsExecutor.start(new Context(action), action); + fail(); + } + catch (ActionExecutorException e) { + + } + catch (Exception e) { + fail(); + } + } + + public void testUnknownHostException() throws ActionExecutorException { + ActionService as = Services.get().get(ActionService.class); + ActionExecutor fsExecutor = as.getExecutor(FsActionExecutor.ACTION_TYPE); + + ActionBean action = new ActionBean(); + String fsUri = "hdfs://blabla:9000"; + action.setConf(XLog.format(" ", fsUri, getTestCaseDir())); + try { + fsExecutor.start(new Context(action), action); + fail(); + } + catch (ActionExecutorException ex) { + assertEquals(ActionExecutorException.ErrorType.TRANSIENT, ex.getErrorType()); + assertEquals("UNKNOWN_HOST", ex.getErrorCode()); + } + catch (Exception e) { + fail(); + } + } + + public void testCouldNotConnectException() throws ActionExecutorException { + ActionService as = Services.get().get(ActionService.class); + ActionExecutor fsExecutor = as.getExecutor(FsActionExecutor.ACTION_TYPE); + + ActionBean action = new ActionBean(); + String fsUri = "hdfs://localhost:32222"; + action.setConf(XLog.format(" ", fsUri, getTestCaseDir())); + try { + fsExecutor.start(new Context(action), action); + fail(); + } + catch (ActionExecutorException ex) { + assertEquals(ActionExecutorException.ErrorType.TRANSIENT, ex.getErrorType()); + assertEquals("COULD_NOT_CONNECT", ex.getErrorCode()); + } + catch (Exception e) { + fail(); + } + } + + public void testAccessDeniedException() throws Exception { + ActionService as = Services.get().get(ActionService.class); + + ActionExecutor fsExecutor = as.getExecutor(FsActionExecutor.ACTION_TYPE); + + ActionBean action = new ActionBean(); + action.setConf(XLog.format(" ", new Path(getFsTestCaseDir(), "test"))); + try { + fsExecutor.start(new Context(action, "otheruser"), action); + fail(); + } + catch (ActionExecutorException ex) { + assertEquals(ActionExecutorException.ErrorType.NON_TRANSIENT, ex.getErrorType()); + assertEquals("DENIED", ex.getErrorCode()); + } + catch (Exception e) { + fail(); + } + } + + public void testScheme() { + ActionService as = Services.get().get(ActionService.class); + ActionExecutor fsExecutor = as.getExecutor(FsActionExecutor.ACTION_TYPE); + + ActionBean action = new ActionBean(); + String fsUri = "file://" + getNameNodeUri(); + action.setConf(XLog.format(" ", fsUri, getTestCaseDir())); + try { + fsExecutor.start(new Context(action), action); + fail(); + } + catch (ActionExecutorException ex) { + assertEquals(ActionExecutorException.ErrorType.ERROR, ex.getErrorType()); + assertEquals("SCHEME_NOT_SUPPORTED", ex.getErrorCode()); + } + catch (Exception e) { + fail(); + } + } + + public void testNoScheme() { + ActionService as = Services.get().get(ActionService.class); + ActionExecutor fsExecutor = as.getExecutor(FsActionExecutor.ACTION_TYPE); + + ActionBean action = new ActionBean(); + String fsUri = "localhost:9000"; + action.setConf(XLog.format(" ", fsUri, getTestCaseDir())); + try { + fsExecutor.start(new Context(action), action); + fail(); + } + catch (ActionExecutorException ex) { + assertEquals(ActionExecutorException.ErrorType.ERROR, ex.getErrorType()); + assertEquals("OTHER", ex.getErrorCode()); + } + catch (Exception e) { + fail(); + } + + fsUri = ""; + action.setConf(XLog.format(" ", fsUri, getTestCaseDir())); + try { + fsExecutor.start(new Context(action), action); + fail(); + } + catch (ActionExecutorException ex) { + assertEquals(ActionExecutorException.ErrorType.ERROR, ex.getErrorType()); + assertEquals("MISSING_SCHEME", ex.getErrorCode()); + } + catch (Exception e) { + fail(); + } + } + +} Index: oozie/core/src/test/java/org/apache/oozie/dag/action/hadoop/TestPigActionExecutor.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/action/hadoop/TestPigActionExecutor.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/action/hadoop/TestPigActionExecutor.java (revision 0) @@ -0,0 +1,177 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.action.hadoop; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobID; +import org.apache.oozie.dag.ActionBean; +import org.apache.oozie.dag.WorkflowBean; +import org.apache.oozie.dag.service.WorkflowAppService; +import org.apache.oozie.util.XConfiguration; +import org.apache.oozie.util.IOUtils; +import org.apache.oozie.util.ClassUtils; +import org.apache.oozie.service.Services; +import org.apache.pig.PigServer; + +import java.io.Writer; +import java.io.InputStream; +import java.io.FileInputStream; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.util.List; +import java.util.ArrayList; + +import jline.ConsoleReaderInputStream; + +public class TestPigActionExecutor extends HadoopActionExecutorTestCase { + private static final int JOB_TIMEOUT = 200 * 1000; + + public void setUp() throws Exception { + super.setUp(); + setSystemProperty(Services.CONF_DELETE_RUNTIME_DIR, "false"); + Services services = new Services(); + cleanUpDB(services.getConf()); + services.init(); + } + + public void testType() { + PigActionExecutor pig = new PigActionExecutor(); + assertEquals(PigActionExecutor.ACTION_TYPE, pig.getType()); + } + + private static final String PIG_SCRIPT = + "set debug on\n" + + "A = load '${IN}' using PigStorage(':');\n" + + "B = foreach A generate $0 as id;\n" + + "store B into '${OUT}' USING PigStorage();\n"; + + public void testPigJob() throws Exception { + FileSystem fs = getFileSystem(); + fs.mkdirs(new Path(getAppPath(), "lib")); + + List jarPaths = new ArrayList(); + + // getting necessary JARs to run PIG + InputStream is = new FileInputStream(ClassUtils.findContainingJar(PigServer.class)); + OutputStream os = fs.create(new Path(getAppPath(), "lib/pig.jar")); + IOUtils.copyStream(is, os); + jarPaths.add("lib/pig.jar"); + is = new FileInputStream(ClassUtils.findContainingJar(ConsoleReaderInputStream.class)); + os = fs.create(new Path(getAppPath(), "lib/jline.jar")); + IOUtils.copyStream(is, os); + jarPaths.add("lib/jline.jar"); + + XConfiguration protoConf = getBaseProtoConf(); + protoConf.setStrings(WorkflowAppService.APP_LIB_JAR_PATH_LIST, jarPaths.toArray(new String[jarPaths.size()])); + WorkflowBean workflow = createBaseWorkflow(protoConf, "P"); + + Path pigPath = new Path(getAppPath(), "my.pig"); + Writer writer = new OutputStreamWriter(fs.create(pigPath)); + writer.write(PIG_SCRIPT); + writer.close(); + + Path inputFile = new Path(getFsTestCaseDir(), "input.txt"); + writer = new OutputStreamWriter(fs.create(inputFile)); + writer.write("hello pig\n"); + writer.close(); + + Path outputDir = new Path(getFsTestCaseDir(), "output"); + + final ActionBean action = (ActionBean) workflow.getActions().get(0); + action.setConf("" + + " " + getJobTrackerUri() + "" + + " " + getNameNodeUri() + "" + + " " + + " " + + " a" + + " A" + + " " + + " " + + " " + + " IN=" + inputFile + "" + + " OUT=" + outputDir + "" + + ""); + + PigActionExecutor pig = new PigActionExecutor(); + + JobConf jobConf = pig.createMapRedJobConf(new Context(workflow, action), action); + final JobClient jobClient = new JobClient(jobConf); + + pig.start(new Context(workflow, action), action); + + waitFor(JOB_TIMEOUT, new Predicate() { + public boolean evaluate() throws Exception { + return jobClient.getJob(JobID.forName(action.getExternalId())).isComplete(); + } + }); + + assertTrue(jobClient.getJob(JobID.forName(action.getExternalId())).isComplete()); + assertTrue(jobClient.getJob(JobID.forName(action.getExternalId())).isSuccessful()); + + assertTrue(fs.exists(outputDir)); + } + + private void _testCleanUp(boolean cleanUp) throws Exception { + FileSystem fs = getFileSystem(); + Services.get().destroy(); + setSystemProperty(PigActionExecutor.CONF_CLEAN_UP_TEMP_DIR, Boolean.toString(cleanUp)); + new Services().init(); + + XConfiguration protoConf = getBaseProtoConf(); + WorkflowBean workflow = createBaseWorkflow(protoConf, "P"); + + Path pigPath = new Path(getAppPath(), "my.pig"); + Writer writer = new OutputStreamWriter(fs.create(pigPath)); + writer.write(PIG_SCRIPT); + writer.close(); + + final ActionBean action = (ActionBean) workflow.getActions().get(0); + action.setConf("" + + " " + getJobTrackerUri() + "" + + " " + getNameNodeUri() + "" + + " " + + " IN=in" + + " OUT=out" + + ""); + + PigActionExecutor pig = new PigActionExecutor(); + + pig.createMapRedJobConf(new Context(workflow, action), action); + + Path path = pig.getActionDir(workflow.getId(), action, PigActionExecutor.ACTION_TYPE, false); + + assertTrue(fs.exists(path)); + + action.setExecutionData("SUCCEEDED", null); + pig.end(new Context(workflow, action), action); + + assertEquals(!cleanUp, fs.exists(path)); + } + + public void testWithCleanUp() throws Exception { + _testCleanUp(true); + } + + public void testWithNoCleanUp() throws Exception { + _testCleanUp(false); + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/dag/action/TestActionExecutor.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/action/TestActionExecutor.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/action/TestActionExecutor.java (revision 0) @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.action; + +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.client.Action; + +import java.io.IOException; +import java.rmi.RemoteException; + +public class TestActionExecutor extends XTestCase { + + private static class MyActionExecutor extends ActionExecutor { + + protected MyActionExecutor() { + super("type"); + } + + public void initActionType() { + super.initActionType(); + registerError("java.rmi.RemoteException", ActionExecutorException.ErrorType.NON_TRANSIENT, "RMI"); + registerError("java.io.IOException", ActionExecutorException.ErrorType.TRANSIENT, "IO"); + registerError("foo.Exception", ActionExecutorException.ErrorType.TRANSIENT, "FO"); + } + + protected MyActionExecutor(int maxRetries, int retryInterval) { + super("type", maxRetries, retryInterval); + } + + public void start(Context context, Action action) throws ActionExecutorException { + assertEquals("type", getType()); + assertEquals(ActionExecutor.MAX_RETRIES, getMaxRetries()); + assertEquals(ActionExecutor.RETRY_INTERVAL, getRetryInterval()); + } + + public void end(Context context, Action action) throws ActionExecutorException { + } + + public void check(Context context, Action action) throws ActionExecutorException { + assertEquals("type", getType()); + assertEquals(1, getMaxRetries()); + assertEquals(2, getRetryInterval()); + } + + public void kill(Context context, Action action) throws ActionExecutorException { + } + + public boolean isCompleted(String externalStatus) { + return true; + } + } + + public void testActionExecutor() throws Exception { + ActionExecutor.enableInit(); + ActionExecutor.resetInitInfo(); + ActionExecutor ae = new MyActionExecutor(); + ae.initActionType(); + ActionExecutor.disableInit(); + + ae.start(null, null); + + ae = new MyActionExecutor(1, 2); + + ae.check(null, null); + + Exception cause = new IOException(); + try { + throw ae.convertException(cause); + } + catch (ActionExecutorException ex) { + assertEquals(cause, ex.getCause()); + assertEquals(ActionExecutorException.ErrorType.TRANSIENT, ex.getErrorType()); + assertEquals("IO", ex.getErrorCode()); + } + catch (Exception ex) { + fail(); + } + + cause = new RemoteException(); + try { + throw ae.convertException(cause); + } + catch (ActionExecutorException ex) { + assertEquals(cause, ex.getCause()); + assertEquals(ActionExecutorException.ErrorType.NON_TRANSIENT, ex.getErrorType()); + assertEquals("RMI", ex.getErrorCode()); + } + catch (Exception ex) { + fail(); + } + + cause = new RuntimeException(); + try { + throw ae.convertException(cause); + } + catch (ActionExecutorException ex) { + assertEquals(cause, ex.getCause()); + assertEquals(ActionExecutorException.ErrorType.ERROR, ex.getErrorType()); + assertEquals(ActionExecutor.ERROR_OTHER, ex.getErrorCode()); + } + catch (Exception ex) { + fail(); + } + + cause = new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "x", "x"); + try { + throw ae.convertException(cause); + } + catch (ActionExecutorException ex) { + assertEquals(cause, ex); + } + catch (Exception ex) { + fail(); + } + + } +} Index: oozie/core/src/test/java/org/apache/oozie/dag/action/decision/TestDecisionActionExecutor.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/action/decision/TestDecisionActionExecutor.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/action/decision/TestDecisionActionExecutor.java (revision 0) @@ -0,0 +1,168 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.action.decision; + +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.dag.action.ActionExecutor; +import org.apache.oozie.dag.action.ActionExecutorException; +import org.apache.oozie.dag.ActionBean; +import org.apache.oozie.dag.service.CallbackService; +import org.apache.oozie.client.Workflow; +import org.apache.oozie.client.Action; +import org.apache.oozie.util.ELEvaluator; +import org.apache.oozie.service.Services; +import org.apache.hadoop.conf.Configuration; + +import java.util.Properties; + +public class TestDecisionActionExecutor extends XTestCase { + + private static class Context implements ActionExecutor.Context { + private ActionBean action; + boolean executed; + boolean ended; + + public Context(ActionBean action) { + this.action = action; + } + + public String getCallbackUrl(String externalStatusVar) { + return Services.get().get(CallbackService.class).createCallBackUrl(action.getId(), externalStatusVar); + } + + public Action getAction() { + return action; + } + + public Configuration getProtoActionConf() { + throw new UnsupportedOperationException(); + } + + public Workflow getWorkflow() { + throw new UnsupportedOperationException(); + } + + public ELEvaluator getELEvaluator() { + throw new UnsupportedOperationException(); + } + + public void setVar(String name, String value) { + throw new UnsupportedOperationException(); + } + + public String getVar(String name) { + throw new UnsupportedOperationException(); + } + + public void setStartData(String externalId, String trackerUri, String consoleUrl) { + action.setStartData(externalId, trackerUri, consoleUrl); + } + + public void setExecutionData(String externalStatus, Properties actionData) { + action.setExecutionData(externalStatus, actionData); + executed = true; + } + + public void setEndData(Action.Status status, String signalValue) { + action.setEndData(status, signalValue); + ended = true; + } + + public boolean isRetry() { + throw new UnsupportedOperationException(); + } + + public boolean isStarted() { + throw new UnsupportedOperationException(); + } + + public boolean isExecuted() { + throw new UnsupportedOperationException(); + } + + public boolean isEnded() { + return ended; + } + + public void setExternalStatus(String externalStatus) { + action.setExternalStatus(externalStatus); + } + } + + public void testDecision() throws Exception { + ActionExecutor decision = new DecisionActionExecutor(); + + assertEquals(DecisionActionExecutor.ACTION_TYPE, decision.getType()); + + ActionBean action = new ActionBean(); + action.setConf("" + + "true" + + "true" + + "false" + + ""); + + decision.start(new Context(action), action); + assertEquals(Action.Status.DONE, action.getStatus()); + decision.end(new Context(action), action); + assertEquals(Action.Status.OK, action.getStatus()); + assertEquals("a", action.getExternalStatus()); + + action.setConf("" + + "false" + + "true" + + "false" + + ""); + + decision.start(new Context(action), action); + assertEquals(Action.Status.DONE, action.getStatus()); + decision.end(new Context(action), action); + assertEquals(Action.Status.OK, action.getStatus()); + assertEquals("b", action.getExternalStatus()); + + + action.setConf("" + + "false" + + "false" + + "false" + + ""); + + decision.start(new Context(action), action); + assertEquals(Action.Status.DONE, action.getStatus()); + decision.end(new Context(action), action); + assertEquals(Action.Status.OK, action.getStatus()); + assertEquals("d", action.getExternalStatus()); + + try { + action.setConf("" + + "false" + + "false" + + "false" + + ""); + + decision.start(new Context(action), action); + fail(); + } + catch (ActionExecutorException ex) { + assertEquals(ActionExecutorException.ErrorType.FAILED, ex.getErrorType()); + assertEquals(DecisionActionExecutor.XML_ERROR, ex.getErrorCode()); + } + catch (Exception ex) { + fail(); + } + } +} Index: oozie/core/src/test/java/org/apache/oozie/dag/action/ssh/TestSshActionExecutor.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/action/ssh/TestSshActionExecutor.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/action/ssh/TestSshActionExecutor.java (revision 0) @@ -0,0 +1,333 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.action.ssh; + +import org.apache.oozie.dag.service.CallbackService; + +import org.apache.oozie.util.PropertiesUtils; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.oozie.client.Workflow; +import org.apache.oozie.client.WorkflowClient; +import org.apache.oozie.client.Action.Status; +import org.apache.oozie.dag.ActionBean; +import org.apache.oozie.dag.WorkflowBean; +import org.apache.oozie.dag.action.ActionExecutor; +import org.apache.oozie.dag.action.ActionExecutorException; +import org.apache.oozie.dag.action.hadoop.FsActionExecutor; +import org.apache.oozie.dag.service.WorkflowAppService; +import org.apache.oozie.service.Services; +import org.apache.oozie.service.UUIDService; +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.ELEvaluator; +import org.apache.oozie.util.XConfiguration; + +public class TestSshActionExecutor extends XTestCase { + + private Services services; + + private static class Context implements ActionExecutor.Context { + private ActionBean action; + private WorkflowBean workflow; + + public Context(WorkflowBean workflow, ActionBean action) { + this.workflow = workflow; + this.action = action; + } + + public Configuration getProtoActionConf() { + String s = workflow.getProtoActionConf(); + try { + return new XConfiguration(new StringReader(s)); + } + catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + public Workflow getWorkflow() { + return workflow; + } + + public ELEvaluator getELEvaluator() { + throw new UnsupportedOperationException(); + } + + public void setVar(String name, String value) { + throw new UnsupportedOperationException(); + } + + public String getVar(String name) { + throw new UnsupportedOperationException(); + } + + public boolean isRetry() { + throw new UnsupportedOperationException(); + } + + @Override + public void setExternalStatus(String externalStatus) { + action.setExternalStatus(externalStatus); + } + + @Override + public void setEndData(Status status, String signalValue) { + action.setEndData(status, signalValue); + } + + @Override + public void setExecutionData(String externalStatus, Properties actionData) { + action.setExecutionData(externalStatus, actionData); + } + + @Override + public void setStartData(String externalId, String trackerUri, String consoleUrl) { + action.setStartData(externalId, trackerUri, consoleUrl); + } + + @Override + public String getCallbackUrl(String externalStatusVar) { + return Services.get().get(CallbackService.class).createCallBackUrl(action.getId(), externalStatusVar); } + } + + protected void setUp() throws Exception { + super.setUp(); + services = new Services(); + services.init(); + + XConfiguration conf = new XConfiguration(); + conf.setStrings(WorkflowAppService.HADOOP_USER, System.getProperty("user.name")); + conf.setStrings(WorkflowAppService.HADOOP_UGI, System.getProperty("user.name") + ",other"); + Path path = new Path(getNameNodeUri(), getTestCaseDir()); + FileSystem fs = new FsActionExecutor().getFileSystem(path.toUri(), conf); + fs.delete(path, true); + } + + public void testJobStart() throws ActionExecutorException { + String baseDir = getTestCaseDir(); + Path appPath = new Path(getNameNodeUri(), baseDir); + + XConfiguration protoConf = new XConfiguration(); + protoConf.setStrings(WorkflowAppService.HADOOP_USER, System.getProperty("user.name")); + protoConf.setStrings(WorkflowAppService.HADOOP_UGI, System.getProperty("user.name") + ",other"); + + XConfiguration wfConf = new XConfiguration(); + wfConf.set(WorkflowClient.APP_PATH, appPath.toString()); + + WorkflowBean workflow = new WorkflowBean(); + workflow.setConf(wfConf.toXmlString()); + workflow.setAppPath(wfConf.get(WorkflowClient.APP_PATH)); + workflow.setProtoActionConf(protoConf.toXmlString()); + workflow.setId(Services.get().get(UUIDService.class).generateId()); + + final ActionBean action = new ActionBean(); + action.setId("actionId"); + action.setConf("" + + "localhost" + + "echo" + + "" + + "\"prop1=something\"" + + ""); + action.setName("ssh"); + final SshActionExecutor ssh = new SshActionExecutor(); + final Context context = new Context(workflow, action); + ssh.start(context, action); + + waitFor(30 * 1000, new Predicate() { + public boolean evaluate() throws Exception { + ssh.check(context, action); + return Status.DONE == action.getStatus(); + } + }); + ssh.end(context, action); + assertEquals(Status.OK, action.getStatus()); + assertEquals("something", PropertiesUtils.stringToProperties(action.getData()).getProperty("prop1")); + } + + public void testJobRecover() throws ActionExecutorException, InterruptedException { + String baseDir = getTestCaseDir(); + Path appPath = new Path(getNameNodeUri(), baseDir); + + XConfiguration protoConf = new XConfiguration(); + protoConf.setStrings(WorkflowAppService.HADOOP_USER, System.getProperty("user.name")); + protoConf.setStrings(WorkflowAppService.HADOOP_UGI, System.getProperty("user.name") + ",other"); + + XConfiguration wfConf = new XConfiguration(); + wfConf.set(WorkflowClient.APP_PATH, appPath.toString()); + + WorkflowBean workflow = new WorkflowBean(); + workflow.setConf(wfConf.toXmlString()); + workflow.setAppPath(wfConf.get(WorkflowClient.APP_PATH)); + workflow.setProtoActionConf(protoConf.toXmlString()); + workflow.setId(Services.get().get(UUIDService.class).generateId()); + + final ActionBean action = new ActionBean(); + action.setId("actionId"); + action.setConf("" + + "localhost" + + "echo" + + "" + + "\"prop1=something\"" + + ""); + action.setName("ssh"); + final SshActionExecutor ssh = new SshActionExecutor(); + final Context context = new Context(workflow, action); + ssh.start(context, action); + + Thread.sleep(200); + final ActionBean action1 = new ActionBean(); + action1.setId("actionId"); + action1.setConf("" + + "localhost" + + "echo" + + "" + + "\"prop1=nothing\"" + + ""); + action1.setName("ssh"); + final SshActionExecutor ssh1 = new SshActionExecutor(); + final Context context1 = new Context(workflow, action1); + Thread.sleep(500); + ssh1.start(context1, action1); + assertEquals(action1.getExternalId(), action.getExternalId()); + + waitFor(30 * 1000, new Predicate() { + public boolean evaluate() throws Exception { + ssh.check(context1, action1); + return Status.DONE == action1.getStatus(); + } + }); + ssh1.end(context1, action1); + assertEquals(Status.OK, action1.getStatus()); + assertEquals("something", PropertiesUtils.stringToProperties(action1.getData()).getProperty("prop1")); + } + + // TODO Move this test case over to a new class. Conflict between this one + // and testConnectionErrors. The property to replace the ssh user cannot be + // reset in a good way during runtime. + // +// public void testOozieUserMismatch() throws ActionExecutorException { +// String baseDir = getTestCaseDir(); +// Path appPath = new Path(getNameNodeUri(), baseDir); +// +// Services.get().getConf().setBoolean(SshActionExecutor.CONF_SSH_ALLOW_USER_AT_HOST, false); +// XConfiguration protoConf = new XConfiguration(); +// protoConf.setStrings(WorkflowAppService.HADOOP_USER, System.getProperty("user.name")); +// protoConf.setStrings(WorkflowAppService.HADOOP_UGI, System.getProperty("user.name") + ",other"); +// +// XConfiguration wfConf = new XConfiguration(); +// wfConf.set(WorkflowClient.APP_PATH, appPath.toString()); +// +// WorkflowBean workflow = new WorkflowBean(); +// workflow.setConf(wfConf.toXmlString()); +// workflow.setAppPath(wfConf.get(WorkflowClient.APP_PATH)); +// workflow.setProtoActionConf(protoConf.toXmlString()); +// workflow.setId("wfId"); +// +// final ActionBean action = new ActionBean(); +// action.setId("actionId_" + System.currentTimeMillis()); +// action.setConf("" + +// "invalid@localhost" + +// "echo" + +// "" + +// "\"prop1=something\"" + +// ""); +// action.setName("ssh"); +// +// final SshActionExecutor ssh = new SshActionExecutor(); +// +// final Context context = new Context(workflow, action); +// try { +// ssh.start(context, action); +// assertTrue(false); +// } catch (ActionExecutorException ex) { +// System.err.println("Caught exception, Error Code: " + ex.getErrorCode()); +// assertEquals(SshActionExecutor.ERR_USER_MISMATCH, ex.getErrorCode()); +// } +// } + + public void testConnectionErrors() throws ActionExecutorException { + String baseDir = getTestCaseDir(); + Path appPath = new Path(getNameNodeUri(), baseDir); + + XConfiguration protoConf = new XConfiguration(); + protoConf.setStrings(WorkflowAppService.HADOOP_USER, System.getProperty("user.name")); + protoConf.setStrings(WorkflowAppService.HADOOP_UGI, System.getProperty("user.name") + ",other"); + + XConfiguration wfConf = new XConfiguration(); + wfConf.set(WorkflowClient.APP_PATH, appPath.toString()); + + WorkflowBean workflow = new WorkflowBean(); + workflow.setConf(wfConf.toXmlString()); + workflow.setAppPath(wfConf.get(WorkflowClient.APP_PATH)); + workflow.setProtoActionConf(protoConf.toXmlString()); + workflow.setId(Services.get().get(UUIDService.class).generateId()); + + final ActionBean action = new ActionBean(); + action.setId("actionId"); + action.setConf("" + + "blabla" + + "echo" + + "\"prop1=something\"" + + ""); + action.setName("ssh"); + final SshActionExecutor ssh = new SshActionExecutor(); + final Context context = new Context(workflow, action); + try { + ssh.start(context, action); + } + catch (ActionExecutorException ex) { + assertEquals("COULD_NOT_RESOLVE_HOST", ex.getErrorCode()); + assertEquals(ActionExecutorException.ErrorType.TRANSIENT, ex.getErrorType()); + } + action.setConf("" + + "11.11.11.11" + + "echo" + + "\"prop1=something\"" + + ""); + try { + ssh.start(context, action); + } + catch (ActionExecutorException ex) { + assertEquals("COULD_NOT_CONNECT", ex.getErrorCode()); + assertEquals(ActionExecutorException.ErrorType.TRANSIENT, ex.getErrorType()); + } + action.setConf("" + + "y@localhost" + + "echo" + + "\"prop1=something\"" + + ""); + try { + ssh.start(context, action); + } + catch (ActionExecutorException ex) { + assertEquals("AUTH_FAILED", ex.getErrorCode()); + assertEquals(ActionExecutorException.ErrorType.NON_TRANSIENT, ex.getErrorType()); + } + } + + protected void tearDown() throws Exception { + services.destroy(); + super.tearDown(); + } +} Index: oozie/core/src/test/java/org/apache/oozie/dag/command/TestActionErrors.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/command/TestActionErrors.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/command/TestActionErrors.java (revision 0) @@ -0,0 +1,433 @@ +/** + * + */ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.command; + +import java.io.FileWriter; +import java.io.Reader; +import java.io.Writer; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.oozie.client.Workflow; +import org.apache.oozie.client.WorkflowClient; +import org.apache.oozie.dag.ActionBean; +import org.apache.oozie.dag.DagEngine; +import org.apache.oozie.dag.ForTestingActionExecutor; +import org.apache.oozie.dag.WorkflowBean; +import org.apache.oozie.dag.service.ActionService; +import org.apache.oozie.dag.service.WorkflowStoreService; +import org.apache.oozie.dag.service.WorkflowSchemaService; +import org.apache.oozie.dag.store.WorkflowStore; +import org.apache.oozie.dag.workflow.WorkflowInstance; +import org.apache.oozie.service.Services; +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.IOUtils; +import org.apache.oozie.util.XConfiguration; + +/** + * Test cases for checking correct functionality in case of errors while + * executing Actions. + */ +public class TestActionErrors extends XTestCase { + + private Services services; + + @Override + public void setUp() throws Exception { + super.setUp(); + setSystemProperty(WorkflowSchemaService.CONF_EXT_SCHEMAS, "wf-ext-schema.xsd"); + services = new Services(); + cleanUpDB(services.getConf()); + services.init(); + services.get(ActionService.class).register(ForTestingActionExecutor.class); + } + + public void tearDown() { + services.destroy(); + } + + /** + * Tests for correct functionality when a + * {@link org.apache.oozie.dag.action.ActionExecutorException.ErrorType#NON_TRANSIENT} error is + * generated while attempting to start an action. + *

+ * It first generates a + * {@link org.apache.oozie.dag.action.ActionExecutorException.ErrorType#NON_TRANSIENT} error and checks + * for the job to go into {@link Workflow.Status#SUSPENDED} state. The state + * of the single action in the job is checked to be at + * {@link ActionBean.Status#START_MANUAL} and it's error code and error + * message are verified. + *

+ * The job is subsequently fixed to not generate any errors, and is resumed. + * The job state and the action state are verified to be + * {@link Workflow.Status#SUCCEEDED} and {@link ActionBean.Status#OK} + * respectively. The action error code and error message are checked to be + * emtpy. + * + * @throws Exception + */ + public void testStartNonTransient() throws Exception { + _testNonTransient("start.non-transient", ActionBean.Status.START_MANUAL, "start"); + assertTrue(true); + } + + /** + * Tests for correct functionality when a + * {@link org.apache.oozie.dag.action.ActionExecutorException.ErrorType#NON_TRANSIENT} error is + * generated while attempting to end an action. + *

+ * It first generates a + * {@link org.apache.oozie.dag.action.ActionExecutorException.ErrorType#NON_TRANSIENT} error and checks + * for the job to go into {@link Workflow.Status#SUSPENDED} state. The state + * of the single action in the job is checked to be at + * {@link ActionBean.Status#END_MANUAL} and it's error code and error + * message are verified. + *

+ * The job is subsequently fixed to not generate any errors, and is resumed. + * The job state and the action state are verified to be + * {@link Workflow.Status#SUCCEEDED} and {@link ActionBean.Status#OK} + * respectively. The action error code and error message are checked to be + * emtpy. + * + * @throws Exception + */ + public void testEndNonTransient() throws Exception { + _testNonTransient("end.non-transient", ActionBean.Status.END_MANUAL, "end"); + assertTrue(true); + } + + /** + * Tests for correct functionality when a + * {@link org.apache.oozie.dag.action.ActionExecutorException.ErrorType#TRANSIENT} error is generated + * when trying to start an action. + *

+ * It first generates a {@link org.apache.oozie.dag.action.ActionExecutorException.ErrorType#TRANSIENT} + * error. 2 retries with an interval of 10 seconds between them are allowed. + * The state of the action is checked after each attempt to be at + * {@link ActionBean.Status#START_RETRY}. Error message and Error code for + * the action are verified. + *

+ * After the configured number of retry attempts, the job and actions status + * are checked to be {@link Workflow.Status#SUSPENDED} and + * {@link ActionBean.Status#END_MANUAL} respectively. The error message and + * code are verified again. + * + * @throws Exception + */ + public void testStartTransient() throws Exception { + _testTransient("start.transient", ActionBean.Status.START_RETRY, ActionBean.Status.START_MANUAL, "start"); + assertTrue(true); + } + + /** + * Tests for correct functionality when a + * {@link org.apache.oozie.dag.action.ActionExecutorException.ErrorType#TRANSIENT} error is generated + * when trying to end an action. + *

+ * It first generates a {@link org.apache.oozie.dag.action.ActionExecutorException.ErrorType#TRANSIENT} + * error. 2 retries with an interval of 10 seconds between them are allowed. + * The state of the action is checked after each attempt to be at + * {@link ActionBean.Status#END_RETRY}. Error message and Error code for + * the action are verified. + *

+ * After the configured number of retry attempts, the job and actions status + * are checked to be {@link Workflow.Status#SUSPENDED} and + * {@link ActionBean.Status#START_MANUAL} respectively. The error message + * and code are verified again. + * + * @throws Exception + */ + public void testEndTransient() throws Exception { + _testTransient("end.transient", ActionBean.Status.END_RETRY, ActionBean.Status.END_MANUAL, "end"); + assertTrue(true); + } + + /** + * Tests for correct functionality when a + * {@link org.apache.oozie.dag.action.ActionExecutorException.ErrorType#ERROR} is generated when + * executing start. + *

+ * Checks for the job to go into {@link Workflow.Status#KILLED} state. + * + * @throws Exception + */ + public void testStartError() throws Exception { + _testError("start.error", "error", "based_on_action_status"); + assertTrue(true); + } + + /** + * Tests for correct functionality when a + * {@link org.apache.oozie.dag.action.ActionExecutorException.ErrorType#ERROR} is generated when + * executing end. + *

+ * Checks for the job to go into {@link Workflow.Status#KILLED} state. + * + * @throws Exception + */ + public void testEndError() throws Exception { + _testError("end.error", "ok", "OK"); + assertTrue(true); + } + + /** + * Tests for the job to be KILLED and status set to FAILED in case an Action + * Handler does not call setExecutionData in it's start() implementation. + * + * @throws Exception + */ + public void testExecutionDataNotSet() throws Exception { + _testDataNotSet("avoid-set-execution-data", ActionStartCommand.START_DATA_MISSING); + } + + /** + * Tests for the job to be KILLED and status set to FAILED in case an Action + * Handler does not call setEndData in it's end() implementation. + * + * @throws Exception + */ + public void testEndDataNotSet() throws Exception { + _testDataNotSet("avoid-set-end-data", ActionEndCommand.END_DATA_MISSING); + } + + /** + * Provides functionality to test non transient failures. + * + * @param errorType the error type. (start.non-transient, end.non-transient) + * @param expStatus1 expected status. (START_MANUAL, END_MANUAL) + * @param expErrorMsg expected error message. + * @throws Exception + */ + private void _testNonTransient(String errorType, ActionBean.Status expStatus1, String expErrorMsg) throws Exception { + Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1); + Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); + IOUtils.copyCharStream(reader, writer); + + final DagEngine engine = new DagEngine("u", "g", "a"); + Configuration conf = new XConfiguration(); + conf.set(WorkflowClient.APP_PATH, getTestCaseDir()); + conf.set(WorkflowClient.USER_NAME, "u"); + conf.set(WorkflowClient.GROUP_NAME, "g"); + conf.set(WorkflowClient.LOG_TOKEN, "t"); + conf.set("signal-value", "OK"); + conf.set("external-status", "ok"); + conf.set("error", errorType); + + final String jobId = engine.submitJob(conf, true); + + waitFor(5000, new Predicate() { + public boolean evaluate() throws Exception { + return (engine.getJob(jobId).getStatus() == Workflow.Status.SUSPENDED); + } + }); + + final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create(); + List actions = store.getActionsForWorkflow(jobId, true); + ActionBean action = actions.get(0); + assertEquals("TEST_ERROR", action.getErrorCode()); + assertEquals(expErrorMsg, action.getErrorMessage()); + assertEquals(expStatus1, action.getStatus()); + + assertTrue(engine.getJob(jobId).getStatus() == Workflow.Status.SUSPENDED); + + String actionConf = action.getConf(); + String fixedActionConf = actionConf.replaceAll(errorType, "none"); + action.setConf(fixedActionConf); + store.updateAction(action); + store.commit(); + store.close(); + + engine.resume(jobId); + + waitFor(5000, new Predicate() { + public boolean evaluate() throws Exception { + return (engine.getJob(jobId).getStatus() == Workflow.Status.SUCCEEDED); + } + }); + + assertEquals(Workflow.Status.SUCCEEDED, engine.getJob(jobId).getStatus()); + + final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create(); + actions = store2.getActionsForWorkflow(jobId, false); + action = actions.get(0); + assertEquals(null, action.getErrorCode()); + assertEquals(null, action.getErrorMessage()); + assertEquals(ActionBean.Status.OK, action.getStatus()); + store2.close(); + } + + /** + * Provides functionality to test non transient errors. + * + * @param errorType the error type. (start.non-transient, end.non-transient) + * @param expStatus1 expected status. (START_MANUAL, END_MANUAL) + * @param expErrorMsg the expected error message. + * @throws Exception + */ + + /** + * Provides functionality to test transient failures. + * + * @param errorType the error type. (start.transient, end.transient) + * @param expStatus1 expected status after the first step (START_RETRY, + * END_RETRY) + * @param expStatus2 expected status after the second step (START_MANUAL, + * END_MANUAL) + * @param expErrorMsg the expected error message. + * @throws Exception + */ + private void _testTransient(String errorType, ActionBean.Status expStatus1, ActionBean.Status expStatus2, + String expErrorMsg) throws Exception { + Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1); + Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); + IOUtils.copyCharStream(reader, writer); + + final int maxRetries = 2; + final int retryInterval = 10; + + final DagEngine engine = new DagEngine("u", "g", "a"); + Configuration conf = new XConfiguration(); + conf.set(WorkflowClient.APP_PATH, getTestCaseDir()); + conf.set(WorkflowClient.USER_NAME, "u"); + conf.set(WorkflowClient.GROUP_NAME, "g"); + conf.set(WorkflowClient.LOG_TOKEN, "t"); + conf.set("signal-value", "OK"); + conf.set("external-status", "ok"); + conf.set("error", errorType); + conf.setInt(WorkflowClient.ACTION_MAX_RETRIES, maxRetries); + conf.setInt(WorkflowClient.ACTION_RETRY_INTERVAL, retryInterval); + + final String jobId = engine.submitJob(conf, true); + + int retryCount = 1; + ActionBean.Status expectedStatus = expStatus1; + int expectedRetryCount = 1; + + final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create(); + + Thread.sleep(2000); + while (retryCount <= maxRetries) { + List actions = store.getActionsForWorkflow(jobId, false); + ActionBean action = actions.get(0); + assertEquals(expectedStatus, action.getStatus()); + assertEquals(expectedRetryCount, action.getRetries()); + assertEquals("TEST_ERROR", action.getErrorCode()); + assertEquals(expErrorMsg, action.getErrorMessage()); + if (action.getRetries() == maxRetries) { + expectedRetryCount = 0; + expectedStatus = expStatus2; + } + else { + expectedRetryCount++; + } + Thread.sleep(retryInterval * 1000); + retryCount++; + } + + List actions = store.getActionsForWorkflow(jobId, false); + ActionBean action = actions.get(0); + assertEquals(expStatus2, action.getStatus()); + assertEquals("TEST_ERROR", action.getErrorCode()); + assertEquals(expErrorMsg, action.getErrorMessage()); + + assertEquals(Workflow.Status.SUSPENDED, engine.getJob(jobId).getStatus()); + store.close(); + } + + /** + * Provides functionality to test errors + * + * @param errorType the error type. (start.non-transient, end.non-transient) + * @param externalStatus the external status to set. + * @param signalValue the signal value to set. + * @throws Exception + */ + private void _testError(String errorType, String externalStatus, String signalValue) throws Exception { + Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1); + Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); + IOUtils.copyCharStream(reader, writer); + + final DagEngine engine = new DagEngine("u", "g", "a"); + Configuration conf = new XConfiguration(); + conf.set(WorkflowClient.APP_PATH, getTestCaseDir()); + conf.set(WorkflowClient.USER_NAME, "u"); + conf.set(WorkflowClient.GROUP_NAME, "g"); + conf.set(WorkflowClient.LOG_TOKEN, "t"); + conf.set("error", errorType); + conf.set("external-status", externalStatus); + conf.set("signal-value", signalValue); + + final String jobId = engine.submitJob(conf, true); + + final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create(); + + waitFor(5000, new Predicate() { + public boolean evaluate() throws Exception { + WorkflowBean bean = store.getWorkflow(jobId, false); + return (bean.getWorkflowInstance().getStatus() == WorkflowInstance.Status.KILLED); + } + }); + assertEquals(Workflow.Status.KILLED, engine.getJob(jobId).getStatus()); + store.close(); + } + + /** + * Provides functionality to test for set*Data calls not being made by the + * Action Handler. + * + * @param avoidParam set*Data function call to avoid. + * @param expActionErrorCode the expected action error code. + * @throws Exception + */ + private void _testDataNotSet(String avoidParam, String expActionErrorCode) throws Exception { + Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1); + Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); + IOUtils.copyCharStream(reader, writer); + + final DagEngine engine = new DagEngine("u", "g", "a"); + Configuration conf = new XConfiguration(); + conf.set(WorkflowClient.APP_PATH, getTestCaseDir()); + conf.set(WorkflowClient.USER_NAME, "u"); + conf.set(WorkflowClient.GROUP_NAME, "g"); + conf.set(WorkflowClient.LOG_TOKEN, "t"); + conf.set("external-status", "ok"); + conf.set("signal-value", "based_on_action_status"); + conf.set(avoidParam, "true"); + + final String jobId = engine.submitJob(conf, true); + + final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create(); + + waitFor(5000, new Predicate() { + public boolean evaluate() throws Exception { + WorkflowBean bean = store.getWorkflow(jobId, false); + return (bean.getWorkflowInstance().getStatus() == WorkflowInstance.Status.FAILED); + } + }); + assertEquals(WorkflowInstance.Status.FAILED, store.getWorkflow(jobId, false).getWorkflowInstance().getStatus()); + assertEquals(Workflow.Status.FAILED, engine.getJob(jobId).getStatus()); + + List actions = store.getActionsForWorkflow(jobId, false); + ActionBean action = actions.get(0); + assertEquals(expActionErrorCode, action.getErrorCode()); + + store.close(); + } +} Index: oozie/core/src/test/java/org/apache/oozie/dag/command/TestDagCommand.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/command/TestDagCommand.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/command/TestDagCommand.java (revision 0) @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.command; + +import org.apache.oozie.dag.store.StoreException; +import org.apache.oozie.dag.store.WorkflowStore; +import org.apache.oozie.dag.service.DagXLogInfoService; +import org.apache.oozie.service.Services; +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.XCallable; +import org.apache.oozie.util.XLog; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class TestDagCommand extends XTestCase { + private static List EXECUTED = Collections.synchronizedList(new ArrayList()); + + private static class DummyXCallable implements XCallable { + private String name; + + public DummyXCallable(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public int getPriority() { + return 0; + } + + public Void call() throws Exception { + EXECUTED.add(name); + return null; + } + } + + private static class MyCommand extends DagCommand { + private boolean exception; + + public MyCommand(boolean exception) { + super("test", 1, XLog.OPS); + this.exception = exception; + } + + protected Object call(WorkflowStore store) throws StoreException, DagCommandException { + assertTrue(logInfo.createPrefix().contains("JOB[job]")); + assertTrue(XLog.Info.get().createPrefix().contains("JOB[job]")); + assertTrue(logInfo.createPrefix().contains("ACTION[action]")); + assertTrue(XLog.Info.get().createPrefix().contains("ACTION[action]")); + assertNotNull(store); + assertEquals("test", getName()); + assertEquals(1, getPriority()); + queueCallable(new DummyXCallable("a")); + queueCallable(Arrays.asList(new DummyXCallable("b"), new DummyXCallable("c"))); + queueCallable(new DummyXCallable("d"), 300); + queueCallable(new DummyXCallable("e"), 200); + queueCallable(new DummyXCallable("f"), 100); + queueCallableForException(new DummyXCallable("ex")); + if (exception) { + throw new DagCommandException(DagCommandException.ErrorCode.E1400); + } + return null; + } + } + + public void testDagCommand() throws Exception { + Services services = new Services(); + services.init(); + + XLog.Info.get().clear(); + XLog.Info.get().setParameter(DagXLogInfoService.JOB, "job"); + XLog.Info.get().setParameter(DagXLogInfoService.ACTION, "action"); + + DagCommand command = new MyCommand(false); + + XLog.Info.get().clear(); + command.call(); + + assertTrue(XLog.Info.get().createPrefix().contains("JOB[job]")); + assertTrue(XLog.Info.get().createPrefix().contains("ACTION[action]")); + command.resetLogInfoWorkflow(); + assertTrue(XLog.Info.get().createPrefix().contains("JOB[-]")); + assertTrue(XLog.Info.get().createPrefix().contains("ACTION[action]")); + command.resetLogInfoAction(); + assertTrue(XLog.Info.get().createPrefix().contains("ACTION[-]")); + + waitFor(2000, new Predicate() { + public boolean evaluate() throws Exception { + return EXECUTED.size() == 6; + } + }); + + assertEquals(6, EXECUTED.size()); + assertEquals(Arrays.asList("a", "b", "c", "d", "e", "f"), EXECUTED); + + EXECUTED.clear(); + + XLog.Info.get().setParameter(DagXLogInfoService.JOB, "job"); + XLog.Info.get().setParameter(DagXLogInfoService.ACTION, "action"); + command = new MyCommand(true); + + try { + command.call(); + fail(); + } + catch (DagCommandException ex) { + //nop + } + + waitFor(200, new Predicate() { + public boolean evaluate() throws Exception { + return EXECUTED.size() == 2; + } + }); + + assertEquals(1, EXECUTED.size()); + assertEquals(Arrays.asList("ex"), EXECUTED); + + + services.destroy(); + } + +} Index: oozie/core/src/test/java/org/apache/oozie/dag/command/TestReRunCommand.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/dag/command/TestReRunCommand.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/dag/command/TestReRunCommand.java (revision 0) @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.dag.command; + +import java.util.Properties; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Reader; +import java.io.Writer; +import org.apache.hadoop.fs.Path; +import org.apache.oozie.dag.local.LocalOozie; +import org.apache.oozie.client.Workflow; +import org.apache.oozie.client.WorkflowClient; +import org.apache.oozie.client.WorkflowClientException; +import org.apache.oozie.test.XFsTestCase; +import org.apache.oozie.util.IOUtils; + +public class TestReRunCommand extends XFsTestCase { + @Override + protected void setUp() throws Exception { + super.setUp(); + LocalOozie.start(); + } + + @Override + protected void tearDown() throws Exception { + LocalOozie.stop(); + super.tearDown(); + } + + public void testRerun() throws IOException, WorkflowClientException { + Reader reader = IOUtils.getResourceAsReader("rerun-wf.xml", -1); + Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); + IOUtils.copyCharStream(reader, writer); + + final WorkflowClient wfClient = LocalOozie.getClient(); + Properties conf = wfClient.createConfiguration(); + conf.setProperty(WorkflowClient.APP_PATH, getTestCaseDir()); + conf.setProperty("inPath", getFsTestCaseDir().toString()); + conf.setProperty("checkDir", getFsTestCaseDir().toString() + "/input1"); + + final String jobId1 = wfClient.submit(conf); + wfClient.start(jobId1); + waitFor(5 * 1000, new Predicate() { + public boolean evaluate() throws Exception { + return wfClient.getJobInfo(jobId1).getStatus() == Workflow.Status.KILLED; + } + }); + assertEquals(wfClient.getJobInfo(jobId1).getStatus(), Workflow.Status.KILLED); + Properties skipConf = wfClient.createConfiguration(); + // Test to skip a failed node + skipConf.setProperty(WorkflowClient.APP_PATH, getTestCaseDir()); + skipConf.setProperty("inPath", getFsTestCaseDir().toString()); + skipConf.setProperty("checkDir", getFsTestCaseDir().toString() + "/input1"); + skipConf.setProperty(WorkflowClient.RERUN_SKIP_NODES, "hdfs11,hdfs32"); + + boolean failed = false; + try { + wfClient.reRun(jobId1, skipConf); + } + catch (WorkflowClientException e) { + failed = true; + assertEquals(true, e.getCause().getMessage().startsWith( + DagCommandException.ErrorCode.E1407.getTemplate().split("\\[")[0])); + } + assertEquals(true, failed); + Path inputDir = new Path(getFsTestCaseDir(), "input1"); + getFileSystem().delete(inputDir, true); + inputDir = new Path(getFsTestCaseDir(), "input2"); + getFileSystem().delete(inputDir, true); + inputDir = new Path(getFsTestCaseDir(), "input3"); + getFileSystem().delete(inputDir, true); + skipConf.setProperty(WorkflowClient.RERUN_SKIP_NODES, "hdfs11,hdfs21,hdfs31,dec1"); + wfClient.reRun(jobId1, skipConf); + waitFor(5 * 1000, new Predicate() { + public boolean evaluate() throws Exception { + return wfClient.getJobInfo(jobId1).getStatus() == Workflow.Status.SUCCEEDED; + } + }); + assertEquals(wfClient.getJobInfo(jobId1).getStatus(), Workflow.Status.SUCCEEDED); + } + + public void testRedeploy() throws IOException, WorkflowClientException { + Reader reader = IOUtils.getResourceAsReader("rerun-elerr-wf.xml", -1); + Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); + IOUtils.copyCharStream(reader, writer); + + final WorkflowClient wfClient = LocalOozie.getClient(); + Properties conf = wfClient.createConfiguration(); + conf.setProperty(WorkflowClient.APP_PATH, getTestCaseDir()); + conf.setProperty("inPath", getFsTestCaseDir().toString()); + conf.setProperty("checkDir", getFsTestCaseDir().toString() + "/check"); + + final String jobId1 = wfClient.submit(conf); + wfClient.start(jobId1); + waitFor(5 * 1000, new Predicate() { + public boolean evaluate() throws Exception { + return wfClient.getJobInfo(jobId1).getStatus() == Workflow.Status.FAILED; + } + }); + assertEquals(wfClient.getJobInfo(jobId1).getStatus(), Workflow.Status.FAILED); + + reader = IOUtils.getResourceAsReader("rerun-el-wf.xml", -1); + writer = new FileWriter(getTestCaseDir() + "/workflow.xml"); + IOUtils.copyCharStream(reader, writer); + + conf.setProperty(WorkflowClient.RERUN_SKIP_NODES, "hdfs11"); + wfClient.reRun(jobId1, conf); + waitFor(5 * 1000, new Predicate() { + public boolean evaluate() throws Exception { + return wfClient.getJobInfo(jobId1).getStatus() == Workflow.Status.SUCCEEDED; + } + }); + assertEquals(wfClient.getJobInfo(jobId1).getStatus(), Workflow.Status.SUCCEEDED); + } +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/util/TestXConfiguration.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/util/TestXConfiguration.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/util/TestXConfiguration.java (revision 0) @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.util; + +import java.io.InputStream; +import java.io.Reader; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.MapFile; +import org.apache.oozie.test.XTestCase; + +public class TestXConfiguration extends XTestCase { + + public void testFromStream() throws Exception { + String configPath = "test-oozie-default.xml"; + InputStream is = IOUtils.getResourceAsStream(configPath, -1); + XConfiguration conf = new XConfiguration(is); + assertEquals("DEFAULT", conf.get("oozie.dummy")); + } + + public void testFromReader() throws Exception { + String configPath = "test-oozie-default.xml"; + Reader reader = IOUtils.getResourceAsReader(configPath, -1); + XConfiguration conf = new XConfiguration(reader); + assertEquals("DEFAULT", conf.get("oozie.dummy")); + } + + public void testCopy() throws Exception { + Configuration srcConf = new Configuration(false); + Configuration targetConf = new Configuration(false); + + srcConf.set("testParameter1", "valueFromSource"); + srcConf.set("testParameter2", "valueFromSource"); + + targetConf.set("testParameter2", "valueFromTarget"); + targetConf.set("testParameter3", "valueFromTarget"); + + XConfiguration.copy(srcConf, targetConf); + + assertEquals(targetConf.get("testParameter1"), "valueFromSource"); + assertEquals(targetConf.get("testParameter2"), "valueFromSource"); + assertEquals(targetConf.get("testParameter3"), "valueFromTarget"); + + } + + public void testInjectDefaults() throws Exception { + Configuration srcConf = new Configuration(false); + Configuration targetConf = new Configuration(false); + + srcConf.set("testParameter1", "valueFromSource"); + srcConf.set("testParameter2", "valueFromSource"); + + targetConf.set("testParameter2", "originalValueFromTarget"); + targetConf.set("testParameter3", "originalValueFromTarget"); + + XConfiguration.injectDefaults(srcConf, targetConf); + + assertEquals(targetConf.get("testParameter1"), "valueFromSource"); + assertEquals(targetConf.get("testParameter2"), "originalValueFromTarget"); + assertEquals(targetConf.get("testParameter3"), "originalValueFromTarget"); + + assertEquals(srcConf.get("testParameter1"), "valueFromSource"); + assertEquals(srcConf.get("testParameter2"), "valueFromSource"); + assertNull(srcConf.get("testParameter3")); + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/util/TestXLogFilter.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/util/TestXLogFilter.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/util/TestXLogFilter.java (revision 0) @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.util; + +import org.apache.oozie.service.ServiceException; +import org.apache.oozie.service.Services; +import org.apache.oozie.util.XLogStreamer; +import java.util.ArrayList; +import org.apache.oozie.test.XTestCase; + +public class TestXLogFilter extends XTestCase { + public void testXLogFileter() throws ServiceException { + Services services = new Services(); + services.init(); + XLogStreamer.Filter xf2 = new XLogStreamer.Filter(); + xf2.constructPattern(); + ArrayList a = new ArrayList(); + a.add("02:43:13,958 DEBUG"); + a.add(" WorkflowRunnerCallable:323 - " + XLog.Info.get().createPrefix() + " test log"); + assertEquals(true ,xf2.matches(a)); + services.destroy(); + + XLogStreamer.Filter.reset(); + XLogStreamer.Filter.defineParameter("USER"); + XLogStreamer.Filter.defineParameter("GROUP"); + XLogStreamer.Filter.defineParameter("TOKEN"); + XLogStreamer.Filter.defineParameter("APP"); + XLogStreamer.Filter.defineParameter("JOB"); + XLogStreamer.Filter.defineParameter("ACTION"); + XLogStreamer.Filter xf = new XLogStreamer.Filter(); + + assertEquals(7, matches(xf)); + xf.setLogLevel(XLog.Level.WARN.toString()); + assertEquals(2, matches(xf)); + + xf.setLogLevel(XLog.Level.WARN.toString()); + xf.setParameter("APP", "example-forkjoinwf"); + assertEquals(0, matches(xf)); + + xf.setLogLevel(XLog.Level.DEBUG.toString() + "|" + XLog.Level.INFO.toString()); + xf.setParameter("JOB", "14-200904160239--example-forkjoinwf"); + assertEquals(2, matches(xf)); + + XLogStreamer.Filter xf1 = new XLogStreamer.Filter(); + xf1.setParameter("USER", "oozie"); + assertEquals(3, matches(xf1)); + + xf1.setParameter("GROUP", "oozie"); + assertEquals(2, matches(xf1)); + + xf1.setParameter("TOKEN", "MYtoken"); + assertEquals(1, matches(xf1)); + } + + private int matches(XLogStreamer.Filter xf) { + xf.constructPattern(); + ArrayList a = new ArrayList(); + a.add("02:43:13,958 DEBUG WorkflowRunnerCallable:323 - USER[oozie] GROUP[-] TOKEN[-] APP[example-forkjoinwf] JOB[14-200904160239--example-forkjoinwf] ACTION[-] End workflow state change"); + a.add("02:43:13,961 INFO WorkflowRunnerCallable:317 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] JOB[14-200904160239--example-forkjoinwf] ACTION[-] [org.apache.oozie.core.command.WorkflowRunnerCallable] released lock"); + a.add("02:43:13,986 WARN JobClient:539 - Use GenericOptionsParser for parsing the arguments. Applications should implement Tool for the same."); + a.add("02:43:14,431 WARN JobClient:661 - No job jar file set. User classes may not be found. See JobConf(Class) or JobConf#setJar(String)."); + a.add("02:43:14,505 INFO ActionExecutorCallable:317 - USER[oozie] GROUP[oozie] TOKEN[-] APP[-] JOB[-] ACTION[-] Released Lock"); + a.add("02:43:19,344 DEBUG PendingSignalsCallable:323 - USER[oozie] GROUP[oozie] TOKEN[MYtoken] APP[-] JOB[-] ACTION[-] Number of pending signals to check [0]"); + a.add("02:43:29,151 DEBUG PendingActionsCallable:323 - USER[-] GROUP[-] TOKEN[-] APP[-] JOB[-] ACTION[-] Number of pending actions [0] "); + int matchCnt = 0; + for (int i = 0; i < a.size(); i++) { + if (xf.matches(xf.splitLogMessage(a.get(i)))) { + matchCnt++; + } + } + return matchCnt; + } +} Index: oozie/core/src/test/java/org/apache/oozie/util/TestELEvaluator.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/util/TestELEvaluator.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/util/TestELEvaluator.java (revision 0) @@ -0,0 +1,170 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.util; + +import org.apache.oozie.test.XTestCase; + +import javax.servlet.jsp.el.ELException; +import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.Map; + +public class TestELEvaluator extends XTestCase { + + public static String functionA() { + assertEquals("A", ELEvaluator.getCurrent().getVariable("a")); + return "a"; + } + + public String functionB() { + return "b"; + } + + private static String functionC() { + return "c"; + } + + public static String functionError() throws ELEvaluationException { + throw new ELEvaluationException("m", null); + } + + private static Method functionA; + private static Method functionB; + private static Method functionC; + private static Method functionError; + + static { + try { + functionA = TestELEvaluator.class.getMethod("functionA"); + functionB = TestELEvaluator.class.getMethod("functionB"); + functionC = TestELEvaluator.class.getDeclaredMethod("functionC"); + functionError = TestELEvaluator.class.getDeclaredMethod("functionError"); + } + catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + public void testContextVars() throws Exception { + ELEvaluator.Context support = new ELEvaluator.Context(); + assertNull(support.getVariable("a")); + support.setVariable("a", "A"); + assertEquals("A", support.getVariable("a")); + Map vars = new HashMap(); + vars.put("a", "AA"); + vars.put("b", "BB"); + support.setVariables(vars); + assertEquals("AA", support.getVariable("a")); + assertEquals("BB", support.getVariable("b")); + try { + support.resolveVariable("c"); + fail(); + } + catch (ELException ex) { + //nop + } + } + + + public void testContextFunctions() throws Exception { + ELEvaluator.Context support = new ELEvaluator.Context(); + support.addFunction("a", "a", functionA); + + try { + support.addFunction("b", "b", functionB); + fail(); + } + catch (IllegalArgumentException ex) { + //nop + } + + try { + support.addFunction("c", "c", functionC); + fail(); + } + catch (IllegalArgumentException ex) { + //nop + } + + assertEquals(functionA, support.resolveFunction("a", "a")); + } + + public void testVars() throws Exception { + ELEvaluator.Context support = new ELEvaluator.Context(); + support.setVariable("a", "A"); + ELEvaluator evaluator = new ELEvaluator(support); + assertEquals("A", evaluator.getVariable("a")); + assertEquals("A", evaluator.getContext().getVariable("a")); + + Map vars = new HashMap(); + vars.put("a", "AA"); + vars.put("b", "BB"); + support.setVariables(vars); + assertEquals("AA", support.getVariable("a")); + assertEquals("BB", support.getVariable("b")); + try { + support.resolveVariable("c"); + fail(); + } + catch (ELException ex) { + //nop + } + } + + public void testFunctions() throws Exception { + ELEvaluator.Context support = new ELEvaluator.Context(); + support.addFunction("a", "a", functionA); + ELEvaluator evaluator = new ELEvaluator(support); + assertEquals(functionA, evaluator.getContext().resolveFunction("a", "a")); + } + + public void testEval() throws Exception { + ELEvaluator.Context support = new ELEvaluator.Context(); + support.setVariable("a", "A"); + support.addFunction("a", "a", functionA); + ELEvaluator evaluator = new ELEvaluator(support); + assertEquals("Aa", evaluator.evaluate("${a}${a:a()}", String.class)); + } + + public void testCurrent() throws Exception { + ELEvaluator.Context support = new ELEvaluator.Context(); + support.setVariable("a", "A"); + support.addFunction("a", "a", functionA); + ELEvaluator evaluator = new ELEvaluator(support); + assertNull(ELEvaluator.getCurrent()); + assertEquals("a", evaluator.evaluate("${a:a()}", String.class)); + assertNull(ELEvaluator.getCurrent()); + } + + public void testFunctionELEvaluationError() throws Exception { + try { + ELEvaluator.Context support = new ELEvaluator.Context(); + support.addFunction("a", "a", functionError); + ELEvaluator evaluator = new ELEvaluator(support); + evaluator.evaluate("${a:a()}", String.class); + fail(); + } + catch (ELEvaluationException ex) { + //nop + } + catch (ELException ex) { + fail(); + } + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/util/TestInstrumentation.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/util/TestInstrumentation.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/util/TestInstrumentation.java (revision 0) @@ -0,0 +1,289 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.util; + +import org.apache.oozie.test.XTestCase; + +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ScheduledExecutorService; + +public class TestInstrumentation extends XTestCase { + private static final long INTERVAL = 300; + private static final long TOLERANCE = 30; + + public void testCron() throws Exception { + Instrumentation.Cron cron = new Instrumentation.Cron(); + long start = System.currentTimeMillis(); + assertEquals("", 0, cron.getStart(), TOLERANCE); + assertEquals("", 0, cron.getEnd(), TOLERANCE); + assertEquals(cron.getStart(), cron.getEnd()); + assertEquals(0, cron.getOwn()); + assertEquals(0, cron.getTotal()); + + cron.start(); + Thread.sleep(INTERVAL); + cron.stop(); + long now = System.currentTimeMillis(); + assertEquals("", start, cron.getStart(), TOLERANCE); + assertEquals("", now, cron.getEnd(), TOLERANCE); + assertEquals("", INTERVAL, cron.getTotal(), TOLERANCE); + assertEquals("", INTERVAL, cron.getOwn(), TOLERANCE); + assertEquals("", cron.getTotal(), cron.getOwn(), TOLERANCE); + + Thread.sleep(INTERVAL); + + cron.start(); + Thread.sleep(INTERVAL); + cron.stop(); + now = System.currentTimeMillis(); + assertEquals("", start, cron.getStart(), TOLERANCE); + assertEquals("", now, cron.getEnd(), TOLERANCE); + assertEquals("", INTERVAL * 3, cron.getTotal(), TOLERANCE); + assertEquals("", INTERVAL * 2, cron.getOwn(), TOLERANCE); + } + + public void testTimer() throws Exception { + Instrumentation.Timer timer = new Instrumentation.Timer(); + + assertEquals(0, timer.getTicks()); + assertEquals(0, timer.getTotal()); + assertEquals(0, timer.getOwn()); + assertEquals(0, timer.getOwnAvg()); + assertEquals(0, timer.getTotalAvg()); + assertEquals(0, timer.getOwnSquareSum()); + assertEquals(0, timer.getTotalSquareSum()); + assertEquals(0, timer.getOwnMin()); + assertEquals(0, timer.getOwnMax()); + assertEquals(0, timer.getTotalMin()); + assertEquals(0, timer.getTotalMax()); + + assertEquals(0, timer.getValue().getTicks()); + assertEquals(0, timer.getValue().getTotal()); + assertEquals(0, timer.getValue().getOwn()); + assertEquals(0, timer.getValue().getOwnAvg()); + assertEquals(0, timer.getValue().getTotalAvg()); + assertEquals(0, timer.getValue().getOwnSquareSum()); + assertEquals(0, timer.getValue().getTotalSquareSum()); + assertEquals(0, timer.getValue().getOwnMin()); + assertEquals(0, timer.getValue().getOwnMax()); + assertEquals(0, timer.getValue().getTotalMin()); + assertEquals(0, timer.getValue().getTotalMax()); + + Instrumentation.Cron cron1 = new Instrumentation.Cron(); + cron1.start(); + Thread.sleep(INTERVAL); + cron1.stop(); + timer.addCron(cron1); + + assertEquals(1, timer.getTicks()); + assertEquals(cron1.getTotal(), timer.getTotal()); + assertEquals(cron1.getOwn(), timer.getOwn()); + assertEquals(cron1.getOwn(), timer.getOwnAvg()); + assertEquals(cron1.getTotal(), timer.getTotalAvg()); + assertEquals(cron1.getOwn() * cron1.getOwn(), timer.getOwnSquareSum()); + assertEquals(cron1.getTotal() * cron1.getTotal(), timer.getTotalSquareSum()); + assertEquals(cron1.getOwn(), timer.getOwnMin()); + assertEquals(cron1.getOwn(), timer.getOwnMax()); + assertEquals(cron1.getTotal(), timer.getTotalMin()); + assertEquals(cron1.getTotal(), timer.getTotalMax()); + + assertEquals(1, timer.getValue().getTicks()); + assertEquals(cron1.getTotal(), timer.getValue().getTotal()); + assertEquals(cron1.getOwn(), timer.getValue().getOwn()); + assertEquals(cron1.getOwn(), timer.getValue().getOwnAvg()); + assertEquals(cron1.getTotal(), timer.getValue().getTotalAvg()); + assertEquals(cron1.getOwn() * cron1.getOwn(), timer.getValue().getOwnSquareSum()); + assertEquals(cron1.getTotal() * cron1.getTotal(), timer.getValue().getTotalSquareSum()); + assertEquals(cron1.getOwn(), timer.getValue().getOwnMin()); + assertEquals(cron1.getOwn(), timer.getValue().getOwnMax()); + assertEquals(cron1.getTotal(), timer.getValue().getTotalMin()); + assertEquals(cron1.getTotal(), timer.getValue().getTotalMax()); + + Instrumentation.Cron cron2 = new Instrumentation.Cron(); + cron2.start(); + Thread.sleep(INTERVAL * 2); + cron2.stop(); + timer.addCron(cron2); + + assertEquals(2, timer.getTicks()); + assertEquals(cron1.getTotal() + cron2.getTotal(), timer.getTotal()); + assertEquals(cron1.getOwn() + cron2.getOwn(), timer.getOwn()); + assertEquals((cron1.getOwn() + cron2.getOwn()) / 2, timer.getOwnAvg()); + assertEquals((cron1.getTotal() + cron2.getTotal()) / 2, timer.getTotalAvg()); + assertEquals(cron1.getOwn() * cron1.getOwn() + cron2.getOwn() * cron2.getOwn(), + timer.getOwnSquareSum()); + assertEquals(cron1.getTotal() * cron1.getTotal() + cron2.getTotal() * cron2.getTotal(), + timer.getTotalSquareSum()); + assertEquals(cron1.getOwn(), timer.getOwnMin()); + assertEquals(cron2.getOwn(), timer.getOwnMax()); + assertEquals(cron1.getTotal(), timer.getTotalMin()); + assertEquals(cron2.getTotal(), timer.getTotalMax()); + } + + public void testInstrumentationCounter() throws Exception { + Instrumentation inst = new Instrumentation(); + assertEquals(0, inst.getCounters().size()); + inst.incr("a", "1", 1); + assertEquals(1, inst.getCounters().size()); + assertEquals(1, inst.getCounters().get("a").size()); + inst.incr("a", "2", 2); + assertEquals(1, inst.getCounters().size()); + assertEquals(2, inst.getCounters().get("a").size()); + inst.incr("b", "1", 3); + assertEquals(2, inst.getCounters().size()); + assertEquals(2, inst.getCounters().get("a").size()); + assertEquals(1, inst.getCounters().get("b").size()); + assertEquals(new Long(1), inst.getCounters().get("a").get("1").getValue()); + assertEquals(new Long(2), inst.getCounters().get("a").get("2").getValue()); + assertEquals(new Long(3), inst.getCounters().get("b").get("1").getValue()); + } + + public void testInstrumentationTimer() throws Exception { + Instrumentation inst = new Instrumentation(); + assertEquals(0, inst.getTimers().size()); + Instrumentation.Cron cron1 = new Instrumentation.Cron(); + inst.addCron("a", "1", cron1); + assertEquals(1, inst.getTimers().size()); + assertEquals(1, inst.getTimers().get("a").size()); + Instrumentation.Cron cron2 = new Instrumentation.Cron(); + cron2.start(); + Thread.sleep(INTERVAL); + cron2.stop(); + inst.addCron("a", "2", cron2); + assertEquals(1, inst.getTimers().size()); + assertEquals(2, inst.getTimers().get("a").size()); + Instrumentation.Cron cron3 = new Instrumentation.Cron(); + cron3.start(); + Thread.sleep(INTERVAL * 2); + cron3.stop(); + inst.addCron("b", "1", cron3); + assertEquals(2, inst.getTimers().size()); + assertEquals(2, inst.getTimers().get("a").size()); + assertEquals(1, inst.getTimers().get("b").size()); + + assertEquals(cron1.getOwn(), inst.getTimers().get("a").get("1").getValue().getOwn()); + assertEquals(cron2.getOwn(), inst.getTimers().get("a").get("2").getValue().getOwn()); + assertEquals(cron3.getOwn(), inst.getTimers().get("b").get("1").getValue().getOwn()); + } + + public void testVariables() throws Exception { + Instrumentation inst = new Instrumentation(); + + inst.addVariable("a", "1", new Instrumentation.Variable() { + private long counter = 0; + public Long getValue() { + return counter++; + } + }); + assertEquals(1, inst.getVariables().size()); + assertEquals(1, inst.getVariables().get("a").size()); + + inst.addVariable("a", "2", new Instrumentation.Variable() { + private long counter = 1; + public Long getValue() { + return counter++; + } + }); + assertEquals(1, inst.getVariables().size()); + assertEquals(2, inst.getVariables().get("a").size()); + inst.addVariable("b", "1", new Instrumentation.Variable() { + private long counter = 2; + public Long getValue() { + return counter++; + } + }); + assertEquals(2, inst.getVariables().size()); + assertEquals(2, inst.getVariables().get("a").size()); + assertEquals(1, inst.getVariables().get("b").size()); + + assertEquals(new Long(0), ((Instrumentation.Variable)inst.getVariables().get("a").get("1")).getValue()); + assertEquals(new Long(1), ((Instrumentation.Variable)inst.getVariables().get("a").get("2")).getValue()); + assertEquals(new Long(2), ((Instrumentation.Variable)inst.getVariables().get("b").get("1")).getValue()); + assertEquals(new Long(1), ((Instrumentation.Variable)inst.getVariables().get("a").get("1")).getValue()); + assertEquals(new Long(2), ((Instrumentation.Variable)inst.getVariables().get("a").get("2")).getValue()); + assertEquals(new Long(3), ((Instrumentation.Variable)inst.getVariables().get("b").get("1")).getValue()); + } + + public void testSamplers() throws Exception { + Instrumentation inst = new Instrumentation(); + ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1); + inst.setScheduler(scheduledExecutorService); + + inst.addSampler("a", "1", 10, 1, new Instrumentation.Variable() { + public Long getValue() { + return 1L; + } + }); + assertEquals(1, inst.getSamplers().size()); + assertEquals(1, inst.getSamplers().get("a").size()); + + inst.addSampler("a", "2", 10, 1, new Instrumentation.Variable() { + public Long getValue() { + return 2L; + } + }); + assertEquals(1, inst.getSamplers().size()); + assertEquals(2, inst.getSamplers().get("a").size()); + + inst.addSampler("b", "1", 10, 1, new Instrumentation.Variable() { + private long counter = 0; + public Long getValue() { + return counter++ % 10; + } + }); + assertEquals(2, inst.getSamplers().size()); + assertEquals(2, inst.getSamplers().get("a").size()); + assertEquals(1, inst.getSamplers().get("b").size()); + + waitFor(20 * 1000, new Predicate() { + public boolean evaluate() throws Exception { + return false; + } + }); + + assertEquals("", 1D, inst.getSamplers().get("a").get("1").getValue(), 0.01D); + assertEquals("", 2D, inst.getSamplers().get("a").get("2").getValue(), 0.02D); + assertEquals("", 5D, inst.getSamplers().get("b").get("1").getValue(), 0.5D); + + scheduledExecutorService.shutdownNow(); + } + + public void testAll() throws Exception { + Instrumentation inst = new Instrumentation(); + inst.addVariable("a", "1", new Instrumentation.Variable() { + private long counter = 0; + public Long getValue() { + return counter++; + } + }); + inst.incr("a", "1", 1); + Instrumentation.Cron cron1 = new Instrumentation.Cron(); + inst.addCron("a", "1", cron1); + + assertEquals(4, inst.getAll().size()); + assertEquals(1, inst.getAll().get("variables").size()); + assertEquals(1, inst.getAll().get("counters").size()); + assertEquals(1, inst.getAll().get("timers").size()); + assertEquals(0, inst.getAll().get("samplers").size()); + assertEquals(new Long(0), ((Instrumentation.Element)inst.getAll().get("variables").get("a").get("1")).getValue()); + assertEquals(new Long(1), ((Instrumentation.Element)inst.getAll().get("counters").get("a").get("1")).getValue()); + assertEquals(cron1.getOwn(), ((Instrumentation.Timer)((Instrumentation.Element)inst.getAll(). + get("timers").get("a").get("1")).getValue()).getOwn()); + } + +} \ No newline at end of file Index: oozie/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java (revision 0) @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.util; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.StringWriter; +import java.util.Date; +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.XLogStreamer; + + +public class TestLogStreamer extends XTestCase { + public void testStreamLog() throws IOException { + long currTime = System.currentTimeMillis(); + XLogStreamer.Filter.reset(); + XLogStreamer.Filter.defineParameter("USER"); + XLogStreamer.Filter.defineParameter("GROUP"); + XLogStreamer.Filter.defineParameter("TOKEN"); + XLogStreamer.Filter.defineParameter("APP"); + XLogStreamer.Filter.defineParameter("JOB"); + XLogStreamer.Filter.defineParameter("ACTION"); + XLogStreamer.Filter xf = new XLogStreamer.Filter(); + xf.setParameter("JOB", "14-200904160239--example-forkjoinwf"); + xf.setLogLevel("DEBUG|INFO"); + + FileWriter fw1 = new FileWriter(getTestCaseDir() + "/test.log"); + StringBuilder sb1 = new StringBuilder(); + sb1.append("02:43:13,958 DEBUG _L1_:323 - USER[oozie] GROUP[-] TOKEN[-] APP[example-forkjoinwf] JOB[14-200904160239--example-forkjoinwf] ACTION[-] End workflow state change"); + sb1.append("\n02:43:13,961 INFO _L2_:317 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] JOB[14-200904160239--example-forkjoinwf] ACTION[-] [org.apache.oozie.core.command.WorkflowRunnerCallable] released lock"); + fw1.write(sb1.toString()); + fw1.close(); + File f1 = new File(getTestCaseDir() + "/test.log"); + f1.setLastModified(currTime - 9000); + + FileWriter fw2 = new FileWriter(getTestCaseDir() + "/test.log.1"); + StringBuilder sb2 = new StringBuilder(); + sb2.append("\n02:43:13,986 WARN _L3_:539 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] JOB[14-200904160239--example-forkjoinwf] ACTION[-] Use GenericOptionsParser for parsing the arguments. \n_L3A_Applications should implement Tool for the same. \n_L3B_Multi line test"); + sb2.append("\n02:43:14,431 INFO _L4_:661 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] JOB[14-200904160239--example-forkjoinwf] ACTION[-] No job jar file set. User classes may not be found. See JobConf(Class) or JobConf#setJar(String)."); + fw2.write(sb2.toString()); + fw2.close(); + File f2 = new File(getTestCaseDir() + "/test.log.1"); + f2.setLastModified(currTime - 8000); + + FileWriter fw3 = new FileWriter(getTestCaseDir() + "/test.log.2"); + StringBuilder sb3 = new StringBuilder(); + sb3.append("\n02:43:14,505 INFO _L5_:317 - USER[oozie] GROUP[oozie] TOKEN[-] APP[-] JOB[-] ACTION[-] Released Lock"); + sb3.append("\n02:43:19,344 DEBUG _L6_:323 - USER[oozie] GROUP[oozie] TOKEN[MYtoken] APP[-] JOB[-] ACTION[-] Number of pending signals to check [0]"); + sb3.append("\n02:43:29,151 DEBUG _L7_:323 - USER[-] GROUP[-] TOKEN[-] APP[-] JOB[14-200904160239--example-forkjoinwf] ACTION[-] Number of pending actions [0] "); + fw3.write(sb3.toString()); + fw3.close(); + File f3 = new File(getTestCaseDir() + "/test.log.2"); + f3.setLastModified(currTime); + + FileWriter fwerr = new FileWriter(getTestCaseDir() + "/testerr.log"); + StringBuilder sberr = new StringBuilder(); + sberr.append("02:43:13,958 WARN _L1_:323 - USER[oozie] GROUP[-] TOKEN[-] APP[example-forkjoinwf] JOB[14-200904160239--example-forkjoinwf] ACTION[-] End workflow state change"); + sberr.append("\n02:43:13,961 INFO _L2_:317 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] JOB[14-200904160239--example-forkjoinwf] ACTION[-] [org.apache.oozie.core.command.WorkflowRunnerCallable] released lock"); + fwerr.write(sberr.toString()); + fwerr.close(); + File ferr = new File(getTestCaseDir() + "/testerr.log"); + ferr.setLastModified(currTime - 8000); + + StringWriter sw = new StringWriter(); + XLogStreamer str = new XLogStreamer(xf, sw, getTestCaseDir(), "test.log", 1); + str.streamLog(new Date(currTime - 10000), new Date(currTime - 5000)); + String[] out = sw.toString().split("\n"); + assertEquals(3, out.length); + assertEquals(true, out[0].contains("_L1_")); + assertEquals(true, out[1].contains("_L2_")); + assertEquals(true, out[2].contains("_L4_")); + + StringWriter sw1 = new StringWriter(); + XLogStreamer str1 = new XLogStreamer(xf, sw1, getTestCaseDir(), "test.log", 1); + str1.streamLog(null, null); + out = sw1.toString().split("\n"); + assertEquals(4, out.length); + assertEquals(true, out[0].contains("_L1_")); + assertEquals(true, out[1].contains("_L2_")); + assertEquals(true, out[2].contains("_L4_")); + assertEquals(true, out[3].contains("_L7_")); + } +} Index: oozie/core/src/test/java/org/apache/oozie/util/TestMemoryLocks.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/util/TestMemoryLocks.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/util/TestMemoryLocks.java (revision 0) @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.util; + +import org.apache.oozie.test.XTestCase; + +public class TestMemoryLocks extends XTestCase { + private XLog log = XLog.getLog(getClass()); + + private MemoryLocks locks; + + public void setUp() throws Exception { + super.setUp(); + locks = new MemoryLocks(); + } + + public void tearDown() { + locks = null; + } + + public abstract class Locker implements Runnable { + protected String name; + private String nameIndex; + private StringBuffer sb; + protected long timeout; + + + public Locker(String name, int nameIndex, long timeout, StringBuffer buffer) { + this.name = name; + this.nameIndex = name + ":" + nameIndex; + this.sb = buffer; + this.timeout = timeout; + } + + public void run() { + try { + log.info("Getting lock [{0}]", nameIndex); + MemoryLocks.LockToken token = getLock(); + if (token != null) { + log.info("Got lock [{0}]", nameIndex); + sb.append(nameIndex + "-L "); + synchronized (this) { + wait(); + } + sb.append(nameIndex + "-U "); + token.release(); + log.info("Release lock [{0}]", nameIndex); + } + else { + sb.append(nameIndex + "-N "); + log.info("Did not get lock [{0}]", nameIndex); + } + } + catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + public void finish() { + synchronized (this) { + notify(); + } + } + + protected abstract MemoryLocks.LockToken getLock() throws InterruptedException; + + + } + + public class ReadLocker extends Locker { + + public ReadLocker(String name, int nameIndex, long timeout, StringBuffer buffer) { + super(name, nameIndex, timeout, buffer); + } + + protected MemoryLocks.LockToken getLock() throws InterruptedException { + return locks.getReadLock(name, timeout); + } + } + + public class WriteLocker extends Locker { + + public WriteLocker(String name, int nameIndex, long timeout, StringBuffer buffer) { + super(name, nameIndex, timeout, buffer); + } + + protected MemoryLocks.LockToken getLock() throws InterruptedException { + return locks.getWriteLock(name, timeout); + } + } + + public void testWaitWriteLock() throws Exception { + StringBuffer sb = new StringBuffer(""); + Locker l1 = new WriteLocker("a", 1, -1, sb); + Locker l2 = new WriteLocker("a", 2, -1, sb); + + new Thread(l1).start(); + Thread.sleep(500); + new Thread(l2).start(); + Thread.sleep(500); + l1.finish(); + Thread.sleep(500); + l2.finish(); + Thread.sleep(500); + assertEquals("a:1-L a:1-U a:2-L a:2-U", sb.toString().trim()); + } + + public void testNoWaitWriteLock() throws Exception { + StringBuffer sb = new StringBuffer(""); + Locker l1 = new WriteLocker("a", 1, 0, sb); + Locker l2 = new WriteLocker("a", 2, 0, sb); + + new Thread(l1).start(); + Thread.sleep(500); + new Thread(l2).start(); + Thread.sleep(500); + l1.finish(); + Thread.sleep(500); + l2.finish(); + Thread.sleep(500); + assertEquals("a:1-L a:2-N a:1-U", sb.toString().trim()); + } + + public void testTimeoutWaitingWriteLock() throws Exception { + StringBuffer sb = new StringBuffer(""); + Locker l1 = new WriteLocker("a", 1, 0, sb); + Locker l2 = new WriteLocker("a", 2, 1000, sb); + + new Thread(l1).start(); + Thread.sleep(500); + new Thread(l2).start(); + Thread.sleep(500); + l1.finish(); + Thread.sleep(500); + l2.finish(); + Thread.sleep(500); + assertEquals("a:1-L a:1-U a:2-L a:2-U", sb.toString().trim()); + } + + public void testTimeoutTimingOutWriteLock() throws Exception { + StringBuffer sb = new StringBuffer(""); + Locker l1 = new WriteLocker("a", 1, 0, sb); + Locker l2 = new WriteLocker("a", 2, 50, sb); + + new Thread(l1).start(); + Thread.sleep(500); + new Thread(l2).start(); + Thread.sleep(500); + l1.finish(); + Thread.sleep(500); + l2.finish(); + Thread.sleep(500); + assertEquals("a:1-L a:2-N a:1-U", sb.toString().trim()); + } + + public void testReadLock() throws Exception { + StringBuffer sb = new StringBuffer(""); + Locker l1 = new ReadLocker("a", 1, -1, sb); + Locker l2 = new ReadLocker("a", 2, -1, sb); + + new Thread(l1).start(); + Thread.sleep(500); + new Thread(l2).start(); + Thread.sleep(500); + l1.finish(); + Thread.sleep(500); + l2.finish(); + Thread.sleep(500); + assertEquals("a:1-L a:2-L a:1-U a:2-U", sb.toString().trim()); + } + + public void testReadWriteLock() throws Exception { + StringBuffer sb = new StringBuffer(""); + Locker l1 = new ReadLocker("a", 1, -1, sb); + Locker l2 = new WriteLocker("a", 2, -1, sb); + + new Thread(l1).start(); + Thread.sleep(500); + new Thread(l2).start(); + Thread.sleep(500); + l1.finish(); + Thread.sleep(500); + l2.finish(); + Thread.sleep(500); + assertEquals("a:1-L a:1-U a:2-L a:2-U", sb.toString().trim()); + } + + public void testWriteReadLock() throws Exception { + StringBuffer sb = new StringBuffer(""); + Locker l1 = new WriteLocker("a", 1, -1, sb); + Locker l2 = new ReadLocker("a", 2, -1, sb); + + new Thread(l1).start(); + Thread.sleep(500); + new Thread(l2).start(); + Thread.sleep(500); + l1.finish(); + Thread.sleep(500); + l2.finish(); + Thread.sleep(500); + assertEquals("a:1-L a:1-U a:2-L a:2-U", sb.toString().trim()); + } + +} Index: oozie/core/src/test/java/org/apache/oozie/util/db/TestSchema.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/util/db/TestSchema.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/util/db/TestSchema.java (revision 0) @@ -0,0 +1,204 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.util.db; + +import java.sql.Blob; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.db.Schema.Column; +import org.apache.oozie.util.db.Schema.DBType; +import org.apache.oozie.util.db.Schema.Index; +import org.apache.oozie.util.db.Schema.Table; + +public class TestSchema extends XTestCase { + + public static Map> TABLE_COLUMNS = new HashMap>(); + private static final String DB_NAME = "testdb"; + static { + for (Column column : TestColumns.values()) { + List tColumns = TABLE_COLUMNS.get(column.table()); + if (tColumns == null) { + tColumns = new ArrayList(); + TABLE_COLUMNS.put(column.table(), tColumns); + } + tColumns.add(column); + } + } + + public static enum TestTable implements Table { + TEST_TABLE; + public String toString() { + return DB_NAME + "." + name(); + } + } + + public static enum TestColumns implements Column { + TEST_LONG(TestTable.TEST_TABLE, Long.class, true), + TEST_STRING(TestTable.TEST_TABLE, String.class, false, 100), + TEST_TIMESTAMP(TestTable.TEST_TABLE, Timestamp.class, false), + TEST_BOOLEAN(TestTable.TEST_TABLE, Boolean.class, false), + TEST_BLOB(TestTable.TEST_TABLE, Blob.class, false); + + final Table table; + final Class type; + int length = -1; + final boolean isPrimaryKey; + + TestColumns(Table table, Class type, boolean isPrimaryKey) { + this(table, type, isPrimaryKey, -1); + } + + TestColumns(Table table, Class type, boolean isPrimaryKey, int length) { + this.table = table; + this.type = type; + this.isPrimaryKey = isPrimaryKey; + this.length = length; + } + + @Override + public String asLabel() { + return columnName(); + } + + @Override + public String columnName() { + return name(); + } + + @Override + public int getLength() { + return length; + } + + @Override + public Class getType() { + return type; + } + + @Override + public boolean isPrimaryKey() { + return isPrimaryKey; + } + + @Override + public Table table() { + return table; + } + } + + public static enum TestIndex implements Index { + TEST_INDEX(TestColumns.TEST_LONG); + + private Column idxCol; + + private TestIndex(Column idxCol) { + this.idxCol = idxCol; + } + + @Override + public Column column() { + return idxCol; + } + } + + public static void prepareDB(Connection conn) throws SQLException { + DBType type = DBType.MySQL; + if(Schema.isHsqlConnection(conn)) { + type = DBType.HSQL; + } + conn.prepareStatement("CREATE " + (type.equals(DBType.MySQL) ? "DATABASE " : "SCHEMA ") + DB_NAME + + (type.equals(DBType.HSQL) ? " AUTHORIZATION DBA" : "")).execute(); + for (Table table : TABLE_COLUMNS.keySet()) { + String createStmt = Schema.generateCreateTableScript(table, type, TABLE_COLUMNS.get(table)); + conn.prepareStatement(createStmt).execute(); + } + } + + public static void dropSchema(Connection conn) { + try { + DBType type = DBType.MySQL; + if (Schema.isHsqlConnection(conn)) { + type = DBType.HSQL; + } + conn.prepareStatement( + "DROP " + (type.equals(DBType.MySQL) ? "DATABASE " : "SCHEMA ") + DB_NAME + + (type.equals(DBType.HSQL) ? " CASCADE" : "")).execute(); + } + catch (SQLException e) { + } + } + + public void testGenerateCreateScript() throws SQLException { + Connection conn = getDirectConnection(); + prepareDB(conn); + ResultSet rs = conn.prepareStatement("SELECT COUNT(*) FROM " + TestTable.TEST_TABLE).executeQuery(); + rs.next(); + assertEquals(0, rs.getInt(1)); + rs.close(); + conn.prepareStatement( + "INSERT INTO " + TestTable.TEST_TABLE + "(" + TestColumns.TEST_LONG + ", " + TestColumns.TEST_STRING + + ")" + " VALUES(1, 'abcd')").executeUpdate(); + rs = conn.prepareStatement("SELECT COUNT(*) FROM " + TestTable.TEST_TABLE).executeQuery(); + rs.next(); + assertEquals(1, rs.getInt(1)); + rs.close(); + boolean pkeyTest = false; + try { + conn.prepareStatement( + "INSERT INTO " + TestTable.TEST_TABLE + "(" + TestColumns.TEST_LONG + ", " + + TestColumns.TEST_STRING + ")" + " VALUES(1, 'abcd')").executeUpdate(); + } + catch (SQLException e) { + pkeyTest = true; + } + assertEquals(true, pkeyTest); + String indexStmt = Schema.generateCreateIndexScript(TestIndex.TEST_INDEX, DBType.HSQL); + conn.prepareStatement(indexStmt).execute();// Will throw an exception if + // index cant be created + conn.prepareStatement("DROP TABLE " + TestTable.TEST_TABLE).execute(); + dropSchema(conn); + conn.close(); + } + + public static Connection getDirectConnection() throws SQLException { + String driver = "org.hsqldb.jdbcDriver"; + String url = "jdbc:hsqldb:mem:testdb"; + String user = "sa"; + String password = ""; + try { + Class.forName(driver); + } + catch (ClassNotFoundException ex) { + throw new RuntimeException(ex); + } + Connection conn = DriverManager.getConnection(url, user, password); + conn.setAutoCommit(true); + conn.setTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); + return conn; + } +} Index: oozie/core/src/test/java/org/apache/oozie/util/db/TestSqlStatement.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/util/db/TestSqlStatement.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/util/db/TestSqlStatement.java (revision 0) @@ -0,0 +1,248 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.util.db; + +import static org.apache.oozie.util.db.SqlStatement.*; +import static org.apache.oozie.util.db.TestSchema.TestColumns.*; +import static org.apache.oozie.util.db.TestSchema.TestTable.*; +import org.apache.oozie.service.DataSourceService; +import org.apache.oozie.service.Services; +import java.util.List; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.Calendar; +import java.sql.PreparedStatement; +import java.sql.Timestamp; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.db.Schema.Table; + +public class TestSqlStatement extends XTestCase { + + private Connection conn; + private final String[] names = { "a", "b", "c", "d", "e" }; + private Timestamp currTime; + + @Override + protected void setUp() throws Exception { + super.setUp(); + Services services = new Services(); + services.init(); + DataSourceService dataSourceServ = Services.get().get(DataSourceService.class); + conn = dataSourceServ.getRawConnection(); + TestSchema.prepareDB(conn); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + TestSchema.dropSchema(conn); + conn.close(); + } + + public void testSQLStatements() throws SQLException { + _testInsertAndGetCountAndprepare(); + _testParser(); + _testSelect(); + _testUpdate(); + _testDelete(); + } + + private void _testDelete() throws SQLException { + ResultSet rs = getCount(TEST_TABLE).where(isEqual(TEST_LONG, 0)).prepareAndSetValues(conn).executeQuery(); + rs.next(); + assertEquals(1, rs.getInt(1)); + deleteFrom(TEST_TABLE).where(isEqual(TEST_LONG, 0)).prepareAndSetValues(conn).executeUpdate(); + rs = getCount(TEST_TABLE).where(isEqual(TEST_LONG, 0)).prepareAndSetValues(conn).executeQuery(); + rs.next(); + assertEquals(0, rs.getInt(1)); + } + + private void _testUpdate() throws SQLException { + update(TEST_TABLE).set(TEST_STRING, "test").where(isEqual(TEST_LONG, 0)).prepareAndSetValues(conn) + .executeUpdate(); + ResultSetReader rsReader = parse(selectColumns(TEST_STRING, TEST_LONG).where(isEqual(TEST_LONG, 0)) + .prepareAndSetValues(conn).executeQuery()); + rsReader.next(); + assertEquals("test", rsReader.getString(TEST_STRING)); + rsReader.close(); + + update(TEST_TABLE).set(TEST_STRING, "a").where(isEqual(TEST_LONG, 0)).prepareAndSetValues(conn).executeUpdate(); + rsReader = parse(selectColumns(TEST_STRING, TEST_LONG).where(isEqual(TEST_LONG, 0)).prepareAndSetValues(conn) + .executeQuery()); + rsReader.next(); + assertEquals("a", rsReader.getString(TEST_STRING)); + rsReader.close(); + } + + private void _testSelect() throws SQLException { + ResultSetReader rsReader = parse(selectAllFrom(TEST_TABLE).orderBy(TEST_LONG, true).prepareAndSetValues(conn) + .executeQuery()); + assertEquals(5, checkIdAndName(rsReader)); + + rsReader = parse(selectAllFrom(TEST_TABLE).orderBy(TEST_LONG, true).limit(0, 3).prepareAndSetValues(conn) + .executeQuery()); + assertEquals(3, checkIdAndName(rsReader)); + + rsReader = parse(selectColumns(TEST_STRING, TEST_LONG).orderBy(TEST_LONG, true).prepareAndSetValues(conn) + .executeQuery()); + assertEquals(5, checkIdAndName(rsReader)); + + rsReader = parse(selectColumns(TEST_STRING, TEST_LONG).where(isLike(TEST_STRING, names[0])).orderBy(TEST_LONG, + true).prepareAndSetValues(conn).executeQuery()); + assertEquals(1, checkIdAndName(rsReader)); + + rsReader = parse(selectColumns(TEST_STRING, TEST_LONG).where(isNotLike(TEST_STRING, names[4])).orderBy( + TEST_LONG, true).prepareAndSetValues(conn).executeQuery()); + assertEquals(4, checkIdAndName(rsReader)); + + rsReader = parse(selectColumns(TEST_STRING, TEST_LONG).where(isEqual(TEST_LONG, 0)).orderBy(TEST_LONG, true) + .prepareAndSetValues(conn).executeQuery()); + assertEquals(1, checkIdAndName(rsReader)); + + rsReader = parse(selectColumns(TEST_STRING, TEST_LONG).where(isNotEqual(TEST_LONG, 4)).orderBy(TEST_LONG, true) + .prepareAndSetValues(conn).executeQuery()); + assertEquals(4, checkIdAndName(rsReader)); + + rsReader = parse(selectColumns(TEST_STRING, TEST_LONG).where(lessThan(TEST_LONG, 3)).orderBy(TEST_LONG, true) + .prepareAndSetValues(conn).executeQuery()); + assertEquals(3, checkIdAndName(rsReader)); + + rsReader = parse(selectColumns(TEST_STRING, TEST_LONG).where(lessThanOrEqual(TEST_LONG, 3)).orderBy(TEST_LONG, + true).prepareAndSetValues(conn).executeQuery()); + assertEquals(4, checkIdAndName(rsReader)); + + ResultSet rs = getCount(TEST_TABLE).where(greaterThan(TEST_LONG, 3)).prepareAndSetValues(conn).executeQuery(); + rs.next(); + assertEquals(1, rs.getInt(1)); + + rs = getCount(TEST_TABLE).where(greaterThanOrEqual(TEST_LONG, 3)).prepareAndSetValues(conn).executeQuery(); + rs.next(); + assertEquals(2, rs.getInt(1)); + + rsReader = parse(selectColumns(TEST_STRING, TEST_LONG).where(in(TEST_LONG, 0, 1, 2)).orderBy(TEST_LONG, true) + .prepareAndSetValues(conn).executeQuery()); + assertEquals(3, checkIdAndName(rsReader)); + + rsReader = parse(selectColumns(TEST_STRING, TEST_LONG).where(notIn(TEST_LONG, 3, 4)).orderBy(TEST_LONG, true) + .prepareAndSetValues(conn).executeQuery()); + assertEquals(3, checkIdAndName(rsReader)); + + rs = getCount(TEST_TABLE).where(between(TEST_LONG, 1, 3)).prepareAndSetValues(conn).executeQuery(); + rs.next(); + assertEquals(3, rs.getInt(1)); + + rs = getCount(TEST_TABLE).where(notBetween(TEST_LONG, 1, 3)).prepareAndSetValues(conn).executeQuery(); + rs.next(); + assertEquals(2, rs.getInt(1)); + + rsReader = parse(selectColumns(TEST_STRING, TEST_LONG).where( + and(isEqual(TEST_LONG, 0), isEqual(TEST_STRING, names[1]))).orderBy(TEST_LONG, true) + .prepareAndSetValues(conn).executeQuery()); + assertEquals(0, checkIdAndName(rsReader)); + + rsReader = parse(selectColumns(TEST_STRING, TEST_LONG).where( + and(isEqual(TEST_LONG, 0), isEqual(TEST_STRING, names[0]), isEqual(TEST_BOOLEAN, false))).orderBy( + TEST_LONG, true).prepareAndSetValues(conn).executeQuery()); + assertEquals(1, checkIdAndName(rsReader)); + + rsReader = parse(selectColumns(TEST_STRING, TEST_LONG).where( + or(isEqual(TEST_LONG, 0), isEqual(TEST_STRING, names[1]))).orderBy(TEST_LONG, true) + .prepareAndSetValues(conn).executeQuery()); + assertEquals(2, checkIdAndName(rsReader)); + } + + private void _testInsertAndGetCountAndprepare() throws SQLException { + int i; + List> maps = new ArrayList>(); + SqlStatement insert = insertInto(TEST_TABLE).value(TEST_LONG, "1").value(TEST_STRING, "2").value(TEST_BOOLEAN, + true); + SqlStatement update = update(TEST_TABLE).set(TEST_BOOLEAN, false).where( + and(isEqual(TEST_LONG, "1"), isEqual(TEST_STRING, "2"))); + PreparedStatement pUpdate = update.prepare(conn); + PreparedStatement pInsert = insert.prepare(conn); + for (i = 0; i < 4; i++) { + Map values = new HashMap(); + values.put("1", i); + values.put("2", names[i]); + insert.getNewStatementWithValues(values).prepare(pInsert).execute(); + maps.add(values); + } + + ResultSet rs = getCount(TEST_TABLE).prepareAndSetValues(conn).executeQuery(); + rs.next(); + int cnt = myGetCount(TEST_TABLE); + assertEquals(4, cnt); + assertEquals(rs.getInt(1), cnt); + + ResultSetReader rsReader = parse(selectAllFrom(TEST_TABLE).where(isEqual(TEST_BOOLEAN, true)).orderBy( + TEST_LONG, true).prepareAndSetValues(conn).executeQuery()); + assertEquals(4, checkIdAndName(rsReader)); + + update.prepareForBatch(conn, maps, pUpdate).executeBatch(); + rsReader = parse(selectAllFrom(TEST_TABLE).where(isEqual(TEST_BOOLEAN, false)).orderBy(TEST_LONG, true) + .prepareAndSetValues(conn).executeQuery()); + assertEquals(4, checkIdAndName(rsReader)); + + currTime = new java.sql.Timestamp(Calendar.getInstance().getTimeInMillis()); + SqlStatement stmt = insertInto(TEST_TABLE).value(TEST_LONG, "1").value(TEST_STRING, "2").value(TEST_BOOLEAN, + "3").value(TEST_TIMESTAMP, "4").value(TEST_BLOB, "5"); + Map values = new HashMap(); + values.put("1", i); + values.put("2", names[i]); + values.put("3", true); + values.put("4", currTime); + values.put("5", names[i].getBytes()); + PreparedStatement pstmt = stmt.prepare(conn); + stmt.getNewStatementWithValues(values).prepare(pstmt).executeUpdate(); + assertEquals(5, myGetCount(TEST_TABLE)); + } + + private void _testParser() throws SQLException { + ResultSetReader rsReader = parse(selectAllFrom(TEST_TABLE).where(isEqual(TEST_LONG, 4)).prepareAndSetValues( + conn).executeQuery()); + rsReader.next(); + assertEquals(4, rsReader.getLong(TEST_LONG).longValue()); + assertEquals(names[4], rsReader.getString(TEST_STRING)); + assertEquals(String.format("yyyyy-mm-dd hh:mm", currTime), String.format("yyyyy-mm-dd hh:mm", rsReader + .getTimestamp(TEST_TIMESTAMP))); + assertEquals(true, rsReader.getBoolean(TEST_BOOLEAN).booleanValue()); + assertEquals(names[4], new String(rsReader.getByteArray(TEST_BLOB))); + rsReader.close(); + } + + private int myGetCount(Table table) throws SQLException { + ResultSet rs = conn.prepareStatement("SELECT count(*) FROM " + table).executeQuery(); + rs.next(); + return rs.getInt(1); + } + + private int checkIdAndName(ResultSetReader rsReader) throws SQLException { + int cnt = 0; + while (rsReader.next()) { + assertEquals(cnt, rsReader.getLong(TEST_LONG).longValue()); + assertEquals(names[cnt], rsReader.getString(TEST_STRING)); + cnt++; + } + rsReader.close(); + return cnt; + } +} Index: oozie/core/src/test/java/org/apache/oozie/util/TestParamChecker.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/util/TestParamChecker.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/util/TestParamChecker.java (revision 0) @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.util; + +import org.apache.oozie.test.XTestCase; + +import java.util.ArrayList; +import java.util.Arrays; + +public class TestParamChecker extends XTestCase { + + public void testNotNull() { + ParamChecker.notNull("value", "name"); + try { + ParamChecker.notNull(null, "name"); + fail(); + } + catch (IllegalArgumentException ex) { + //nop + } + } + + public void testNotNullElements() { + ParamChecker.notEmptyElements(new ArrayList(), "name"); + ParamChecker.notEmptyElements(Arrays.asList("a"), "name"); + try { + ParamChecker.notEmptyElements(null, "name"); + fail(); + } + catch (IllegalArgumentException ex) { + //nop + } + try { + ParamChecker.notEmptyElements(Arrays.asList("a", null), "name"); + fail(); + } + catch (IllegalArgumentException ex) { + //nop + } + } + + public void testNotEmpty() { + ParamChecker.notEmpty("value", "name"); + try { + ParamChecker.notEmpty(null, "name"); + fail(); + } + catch (IllegalArgumentException ex) { + //nop + } + try { + ParamChecker.notEmpty("", "name"); + fail(); + } + catch (IllegalArgumentException ex) { + //nop + } + } + + public void testNotEmptyElements() { + ParamChecker.notEmptyElements(new ArrayList(), "name"); + ParamChecker.notEmptyElements(Arrays.asList("a"), "name"); + try { + ParamChecker.notEmptyElements(null, "name"); + fail(); + } + catch (IllegalArgumentException ex) { + //nop + } + try { + ParamChecker.notEmptyElements(Arrays.asList("a", null), "name"); + fail(); + } + catch (IllegalArgumentException ex) { + //nop + } + } + + public void testValidToken() { + ParamChecker.validToken("azAZ09_-", "token"); + try { + ParamChecker.validToken(null, "token"); + fail(); + } + catch (IllegalArgumentException ex) { + //nop + } + try { + ParamChecker.validToken("", "token"); + fail(); + } + catch (IllegalArgumentException ex) { + //nop + } + try { + ParamChecker.validToken("@", "token"); + fail(); + } + catch (IllegalArgumentException ex) { + //nop + } + } + + public void testValidIdentifier() { + assertTrue(ParamChecker.isValidIdentifier("a")); + assertTrue(ParamChecker.isValidIdentifier("a1")); + assertTrue(ParamChecker.isValidIdentifier("a_")); + assertTrue(ParamChecker.isValidIdentifier("_")); + assertFalse(ParamChecker.isValidIdentifier("!")); + assertFalse(ParamChecker.isValidIdentifier("1")); + } + +} Index: oozie/core/src/test/java/org/apache/oozie/util/TestClassUtils.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/util/TestClassUtils.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/util/TestClassUtils.java (revision 0) @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.util; + +import org.apache.oozie.test.XTestCase; +import org.apache.hadoop.io.Writable; + +public class TestClassUtils extends XTestCase { + + public void testContainingJar() { + assertTrue(ClassUtils.findContainingJar(Writable.class).contains("hadoop-core")); + } +} Index: oozie/core/src/test/java/org/apache/oozie/util/TestELConstantsFunctions.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/util/TestELConstantsFunctions.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/util/TestELConstantsFunctions.java (revision 0) @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.util; + +import org.apache.oozie.test.XTestCase; + +import java.text.SimpleDateFormat; +import java.util.TimeZone; + +public class TestELConstantsFunctions extends XTestCase { + + public void testTrim() { + assertEquals("", ELConstantsFunctions.trim(null)); + assertEquals("a", ELConstantsFunctions.trim(" a ")); + } + + public void testConcat() { + assertEquals("a", ELConstantsFunctions.concat("a", null)); + assertEquals("b", ELConstantsFunctions.concat(null, "b")); + assertEquals("ab", ELConstantsFunctions.concat("a", "b")); + assertEquals("", ELConstantsFunctions.concat(null, null)); + } + + public void testTimestamp() throws Exception { + String s = ELConstantsFunctions.timestamp(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + sdf.setTimeZone(TimeZone.getTimeZone("UTC")); + assertNotNull(sdf.parse(s)); + } + + public void testUrlEncode() { + assertEquals("+", ELConstantsFunctions.urlEncode(" ")); + assertEquals("%25", ELConstantsFunctions.urlEncode("%")); + } +} Index: oozie/core/src/test/java/org/apache/oozie/util/TestIOUtils.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/util/TestIOUtils.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/util/TestIOUtils.java (revision 0) @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.util; + +import org.apache.oozie.test.XTestCase; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.StringReader; + +public class TestIOUtils extends XTestCase { + + public void testGetReaderAsString() throws Exception { + try { + IOUtils.getReaderAsString(new StringReader("1234"), 2); + fail(); + } + catch (IllegalArgumentException ex) { + //nop + } + assertEquals("1234", IOUtils.getReaderAsString(new StringReader("1234"), 4)); + } + + public void testGetResourceAsString() throws Exception { + try { + IOUtils.getResourceAsString("invalid-resource.txt", 2); + fail(); + } + catch (IllegalArgumentException ex) { + //nop + } + String s = IOUtils.getResourceAsString("test-ioutils.txt", 10); + assertEquals("abcde", s); + try { + IOUtils.getResourceAsString("test-ioutils.txt", 2); + fail(); + } + catch (IllegalArgumentException ex) { + //nop + } + } + + public void testGetResourceAsReader() throws Exception { + IOUtils.getResourceAsReader("test-ioutils.txt", 10); + } + + public void testCopyStream() throws IOException { + byte[] original = new byte[]{0, 1, 2}; + ByteArrayInputStream is = new ByteArrayInputStream(original); + ByteArrayOutputStream os = new ByteArrayOutputStream(); + IOUtils.copyStream(is, os); + byte[] copy = os.toByteArray(); + assertEquals(3, copy.length); + assertEquals(original.length, copy.length); + for (int i = 0; i < original.length; i++) { + assertEquals(original[i], copy[i]); + } + } + +} Index: oozie/core/src/test/java/org/apache/oozie/util/TestXLog.java =================================================================== --- oozie/core/src/test/java/org/apache/oozie/util/TestXLog.java (revision 0) +++ oozie/core/src/test/java/org/apache/oozie/util/TestXLog.java (revision 0) @@ -0,0 +1,274 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.util; + +import org.apache.commons.logging.impl.SimpleLog; +import org.apache.oozie.test.XTestCase; + +public class TestXLog extends XTestCase { + + public void setUp() { + XLog.Info.reset(); + XLog.Info.remove(); + } + + public void tearDown() { + XLog.Info.reset(); + XLog.Info.remove(); + } + + public static class TestLog extends SimpleLog { + private String message; + + public TestLog() { + super(""); + } + + protected void write(StringBuffer stringBuffer) { + message = stringBuffer.toString(); + } + + public void resetMessage() { + message = null; + } + + public String getMessage() { + return message; + } + } + + public void testInfoReset() { + XLog.Info logInfo = new XLog.Info(); + XLog.Info.defineParameter("A"); + assertEquals("A[-]", logInfo.createPrefix()); + XLog.Info.reset(); + assertEquals("", logInfo.createPrefix()); + } + + public void testInfoParameters() { + XLog.Info logInfo = new XLog.Info(); + assertEquals("", logInfo.createPrefix()); + XLog.Info.defineParameter("A"); + assertEquals("A[-]", logInfo.createPrefix()); + XLog.Info.defineParameter("B"); + assertEquals("A[-] B[-]", logInfo.createPrefix()); + logInfo.setParameter("A", "a"); + assertEquals("A[a] B[-]", logInfo.createPrefix()); + logInfo.setParameter("B", "b"); + assertEquals("A[a] B[b]", logInfo.createPrefix()); + } + + public void testInfoConstructorPropagation() { + XLog.Info.defineParameter("A"); + XLog.Info logInfo = new XLog.Info(); + logInfo.setParameter("A", "a"); + assertEquals("A[a]", logInfo.createPrefix()); + logInfo = new XLog.Info(logInfo); + assertEquals("A[a]", logInfo.createPrefix()); + } + + public void testInfoThreadLocal() throws Exception { + XLog.Info.defineParameter("A"); + assertEquals("A[-]", XLog.Info.get().createPrefix()); + XLog.Info.get().setParameter("A", "a"); + assertEquals("A[a]", XLog.Info.get().createPrefix()); + Thread t = new Thread() { + public void run() { + assertEquals("A[-]", XLog.Info.get().createPrefix()); + XLog.Info.get().setParameter("A", "aa"); + } + }; + t.start(); + t.join(); + assertEquals("A[a]", XLog.Info.get().createPrefix()); + } + + public void testFactory() { + XLog log = XLog.getLog(getClass()); + assertNotNull(log); + assertEquals(XLog.class, log.getClass()); + } + + public void testFactoryLogInfoPrefix() { + XLog.Info.defineParameter("A"); + XLog.Info.get().setParameter("A", "a"); + XLog log = XLog.getLog(getClass()); + assertEquals("A[a]", log.getMsgPrefix()); + } + + public void testXLogFunctionality() { + TestLog log = new TestLog(); + TestLog app = new TestLog(); + TestLog ops = new TestLog(); + XLog xLog = new XLog(log); + + assertEquals("", xLog.getMsgPrefix()); + xLog.setMsgPrefix("prefix"); + assertEquals("prefix", xLog.getMsgPrefix()); + xLog.setMsgPrefix(null); + + xLog.loggers[1] = app; + xLog.loggers[2] = ops; + + log.setLevel(SimpleLog.LOG_LEVEL_OFF); + app.setLevel(SimpleLog.LOG_LEVEL_OFF); + ops.setLevel(SimpleLog.LOG_LEVEL_OFF); + assertFalse(xLog.isDebugEnabled()); + assertFalse(xLog.isErrorEnabled()); + assertFalse(xLog.isFatalEnabled()); + assertFalse(xLog.isInfoEnabled()); + assertFalse(xLog.isWarnEnabled()); + assertFalse(xLog.isTraceEnabled()); + + log.setLevel(SimpleLog.LOG_LEVEL_ALL); + app.setLevel(SimpleLog.LOG_LEVEL_OFF); + ops.setLevel(SimpleLog.LOG_LEVEL_OFF); + assertTrue(xLog.isDebugEnabled()); + assertTrue(xLog.isErrorEnabled()); + assertTrue(xLog.isFatalEnabled()); + assertTrue(xLog.isInfoEnabled()); + assertTrue(xLog.isWarnEnabled()); + assertTrue(xLog.isTraceEnabled()); + + log.setLevel(SimpleLog.LOG_LEVEL_OFF); + app.setLevel(SimpleLog.LOG_LEVEL_ALL); + ops.setLevel(SimpleLog.LOG_LEVEL_OFF); + assertTrue(xLog.isDebugEnabled()); + assertTru