Details
-
Bug
-
Status: Closed
-
Minor
-
Resolution: Fixed
-
trunk, 3.3.1
-
None
Description
in the example below, dataset B is defined in both coordinator.xml and include dataset file. oozie fails to submit job with error:
Error: E0803 : E0803: IO error, null
coordinator.xml:
<datasets>
<include>${include_ds_files}</include>
<dataset name="B" frequency="${coord:minutes(5)}" initial-instance="${ds_start}" timezone="${timezone}">
<uri-template>hcat://${hcat_server}/${hcat_out_db}/${hcat_out_table}/datestamp=${YEAR}${MONTH}${DAY}${HOUR}${MINUTE};region=${hcat_region}</uri-template>
</dataset>
</datasets>
include_ds_files:
<datasets>
<include>${include_ds_files}</include>
<dataset name="B" frequency="${coord:minutes(5)}" initial-instance="${ds_start}" timezone="${timezone}">
<uri-template>hcat://${hcat_server}/${hcat_out_db}/${hcat_out_table}/datestamp=${YEAR}${MONTH}${DAY}${HOUR}${MINUTE};region=${hcat_region}</uri-template>
</dataset>
</datasets>
oozie log:
2013-02-07 01:02:01,308 WARN CoordSubmitXCommand:544 - USER[mchiang] GROUP[users] TOKEN[-] APP[-] JOB[-] ACTION[-] ERROR:
java.util.ConcurrentModificationException
at org.jdom.ContentList$FilterListIterator.checkConcurrentModification(ContentList.java:940)
at org.jdom.ContentList$FilterListIterator.nextIndex(ContentList.java:829)
at org.jdom.ContentList$FilterListIterator.hasNext(ContentList.java:785)
at org.apache.oozie.command.coord.CoordSubmitXCommand.removeDataSet(CoordSubmitXCommand.java:1017)
at org.apache.oozie.command.coord.CoordSubmitXCommand.includeDataSets(CoordSubmitXCommand.java:952)
at org.apache.oozie.command.coord.CoordSubmitXCommand.basicResolveAndIncludeDS(CoordSubmitXCommand.java:558)
at org.apache.oozie.command.coord.CoordSubmitXCommand.submit(CoordSubmitXCommand.java:225)
at org.apache.oozie.command.SubmitTransitionXCommand.execute(SubmitTransitionXCommand.java:81)
at org.apache.oozie.command.SubmitTransitionXCommand.execute(SubmitTransitionXCommand.java:29)
at org.apache.oozie.command.XCommand.call(XCommand.java:277)
at org.apache.oozie.CoordinatorEngine.submitJob(CoordinatorEngine.java:375)
at org.apache.oozie.servlet.V1JobsServlet.submitCoordinatorJob(V1JobsServlet.java:219)
at org.apache.oozie.servlet.V1JobsServlet.submitJob(V1JobsServlet.java:83)
at org.apache.oozie.servlet.BaseJobsServlet.doPost(BaseJobsServlet.java:100)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:637)
at org.apache.oozie.servlet.JsonRestServlet.service(JsonRestServlet.java:286)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at com.yahoo.oozie.security.authentication.filter.YAuthFilter$2.doFilter(YAuthFilter.java:135)
at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:372)
at com.yahoo.oozie.security.authentication.filter.YAuthFilter.doFilter(YAuthFilter.java:139)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.servlet.filter.YHdrsFilter.doFilter(YHdrsFilter.java:68)
at yjava.servlet.filter.YHdrsFilter.doFilter(YHdrsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.cookie.CookieDataFilter.doFilter(CookieDataFilter.java:447)
at yjava.cookie.CookieDataFilter.doFilter(CookieDataFilter.java:219)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.servlet.filter.DoNotTrackFilter.doFilter(DoNotTrackFilter.java:104)
at yjava.servlet.filter.DoNotTrackFilter.doFilter(DoNotTrackFilter.java:50)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.remote.ip.RemoteIPFilter.doFilter(RemoteIPFilter.java:104)
at yjava.remote.ip.RemoteIPFilter.doFilter(RemoteIPFilter.java:65)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.security.yiv.servlet.InputValidationFilter.doFilter(InputValidationFilter.java:219)
at yjava.security.yiv.servlet.InputValidationFilter.doFilter(InputValidationFilter.java:143)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
at yjava.tomcat.valves.YahooConnectionValve.invoke(YahooConnectionValve.java:191)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:293)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:859)
at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:602)
at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:489)
at java.lang.Thread.run(Thread.java:662)
2013-02-07 01:02:01,309 ERROR CoordSubmitXCommand:538 - USER[mchiang] GROUP[users] TOKEN[-] APP[-] JOB[-] ACTION[-] XException,
org.apache.oozie.command.CommandException: E0803: IO error, null
at org.apache.oozie.command.coord.CoordSubmitXCommand.submit(CoordSubmitXCommand.java:297)
at org.apache.oozie.command.SubmitTransitionXCommand.execute(SubmitTransitionXCommand.java:81)
at org.apache.oozie.command.SubmitTransitionXCommand.execute(SubmitTransitionXCommand.java:29)
at org.apache.oozie.command.XCommand.call(XCommand.java:277)
at org.apache.oozie.CoordinatorEngine.submitJob(CoordinatorEngine.java:375)
at org.apache.oozie.servlet.V1JobsServlet.submitCoordinatorJob(V1JobsServlet.java:219)
at org.apache.oozie.servlet.V1JobsServlet.submitJob(V1JobsServlet.java:83)
at org.apache.oozie.servlet.BaseJobsServlet.doPost(BaseJobsServlet.java:100)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:637)
at org.apache.oozie.servlet.JsonRestServlet.service(JsonRestServlet.java:286)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at com.yahoo.oozie.security.authentication.filter.YAuthFilter$2.doFilter(YAuthFilter.java:135)
at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:372)
at com.yahoo.oozie.security.authentication.filter.YAuthFilter.doFilter(YAuthFilter.java:139)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.servlet.filter.YHdrsFilter.doFilter(YHdrsFilter.java:68)
at yjava.servlet.filter.YHdrsFilter.doFilter(YHdrsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.cookie.CookieDataFilter.doFilter(CookieDataFilter.java:447)
at yjava.cookie.CookieDataFilter.doFilter(CookieDataFilter.java:219)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.servlet.filter.DoNotTrackFilter.doFilter(DoNotTrackFilter.java:104)
at yjava.servlet.filter.DoNotTrackFilter.doFilter(DoNotTrackFilter.java:50)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.remote.ip.RemoteIPFilter.doFilter(RemoteIPFilter.java:104)
at yjava.remote.ip.RemoteIPFilter.doFilter(RemoteIPFilter.java:65)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.security.yiv.servlet.InputValidationFilter.doFilter(InputValidationFilter.java:219)
at yjava.security.yiv.servlet.InputValidationFilter.doFilter(InputValidationFilter.java:143)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
at yjava.tomcat.valves.YahooConnectionValve.invoke(YahooConnectionValve.java:191)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:293)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:859)
at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:602)
at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:489)
at java.lang.Thread.run(Thread.java:662)
Caused by: java.util.ConcurrentModificationException
at org.jdom.ContentList$FilterListIterator.checkConcurrentModification(ContentList.java:940)
at org.jdom.ContentList$FilterListIterator.nextIndex(ContentList.java:829)
at org.jdom.ContentList$FilterListIterator.hasNext(ContentList.java:785)
at org.apache.oozie.command.coord.CoordSubmitXCommand.removeDataSet(CoordSubmitXCommand.java:1017)
at org.apache.oozie.command.coord.CoordSubmitXCommand.includeDataSets(CoordSubmitXCommand.java:952)
at org.apache.oozie.command.coord.CoordSubmitXCommand.basicResolveAndIncludeDS(CoordSubmitXCommand.java:558)
at org.apache.oozie.command.coord.CoordSubmitXCommand.submit(CoordSubmitXCommand.java:225)
... 53 more
2013-02-07 01:02:01,310 WARN V1JobsServlet:544 - USER[mchiang] GROUP[users] TOKEN[-] APP[-] JOB[-] ACTION[-] URL[POST http://gsbl90547.blue.ygrid.yahoo.com:4080/oozie/v1/jobs?action=start] error[E0803], E0803: IO error, null
org.apache.oozie.servlet.XServletException: E0803: IO error, null
at org.apache.oozie.servlet.V1JobsServlet.submitCoordinatorJob(V1JobsServlet.java:224)
at org.apache.oozie.servlet.V1JobsServlet.submitJob(V1JobsServlet.java:83)
at org.apache.oozie.servlet.BaseJobsServlet.doPost(BaseJobsServlet.java:100)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:637)
at org.apache.oozie.servlet.JsonRestServlet.service(JsonRestServlet.java:286)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at com.yahoo.oozie.security.authentication.filter.YAuthFilter$2.doFilter(YAuthFilter.java:135)
at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:372)
at com.yahoo.oozie.security.authentication.filter.YAuthFilter.doFilter(YAuthFilter.java:139)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.servlet.filter.YHdrsFilter.doFilter(YHdrsFilter.java:68)
at yjava.servlet.filter.YHdrsFilter.doFilter(YHdrsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.cookie.CookieDataFilter.doFilter(CookieDataFilter.java:447)
at yjava.cookie.CookieDataFilter.doFilter(CookieDataFilter.java:219)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.servlet.filter.DoNotTrackFilter.doFilter(DoNotTrackFilter.java:104)
at yjava.servlet.filter.DoNotTrackFilter.doFilter(DoNotTrackFilter.java:50)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.remote.ip.RemoteIPFilter.doFilter(RemoteIPFilter.java:104)
at yjava.remote.ip.RemoteIPFilter.doFilter(RemoteIPFilter.java:65)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at yjava.servlet.FilterChainInvoker$ServletFilterChainInvoker.invoke(FilterChainInvoker.java:49)
at yjava.security.yiv.servlet.InputValidationFilter.doFilter(InputValidationFilter.java:219)
at yjava.security.yiv.servlet.InputValidationFilter.doFilter(InputValidationFilter.java:143)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
at yjava.tomcat.valves.YahooConnectionValve.invoke(YahooConnectionValve.java:191)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:293)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:859)
at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:602)
at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:489)
at java.lang.Thread.run(Thread.java:662)
Caused by: org.apache.oozie.CoordinatorEngineException: E0803: IO error, null
at org.apache.oozie.CoordinatorEngine.submitJob(CoordinatorEngine.java:378)
at org.apache.oozie.servlet.V1JobsServlet.submitCoordinatorJob(V1JobsServlet.java:219)
... 48 more
Caused by: org.apache.oozie.command.CommandException: E0803: IO error, null
at org.apache.oozie.command.coord.CoordSubmitXCommand.submit(CoordSubmitXCommand.java:297)
at org.apache.oozie.command.SubmitTransitionXCommand.execute(SubmitTransitionXCommand.java:81)
at org.apache.oozie.command.SubmitTransitionXCommand.execute(SubmitTransitionXCommand.java:29)
at org.apache.oozie.command.XCommand.call(XCommand.java:277)
at org.apache.oozie.CoordinatorEngine.submitJob(CoordinatorEngine.java:375)
... 49 more
Caused by: java.util.ConcurrentModificationException
at org.jdom.ContentList$FilterListIterator.checkConcurrentModification(ContentList.java:940)
at org.jdom.ContentList$FilterListIterator.nextIndex(ContentList.java:829)
at org.jdom.ContentList$FilterListIterator.hasNext(ContentList.java:785)
at org.apache.oozie.command.coord.CoordSubmitXCommand.removeDataSet(CoordSubmitXCommand.java:1017)
at org.apache.oozie.command.coord.CoordSubmitXCommand.includeDataSets(CoordSubmitXCommand.java:952)
at org.apache.oozie.command.coord.CoordSubmitXCommand.basicResolveAndIncludeDS(CoordSubmitXCommand.java:558)
at org.apache.oozie.command.coord.CoordSubmitXCommand.submit(CoordSubmitXCommand.java:225)
... 53 more