[HIVE-25683] Close reader in AcidUtils.isRawFormatFile - ASF JIRA

Log work

Agile Board

Rank to Top

Rank to Bottom

Bulk Copy Attachments

Bulk Move Attachments

Voters

Watch issue

Watchers

Create sub-task

Convert to sub-task

Move

Link

Clone

Labels

Update Comment Author

Replace String in Comment

Update Comment Visibility

Delete Comments

XML

Word

Printable

JSON

Details

Type: Bug
Status: Closed
Priority: Major
Resolution: Fixed
Affects Version/s: None
Fix Version/s: 4.0.0-alpha-1
Component/s: None
Labels:
- pull-request-available

Description

The Reader is not closed when the check is finished. This might cause issues when we have limited resources on the DFSClient.

2021-11-09T10:25:08.810Z [mdc@18060 class="conn.ClientConnectionRequestFactory" level="DEBUG" operationLogLevel="EXECUTION" queryId="hive_20211109102256_6174abc6-8422-4441-85cc-fb2c7edd33af" sessionId="dcacf28a-7cf0-4ea4-b2fe-2d44c58c7722" thread="HiveServer2-Background-Pool: Thread-5343"]
java.lang.reflect.InvocationTargetException
    at sun.reflect.GeneratedMethodAccessor71.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at com.amazonaws.http.conn.ClientConnectionRequestFactory$Handler.invoke(ClientConnectionRequestFactory.java:70)
    at com.amazonaws.http.conn.$Proxy70.get(Unknown Source)
    at com.amazonaws.thirdparty.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:190)
    at com.amazonaws.thirdparty.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
    at com.amazonaws.thirdparty.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
    at com.amazonaws.thirdparty.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
    at com.amazonaws.thirdparty.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
    at com.amazonaws.http.apache.client.impl.SdkHttpClient.execute(SdkHttpClient.java:72)
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1333)
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145)
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802)
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770)
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744)
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704)
    at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686)
    at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550)
    at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530)
    at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5227)
    at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5173)
    at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1360)
    at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$getObjectMetadata$10(S3AFileSystem.java:2447)
    at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:414)
    at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:377)
    at org.apache.hadoop.fs.s3a.S3AFileSystem.getObjectMetadata(S3AFileSystem.java:2435)
    at org.apache.hadoop.fs.s3a.S3AFileSystem.getObjectMetadata(S3AFileSystem.java:2415)
    at org.apache.hadoop.fs.s3a.S3AFileSystem.s3GetFileStatus(S3AFileSystem.java:3678)
    at org.apache.hadoop.fs.s3a.S3AFileSystem.innerGetFileStatus(S3AFileSystem.java:3590)
    at org.apache.hadoop.fs.s3a.S3AFileSystem.extractOrFetchSimpleFileStatus(S3AFileSystem.java:5272)
    at org.apache.hadoop.fs.s3a.S3AFileSystem.open(S3AFileSystem.java:1371)
    at org.apache.hadoop.fs.s3a.S3AFileSystem.open(S3AFileSystem.java:1347)
    at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:957)
    at org.apache.orc.impl.ReaderImpl.extractFileTail(ReaderImpl.java:547)
    at org.apache.orc.impl.ReaderImpl.<init>(ReaderImpl.java:370)
    at org.apache.hadoop.hive.ql.io.orc.ReaderImpl.<init>(ReaderImpl.java:61)
    at org.apache.hadoop.hive.ql.io.orc.OrcFile.createReader(OrcFile.java:112)
    at org.apache.hadoop.hive.ql.io.AcidUtils$MetaDataFile.isRawFormatFile(AcidUtils.java:2488)
    at org.apache.hadoop.hive.ql.io.AcidUtils.validateAcidFiles(AcidUtils.java:3035)
    at org.apache.hadoop.hive.ql.io.AcidUtils.validateAcidFiles(AcidUtils.java:3029)
    at org.apache.hadoop.hive.ql.parse.LoadSemanticAnalyzer.applyConstraintsAndGetFiles(LoadSemanticAnalyzer.java:189)
    at org.apache.hadoop.hive.ql.parse.LoadSemanticAnalyzer.analyzeLoad(LoadSemanticAnalyzer.java:324)
    at org.apache.hadoop.hive.ql.parse.LoadSemanticAnalyzer.analyzeInternal(LoadSemanticAnalyzer.java:243)
    at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:307)
    at org.apache.hadoop.hive.ql.Compiler.analyze(Compiler.java:223)
    at org.apache.hadoop.hive.ql.Compiler.compile(Compiler.java:105)
    at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:200)
    at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:632)
    at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:578)
    at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:572)
    at org.apache.hadoop.hive.ql.reexec.ReExecDriver.compileAndRespond(ReExecDriver.java:127)
    at org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:204)
    at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork$1.run(SQLOperation.java:327)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:422)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1898)
    at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork.run(SQLOperation.java:349)
    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
    at java.lang.Thread.run(Thread.java:748)
Caused by: com.amazonaws.thirdparty.apache.http.conn.ConnectionPoolTimeoutException: Timeout waiting for connection from pool
    at com.amazonaws.thirdparty.apache.http.impl.conn.PoolingHttpClientConnectionManager.leaseConnection(PoolingHttpClientConnectionManager.java:316)
    at com.amazonaws.thirdparty.apache.http.impl.conn.PoolingHttpClientConnectionManager$1.get(PoolingHttpClientConnectionManager.java:282)
    ... 65 more