[SPARK-28256] Failed to initialize FileContextBasedCheckpointFileManager with uri without authority

• Type: Improvement
• Status: In Progress
• Priority: Minor
• Resolution: Unresolved
• Affects Version/s: 3.1.0
• Component/s: Structured Streaming
• Assignee: Unassigned
• Reporter: Genmao Yu
• Created: 05/Jul/19 06:06
• Updated: 16/Mar/20 22:54

Description

reproduce code

CREATE TABLE `user_click_count` (`userId` STRING, `click` BIGINT)
USING org.apache.spark.sql.json
OPTIONS (path 'hdfs:///tmp/test');

error:

java.lang.RuntimeException: java.lang.reflect.InvocationTargetException
	at org.apache.hadoop.fs.AbstractFileSystem.newInstance(AbstractFileSystem.java:136)
	at org.apache.hadoop.fs.AbstractFileSystem.createFileSystem(AbstractFileSystem.java:165)
	at org.apache.hadoop.fs.AbstractFileSystem.get(AbstractFileSystem.java:250)
	at org.apache.hadoop.fs.FileContext$2.run(FileContext.java:342)
	at org.apache.hadoop.fs.FileContext$2.run(FileContext.java:339)
	at java.security.AccessController.doPrivileged(Native Method)
	at javax.security.auth.Subject.doAs(Subject.java:422)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1911)
	at org.apache.hadoop.fs.FileContext.getAbstractFileSystem(FileContext.java:339)
	at org.apache.hadoop.fs.FileContext.getFileContext(FileContext.java:456)
	at org.apache.spark.sql.execution.streaming.FileContextBasedCheckpointFileManager.<init>(CheckpointFileManager.scala:297)
	at org.apache.spark.sql.execution.streaming.CheckpointFileManager$.create(CheckpointFileManager.scala:189)
	at org.apache.spark.sql.execution.streaming.HDFSMetadataLog.<init>(HDFSMetadataLog.scala:63)
	at org.apache.spark.sql.execution.streaming.CompactibleFileStreamLog.<init>(CompactibleFileStreamLog.scala:46)
	at org.apache.spark.sql.execution.streaming.FileStreamSinkLog.<init>(FileStreamSinkLog.scala:85)
	at org.apache.spark.sql.execution.streaming.FileStreamSink.<init>(FileStreamSink.scala:98)
	at org.apache.spark.sql.execution.datasources.DataSource.createSink(DataSource.scala:297)
	at org.apache.spark.sql.execution.datasources.FindDataSourceTable$$anonfun$apply$2.applyOrElse(DataSourceStrategy.scala:379)

...

Caused by: java.lang.reflect.InvocationTargetException
	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at org.apache.hadoop.fs.AbstractFileSystem.newInstance(AbstractFileSystem.java:134)
	... 67 more
Caused by: org.apache.hadoop.HadoopIllegalArgumentException: Uri without authority: hdfs:/tmp/test/_spark_metadata
	at org.apache.hadoop.fs.AbstractFileSystem.getUri(AbstractFileSystem.java:313)
	at org.apache.hadoop.fs.AbstractFileSystem.<init>(AbstractFileSystem.java:266)
	at org.apache.hadoop.fs.Hdfs.<init>(Hdfs.java:80)
	... 72 more

Issue Links

• links to: GitHub Pull Request #25056