Uploaded image for project: 'CarbonData'
  1. CarbonData
  2. CARBONDATA-2956

CarbonReader can't support use configuration to read S3 data

    XMLWordPrintableJSON

    Details

    • Type: Bug
    • Status: Resolved
    • Priority: Major
    • Resolution: Fixed
    • Affects Version/s: 1.5.0
    • Fix Version/s: 1.5.0
    • Component/s: other
    • Labels:
      None

      Description

      Test code:

      package org.apache.carbondata.examples.sdk;
      
      import org.apache.carbondata.common.logging.LogService;
      import org.apache.carbondata.common.logging.LogServiceFactory;
      import org.apache.carbondata.core.metadata.datatype.DataTypes;
      import org.apache.carbondata.core.scan.expression.ColumnExpression;
      import org.apache.carbondata.core.scan.expression.LiteralExpression;
      import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression;
      import org.apache.carbondata.sdk.file.*;
      
      import org.apache.hadoop.conf.Configuration;
      
      import static org.apache.hadoop.fs.s3a.Constants.ACCESS_KEY;
      import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT;
      import static org.apache.hadoop.fs.s3a.Constants.SECRET_KEY;
      
      /**
       * Example for testing CarbonWriter on S3
       */
      public class SDKS3Example {
      
          public static void main(String[] args) throws Exception {
              LogService logger = LogServiceFactory.getLogService(SDKS3Example.class.getName());
              if (args == null || args.length < 3) {
                  logger.error("Usage: java CarbonS3Example: <access-key> <secret-key>"
                      + "<s3-endpoint> [table-path-on-s3] [rows]");
                  System.exit(0);
              }
      
              String path = "s3a://sdk/WriterOutput";
              if (args.length > 3) {
                  path=args[3];
              }
      
              int num = 3;
              if (args.length > 4) {
                  num = Integer.parseInt(args[4]);
              }
      
      
              Configuration conf = new Configuration();
              conf.set(ACCESS_KEY,args[0]);
              conf.set(SECRET_KEY,args[1]);
              conf.set(ENDPOINT,args[2]);
      
      //        Field[] fields = new Field[2];
      //        fields[0] = new Field("name", DataTypes.STRING);
      //        fields[1] = new Field("age", DataTypes.INT);
      //        CarbonWriterBuilder builder = CarbonWriter.builder()
      //                .setAccessKey(args[0])
      //                .setSecretKey(args[1])
      //                .setEndPoint(args[2])
      //                .outputPath(path);
      //
      //        CarbonWriter writer = builder.buildWriterForCSVInput(new Schema(fields), conf);
      //
      //        for (int i = 0; i < num; i++) {
      //            writer.write(new String[]{"robot" + (i % 10), String.valueOf(i)});
      //        }
      //        writer.close();
              // Read data
      
              EqualToExpression equalToExpression = new EqualToExpression(
                  new ColumnExpression("name", DataTypes.STRING),
                  new LiteralExpression("robot1", DataTypes.STRING));
      
              CarbonReader reader = CarbonReader
                  .builder(path, "_temp")
                  .projection(new String[]{"name", "age"})
                  .filter(equalToExpression)
                  .build(conf);
      
              System.out.println("\nData:");
              int i = 0;
              while (i < 20 && reader.hasNext()) {
                  Object[] row = (Object[]) reader.readNextRow();
                  System.out.println(row[0] + " " + row[1]);
                  i++;
              }
              System.out.println("\nFinished");
              reader.close();
          }
      }
      
      
      

      Exception:

      log4j:WARN No appenders could be found for logger (org.apache.hadoop.util.Shell).
      log4j:WARN Please initialize the log4j system properly.
      log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
      Exception in thread "main" com.amazonaws.AmazonClientException: Unable to load AWS credentials from any provider in the chain
      	at com.amazonaws.auth.AWSCredentialsProviderChain.getCredentials(AWSCredentialsProviderChain.java:117)
      	at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3521)
      	at com.amazonaws.services.s3.AmazonS3Client.headBucket(AmazonS3Client.java:1031)
      	at com.amazonaws.services.s3.AmazonS3Client.doesBucketExist(AmazonS3Client.java:994)
      	at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:297)
      	at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2653)
      	at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:92)
      	at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2687)
      	at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2669)
      	at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:371)
      	at org.apache.hadoop.fs.Path.getFileSystem(Path.java:295)
      	at org.apache.carbondata.core.datastore.filesystem.AbstractDFSCarbonFile.<init>(AbstractDFSCarbonFile.java:74)
      	at org.apache.carbondata.core.datastore.filesystem.AbstractDFSCarbonFile.<init>(AbstractDFSCarbonFile.java:66)
      	at org.apache.carbondata.core.datastore.filesystem.HDFSCarbonFile.<init>(HDFSCarbonFile.java:41)
      	at org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.<init>(S3CarbonFile.java:41)
      	at org.apache.carbondata.core.datastore.impl.DefaultFileTypeProvider.getCarbonFile(DefaultFileTypeProvider.java:53)
      	at org.apache.carbondata.core.datastore.impl.FileFactory.getCarbonFile(FileFactory.java:99)
      	at org.apache.carbondata.core.metadata.schema.table.CarbonTable.buildTable(CarbonTable.java:241)
      	at org.apache.carbondata.sdk.file.CarbonReaderBuilder.build(CarbonReaderBuilder.java:191)
      	at org.apache.carbondata.examples.sdk.SDKS3Example.main(SDKS3Example.java:91)
      
      Process finished with exit code 1
      
      

      if configure like:

       CarbonReader reader = CarbonReader
                  .builder(path, "_temp")
                  .projection(new String[]{"name", "age"})
                  .filter(equalToExpression)
                  .setAccessKey(args[0])
                  .setSecretKey(args[1])
                  .setEndPoint(args[2])
                  .build(new Configuration(false));
      

      Exception2:

      
      log4j:WARN No appenders could be found for logger (org.apache.hadoop.util.Shell).
      log4j:WARN Please initialize the log4j system properly.
      log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
      Exception in thread "main" com.amazonaws.AmazonClientException: Unable to execute HTTP request: sdk.obs.cn-north-1.myhwclouds.com: nodename nor servname provided, or not known
      	at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:454)
      	at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:232)
      	at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3528)
      	at com.amazonaws.services.s3.AmazonS3Client.headBucket(AmazonS3Client.java:1031)
      	at com.amazonaws.services.s3.AmazonS3Client.doesBucketExist(AmazonS3Client.java:994)
      	at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:297)
      	at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2653)
      	at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:92)
      	at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2687)
      	at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2669)
      	at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:371)
      	at org.apache.hadoop.fs.Path.getFileSystem(Path.java:295)
      	at org.apache.carbondata.core.datastore.filesystem.AbstractDFSCarbonFile.<init>(AbstractDFSCarbonFile.java:74)
      	at org.apache.carbondata.core.datastore.filesystem.AbstractDFSCarbonFile.<init>(AbstractDFSCarbonFile.java:66)
      	at org.apache.carbondata.core.datastore.filesystem.HDFSCarbonFile.<init>(HDFSCarbonFile.java:41)
      	at org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.<init>(S3CarbonFile.java:41)
      	at org.apache.carbondata.core.datastore.impl.DefaultFileTypeProvider.getCarbonFile(DefaultFileTypeProvider.java:53)
      	at org.apache.carbondata.core.datastore.impl.FileFactory.getCarbonFile(FileFactory.java:99)
      	at org.apache.carbondata.core.metadata.schema.table.CarbonTable.buildTable(CarbonTable.java:241)
      	at org.apache.carbondata.sdk.file.CarbonReaderBuilder.build(CarbonReaderBuilder.java:191)
      	at org.apache.carbondata.examples.sdk.SDKS3Example.main(SDKS3Example.java:91)
      Caused by: java.net.UnknownHostException: sdk.obs.cn-north-1.myhwclouds.com: nodename nor servname provided, or not known
      	at java.net.Inet6AddressImpl.lookupAllHostAddr(Native Method)
      	at java.net.InetAddress$2.lookupAllHostAddr(InetAddress.java:928)
      	at java.net.InetAddress.getAddressesFromNameService(InetAddress.java:1323)
      	at java.net.InetAddress.getAllByName0(InetAddress.java:1276)
      	at java.net.InetAddress.getAllByName(InetAddress.java:1192)
      	at java.net.InetAddress.getAllByName(InetAddress.java:1126)
      	at org.apache.http.impl.conn.SystemDefaultDnsResolver.resolve(SystemDefaultDnsResolver.java:45)
      	at org.apache.http.impl.conn.DefaultClientConnectionOperator.resolveHostname(DefaultClientConnectionOperator.java:278)
      	at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:162)
      	at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294)
      	at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:641)
      	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480)
      	at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
      	at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
      	at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:384)
      	... 20 more
      
      Process finished with exit code 1
      
      

        Attachments

          Issue Links

            Activity

              People

              • Assignee:
                xubo245 Bo Xu
                Reporter:
                xubo245 Bo Xu
              • Votes:
                0 Vote for this issue
                Watchers:
                1 Start watching this issue

                Dates

                • Created:
                  Updated:
                  Resolved:

                  Time Tracking

                  Estimated:
                  Original Estimate - Not Specified
                  Not Specified
                  Remaining:
                  Remaining Estimate - 0h
                  0h
                  Logged:
                  Time Spent - 1h 40m
                  1h 40m