[HIVE-25932] two or more sql like "insert into table test paritition (cls=1) select xxx" sometimes one sql was failed with rename ,because the two sql create a same file name

• Type: Improvement
• Status: Open
• Priority: Minor
• Resolution: Unresolved
• Affects Version/s: 2.3.7
• Component/s: Query Planning, Query Processor

Docs Text

            for (final FileStatus srcStatus : srcs) {

// final Path destFile = new Path(destf, srcStatus.getPath().getName());

              final String name = srcStatus.getPath().getName();
              final Path destDirPath = srcStatus.getPath();

              if (null == pool) {
                Path destFile = new Path(destf, name);
                int counter = 1;
                while (!destFs.rename(destDirPath, destFile)) {
                  destFile = new Path(destf, name + ("_copy_" + counter));
                  LOG.info("kugu log destFile is {}.",destFile.getName());
                  counter++;
// throw new IOException("rename for src path: " + srcStatus.getPath() + " to dest:"
// + destf + " returned false");
                }
              } else {
                futures.add(pool.submit(new Callable<Void>() {
                  @Override
                  public Void call() throws Exception {
                    SessionState.setCurrentSessionState(parentSession);
                    final String group = srcStatus.getGroup();

                    Path destFile = new Path(destf, name);
                    boolean rename_succ = false;
                    int counter = 1;
                    while (!rename_succ) {
                      rename_succ = destFs.rename(destDirPath, destFile);
                      if(rename_succ) {
                        if (inheritPerms) {
                          HdfsUtils.setFullFileStatus(conf, desiredStatus, group, destFs, destFile, false);
                        }
                      }else {
                        destFile = new Path(destf, name + ("_copy_" + counter));
                        LOG.info("kugu log destFile is {}.",destFile.getName());
                      }
                      counter++;

                    }
// if(destFs.rename(srcStatus.getPath(), destFile)) {
// if (inheritPerms) {
// HdfsUtils.setFullFileStatus(conf, desiredStatus, group, destFs, destFile, false);
// }
// } else {
// throw new IOException("rename for src path: " + srcStatus.getPath() + " to dest path:"
// + destFile + " returned false");
// }
                    return null;
                  }

• Assignee: lkl
• Reporter: lkl
• Created: 07/Feb/22 15:30
• Updated: 21/Oct/22 07:20

Description

modify the class "Hive.java "

                  for (final FileStatus srcStatus : srcs) {

//              final Path destFile = new Path(destf, srcStatus.getPath().getName());

              final String name = srcStatus.getPath().getName();
              final Path destDirPath = srcStatus.getPath();

              if (null == pool) {
                Path destFile = new Path(destf, name);
                int counter = 1;
                while (!destFs.rename(destDirPath, destFile)) {
                  destFile =  new Path(destf, name + ("_copy_" + counter));
                  LOG.info("kugu log destFile is {}.",destFile.getName());
                  counter++;
//                  throw new IOException("rename for src path: " + srcStatus.getPath() + " to dest:"
//                      + destf + " returned false");
                }
              } else {
                futures.add(pool.submit(new Callable<Void>() {
                  @Override
                  public Void call() throws Exception {
                    SessionState.setCurrentSessionState(parentSession);
                    final String group = srcStatus.getGroup();

                    Path destFile = new Path(destf, name);
                    boolean rename_succ = false;
                    int counter = 1;
                    while (!rename_succ) {
                      rename_succ = destFs.rename(destDirPath, destFile);
                      if(rename_succ) {
                        if (inheritPerms) {
                          HdfsUtils.setFullFileStatus(conf, desiredStatus, group, destFs, destFile, false);
                        }
                      }else {
                        destFile =  new Path(destf, name + ("_copy_" + counter));
                        LOG.info("kugu log destFile is {}.",destFile.getName());
                      }
                      counter++;

                    }
//                    if(destFs.rename(srcStatus.getPath(), destFile)) {
//                      if (inheritPerms) {
//                        HdfsUtils.setFullFileStatus(conf, desiredStatus, group, destFs, destFile, false);
//                      }
//                    } else {
//                      throw new IOException("rename for src path: " + srcStatus.getPath() + " to dest path:"
//                          + destFile + " returned false");
//                    }
                    return null;
                  }