Improve the thread pool size detection logic while loading partitioned table block metadata

  /**
   * Returns the thread pool size to load the file metadata of this table.
   * 'numPaths' is the number of paths for which the file metadata should be loaded.
   *
   * We use different thread pool sizes for HDFS and non-HDFS tables since the latter
   * supports much higher throughput of RPC calls for listStatus/listFiles. For
   * simplicity, the filesystem type is determined based on the table's root path and
   * not for each partition individually. Based on our experiments, S3 showed a linear
   * speed up (up to ~100x) with increasing number of loading threads where as the HDFS
   * throughput was limited to ~5x in un-secure clusters and up to ~3.7x in secure
   * clusters. We narrowed it down to scalability bottlenecks in HDFS RPC implementation
   * (HADOOP-14558) on both the server and the client side.
   */
  private int getLoadingThreadPoolSize(int numPaths) throws CatalogException {
    Preconditions.checkState(numPaths > 0);
    FileSystem tableFs;
    try {
      tableFs  = (new Path(getLocation())).getFileSystem(CONF);
    } catch (IOException e) {
      throw new CatalogException("Invalid table path for table: " + getFullName(), e);
    }
    int threadPoolSize = FileSystemUtil.supportsStorageIds(tableFs) ?
        MAX_HDFS_PARTITIONS_PARALLEL_LOAD : MAX_NON_HDFS_PARTITIONS_PARALLEL_LOAD;
    // Thread pool size need not exceed the number of paths to be loaded.
    return Math.min(numPaths, threadPoolSize);
  }