KMSClientProvider#ValueQueue does synchronous fetch of edeks in background async thread.

  private final LoadingCache<String, LinkedBlockingQueue<E>> keyQueues;
  // Stripped rwlocks based on key name to synchronize the queue from
  // the sync'ed rw-thread and the background async refill thread.
  private final List<ReadWriteLock> lockArray =
      new ArrayList<>(LOCK_ARRAY_SIZE);

public List<E> getAtMost(String keyName, int num) throws IOException,
      ExecutionException {
     ...
     ...
         readLock(keyName);
        E val = keyQueue.poll();
        readUnlock(keyName);
     ...
  }

  private void submitRefillTask(final String keyName,
      final Queue<E> keyQueue) throws InterruptedException {
              ...
              ...
              writeLock(keyName); // It holds the write lock while the key is being asynchronously fetched. So the read requests for all the keys that hashes to this bucket will essentially be blocked.
              try {
                if (keyQueue.size() < threshold && !isCanceled()) {
                  refiller.fillQueueForKey(name, keyQueue,
                      cacheSize - keyQueue.size());
                }
             ...
              } finally {
                writeUnlock(keyName);
              }
            }
  }