List.minus() slow for Numbers

// if (nlgnSort && (head instanceof Comparable)) {
    //n*LOG(n) version
    Set<T> answer;
    if (head instanceof Number) {
        answer = new TreeSet<>(comparator);
        answer.addAll(self1);
        for (T t : self1) {
            if (t instanceof Number) {
                for (Object t2 : removeMe1) {
                    if (t2 instanceof Number) {
                        if (comparator.compare(t, (T) t2) == 0)
                            answer.remove(t);
                    }
                }
            } else {
                if (removeMe1.contains(t))
                    answer.remove(t);
            }
        }
    } else {
        answer = new TreeSet<>(comparator);
        answer.addAll(self1);
        answer.removeAll(removeMe1);
    }

    for (T o : self1) {
        if (answer.contains(o))
            ansCollection.add(o);
    }
} else {
    //n*n version 

if (head instanceof Number) {