IncrementalAssignmentConsumerEventHandler throws error when attempting to remove a partition that isn't assigned

def handle_partitions_revoked(self, event):
    self.revoked_count += 1
    self.state = ConsumerState.Rebalancing
    self.position = {}
    for topic_partition in event["partitions"]:
        topic = topic_partition["topic"]
        partition = topic_partition["partition"]
        self.assignment.remove(TopicPartition(topic, partition))