Tensor flow on Flink taking too much time to process data

--- Thread #140477558937344 name: Thread-55 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 74, in _worker work_item.run() File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 62, in run result = self.fn(*self.args, **self.kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 168, in task self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 132, in _execute response = task() File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 168, in <lambda> self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 219, in do_instruction request.instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 234, in process_bundle bundle_processor.process_bundle(instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", line 437, in process_bundle ].process_encoded(data.data) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", line 125, in process_encoded self.output(decoded_value) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py", line 1448, in process *args, **kwargs)) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py", line 587, in add_inputs accumulator = self.add_input(accumulator, element, *args, **kwargs) File "/usr/local/lib/python2.7/site-packages/tensorflow_transform/beam/analyzer_impls.py", line 264, in add_input summary=tf.constant(summary))) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 905, in run run_metadata_ptr) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1137, in _run feed_dict_tensor, options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1355, in _do_run options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1361, in _do_call return fn(*args) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1332, in _run_fn self._extend_graph() File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1392, in _extend_graph graph_def.SerializeToString(), status)

--- Thread #140471896581888 name: Thread-153 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140480218113792 name: Thread-12 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471528552192 name: Thread-199 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140476451624704 name: Thread-86 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471930152704 name: Thread-143 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471740630784 name: Thread-183 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140479144371968 name: Thread-40 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140479152764672 name: Thread-34 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140472399947520 name: Thread-122 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140475956717312 name: Thread-88 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140479689635584 name: Thread-15 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471804360448 name: Thread-150 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140472391554816 name: Thread-126 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140474602440448 name: Thread-214 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140480226506496 name: Thread-6 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 191, in consume_request_iterator request = next(request_iterator) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/log_handler.py", line 111, in _write_log_entries log_entries = [self._log_entry_queue.get()] File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140476501980928 name: Thread-98 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471731975936 name: Thread-162 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140476996888320 name: Thread-92 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471692900096 name: Thread-167 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140472383162112 name: Thread-128 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140477038851840 name: Thread-74 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140477533759232 name: Thread-120 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140477558937344 name: Thread-55 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 74, in _worker work_item.run() File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 62, in run result = self.fn(*self.args, **self.kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 168, in task self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 132, in _execute response = task() File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 168, in <lambda> self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 219, in do_instruction request.instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 234, in process_bundle bundle_processor.process_bundle(instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", line 437, in process_bundle ].process_encoded(data.data) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", line 125, in process_encoded self.output(decoded_value) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py", line 1448, in process *args, **kwargs)) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py", line 587, in add_inputs accumulator = self.add_input(accumulator, element, *args, **kwargs) File "/usr/local/lib/python2.7/site-packages/tensorflow_transform/beam/analyzer_impls.py", line 264, in add_input summary=tf.constant(summary))) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 905, in run run_metadata_ptr) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1137, in _run feed_dict_tensor, options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1355, in _do_run options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1361, in _do_call return fn(*args) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1332, in _run_fn self._extend_graph() File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1392, in _extend_graph graph_def.SerializeToString(), status) --- Thread #140477575722752 name: Thread-48 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140472374769408 name: Thread-129 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140478070630144 name: Thread-52 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471603033856 name: Thread-181 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140478112593664 name: Thread-41 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471563171584 name: Thread-178 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140478607501056 name: Thread-29 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140472366376704 name: Thread-130 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471493146368 name: Thread-220 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140474594047744 name: Thread-208 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140472357984000 name: Thread-138 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) --- Thread #140471904974592 name: Thread-146 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140472005687040 name: Thread-137 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140479681242880 name: Thread-16 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471795967744 name: Thread-151 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471749023488 name: Thread-176 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140479723206400 name: Thread-11 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 191, in consume_request_iterator request = next(request_iterator) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 104, in get_responses response = self._responses.get() File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471972116224 name: Thread-136 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471997294336 name: Thread-133 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140476988495616 name: Thread-76 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471783118592 name: Thread-155 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471723583232 name: Thread-168 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140479186335488 name: Thread-18 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 191, in consume_request_iterator request = next(request_iterator) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", line 244, in _write_outputs data = [self._to_send.get()] File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140477030459136 name: Thread-68 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471510718208 name: Thread-191 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471988901632 name: Thread-134 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) --- Thread #140476485195520 name: Thread-80 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 74, in _worker work_item.run() File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 62, in run result = self.fn(*self.args, **self.kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 168, in task self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 132, in _execute response = task() File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 168, in <lambda> self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 219, in do_instruction request.instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 234, in process_bundle bundle_processor.process_bundle(instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", line 437, in process_bundle ].process_encoded(data.data) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", line 125, in process_encoded self.output(decoded_value) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py", line 1448, in process *args, **kwargs)) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py", line 587, in add_inputs accumulator = self.add_input(accumulator, element, *args, **kwargs) File "/usr/local/lib/python2.7/site-packages/tensorflow_transform/beam/analyzer_impls.py", line 275, in add_input next_stamp_token=self._stamp_token)) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 905, in run run_metadata_ptr) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1137, in _run feed_dict_tensor, options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1355, in _do_run options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1361, in _do_call return fn(*args) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1332, in _run_fn self._extend_graph() File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1392, in _extend_graph graph_def.SerializeToString(), status) --- Thread #140477567330048 name: Thread-59 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140479698028288 name: read_state --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 390, in pull_responses for response in responses: File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 366, in next return self._next() File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 348, in _next self._state.condition.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140478062237440 name: Thread-63 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471980508928 name: Thread-135 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140474220279552 name: Thread-114 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) --- Thread #140478104200960 name: Thread-37 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471484229376 name: Thread-209 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140478599108352 name: Thread-31 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471938545408 name: Thread-142 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140478641071872 name: Thread-25 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471701554944 name: Thread-170 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140479135979264 name: Thread-50 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140479177942784 name: read_grpc_client_inputs --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", line 273, in <lambda> target=lambda: self._read_inputs(elements_iterator), File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", line 260, in _read_inputs for elements in elements_iterator: File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 366, in next return self._next() File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 348, in _next self._state.condition.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140474568869632 name: Thread-115 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140475948324608 name: Thread-105 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140474203494144 name: Thread-117 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140481212552960 name: MainThread --- File "/usr/local/lib/python2.7/runpy.py", line 174, in _run_module_as_main "__main__", fname, loader, pkg_name) File "/usr/local/lib/python2.7/runpy.py", line 72, in _run_code exec code in run_globals File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker_main.py", line 213, in <module> main(sys.argv) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker_main.py", line 141, in main worker_count=_get_worker_count(sdk_pipeline_options)).run() File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 109, in run for work_request in control_stub.Control(get_responses()): File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 366, in next return self._next() File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 348, in _next self._state.condition.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471913367296 name: Thread-152 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140480209721088 name: Thread-10 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 703, in channel_spin event = state.channel.next_call_event() --- Thread #140471955330816 name: Thread-140 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) --- Thread #140474195101440 name: Thread-212 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140478079022848 name: Thread-44 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 74, in _worker work_item.run() File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 62, in run result = self.fn(*self.args, **self.kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 168, in task self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 132, in _execute response = task() File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 168, in <lambda> self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 219, in do_instruction request.instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 234, in process_bundle bundle_processor.process_bundle(instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", line 437, in process_bundle ].process_encoded(data.data) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", line 125, in process_encoded self.output(decoded_value) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py", line 1448, in process *args, **kwargs)) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py", line 587, in add_inputs accumulator = self.add_input(accumulator, element, *args, **kwargs) File "/usr/local/lib/python2.7/site-packages/tensorflow_transform/beam/analyzer_impls.py", line 275, in add_input next_stamp_token=self._stamp_token)) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 905, in run run_metadata_ptr) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1137, in _run feed_dict_tensor, options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1355, in _do_run options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1361, in _do_call return fn(*args) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1332, in _run_fn self._extend_graph() File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1392, in _extend_graph graph_def.SerializeToString(), status) --- Thread #140471577855744 name: Thread-180 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471946938112 name: Thread-216 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140477550544640 name: Thread-65 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 74, in _worker work_item.run() File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 62, in run result = self.fn(*self.args, **self.kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 168, in task self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 132, in _execute response = task() File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 168, in <lambda> self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 219, in do_instruction request.instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 234, in process_bundle bundle_processor.process_bundle(instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", line 434, in process_bundle instruction_id, expected_targets): File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", line 205, in input_elements data = received.get(timeout=1) File "/usr/local/lib/python2.7/Queue.py", line 177, in get self.not_empty.wait(remaining) File "/usr/local/lib/python2.7/threading.py", line 359, in wait _sleep(delay) --- Thread #140471545337600 name: Thread-184 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471594641152 name: Thread-173 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140474186708736 name: Thread-124 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140478095808256 name: Thread-38 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140474585655040 name: Thread-206 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471774725888 name: Thread-157 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140478632679168 name: Thread-46 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471766071040 name: Thread-158 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) --- Thread #140480260077312 name: Thread-13 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 703, in channel_spin event = state.channel.next_call_event() --- Thread #140471519373056 name: Thread-189 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140479169550080 name: Thread-19 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140475939931904 name: Thread-95 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471554254592 name: Thread-218 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471757678336 name: Thread-159 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140480251684608 name: read_log_control_messages --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/log_handler.py", line 72, in <lambda> target=lambda: self._read_log_control_messages(), File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/log_handler.py", line 131, in _read_log_control_messages for _ in log_control_iterator: File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 366, in next return self._next() File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 348, in _next self._state.condition.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471888189184 name: Thread-201 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140476476802816 name: Thread-90 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 74, in _worker work_item.run() File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 62, in run result = self.fn(*self.args, **self.kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 168, in task self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 132, in _execute response = task() File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 168, in <lambda> self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 219, in do_instruction request.instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 234, in process_bundle bundle_processor.process_bundle(instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", line 437, in process_bundle ].process_encoded(data.data) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", line 125, in process_encoded self.output(decoded_value) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py", line 1448, in process *args, **kwargs)) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py", line 587, in add_inputs accumulator = self.add_input(accumulator, element, *args, **kwargs) File "/usr/local/lib/python2.7/site-packages/tensorflow_transform/beam/analyzer_impls.py", line 275, in add_input next_stamp_token=self._stamp_token)) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 905, in run run_metadata_ptr) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1137, in _run feed_dict_tensor, options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1355, in _do_run options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1361, in _do_call return fn(*args) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1332, in _run_fn self._extend_graph() File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1392, in _extend_graph graph_def.SerializeToString(), status) --- Thread #140480243291904 name: status-server-demon --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker_main.py", line 86, in start httpd.serve_forever() File "/usr/local/lib/python2.7/SocketServer.py", line 233, in serve_forever self._handle_request_noblock() File "/usr/local/lib/python2.7/SocketServer.py", line 290, in _handle_request_noblock self.process_request(request, client_address) File "/usr/local/lib/python2.7/SocketServer.py", line 318, in process_request self.finish_request(request, client_address) File "/usr/local/lib/python2.7/SocketServer.py", line 331, in finish_request self.RequestHandlerClass(request, client_address, self) File "/usr/local/lib/python2.7/SocketServer.py", line 652, in __init__ self.handle() File "/usr/local/lib/python2.7/BaseHTTPServer.py", line 340, in handle self.handle_one_request() File "/usr/local/lib/python2.7/BaseHTTPServer.py", line 328, in handle_one_request method() File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker_main.py", line 74, in do_GET for line in StatusServer.get_thread_dump(): File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker_main.py", line 53, in get_thread_dump lines.append(''.join(traceback.format_stack(frames[t.ident]))) --- Thread #140479706420992 name: Thread-14 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 191, in consume_request_iterator request = next(request_iterator) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 381, in request_iter request = self._requests.get() File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140477013673728 name: Thread-82 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140475965110016 name: Thread-100 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471921760000 name: Thread-154 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140478087415552 name: Thread-64 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471536944896 name: Thread-195 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140479672850176 name: Thread-17 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 703, in channel_spin event = state.channel.next_call_event() --- Thread #140475923146496 name: Thread-97 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140478615893760 name: Thread-28 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140477022066432 name: Thread-70 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140478624286464 name: Thread-27 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140476460017408 name: Thread-102 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140478649464576 name: Thread-61 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140475931539200 name: Thread-103 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140479161157376 name: Thread-20 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140477525366528 name: Thread-77 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140476468410112 name: Thread-84 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140475562456832 name: Thread-106 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140480234899200 name: Thread-5 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 703, in channel_spin event = state.channel.next_call_event() --- Thread #140477005281024 name: Thread-72 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140475554064128 name: Thread-107 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140471963723520 name: Thread-139 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140476493588224 name: Thread-104 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140477542151936 name: Thread-57 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire() --- Thread #140474577262336 name: Thread-204 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()