Details
-
Bug
-
Status: Resolved
-
P2
-
Resolution: Fixed
-
None
-
None
Description
On a few occasions I see this test fail due to a temp directory being missing.
Sample scan from https://builds.apache.org/job/beam_PreCommit_Portable_Python_Cron/745/:
https://scans.gradle.com/s/3ra4xw4hqvlyw/console-log?task=:sdks:python:portableWordCountBatch
[grpc-default-executor-0] ERROR sdk_worker._execute - Error processing instruction 8. Original traceback is Traceback (most recent call last): File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 157, in _execute response = task() File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 190, in <lambda> self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 342, in do_instruction request.instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 368, in process_bundle bundle_processor.process_bundle(instruction_id)) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", line 589, in process_bundle ].process_encoded(data.data) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", line 143, in process_encoded self.output(decoded_value) File "apache_beam/runners/worker/operations.py", line 255, in apache_beam.runners.worker.operations.Operation.output def output(self, windowed_value, output_index=0): File "apache_beam/runners/worker/operations.py", line 256, in apache_beam.runners.worker.operations.Operation.output cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value) File "apache_beam/runners/worker/operations.py", line 143, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive self.consumer.process(windowed_value) File "apache_beam/runners/worker/operations.py", line 593, in apache_beam.runners.worker.operations.DoOperation.process with self.scoped_process_state: File "apache_beam/runners/worker/operations.py", line 594, in apache_beam.runners.worker.operations.DoOperation.process delayed_application = self.dofn_receiver.receive(o) File "apache_beam/runners/common.py", line 778, in apache_beam.runners.common.DoFnRun ner.receive self.process(windowed_value) File "apache_beam/runners/common.py", line 784, in apache_beam.runners.common.DoFnRunner.process self._reraise_augmented(exn) File "apache_beam/runners/common.py", line 851, in apache_beam.runners.common.DoFnRunner._reraise_augmented raise_with_traceback(new_exn) File "apache_beam/runners/common.py", line 782, in apache_beam.runners.common.DoFnRunner.process return self.do_fn_invoker.invoke_process(windowed_value) File "apache_beam/runners/common.py", line 594, in apache_beam.runners.common.PerWindowInvoker.invoke_process self._invoke_process_per_window( File "apache_beam/runners/common.py", line 666, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window windowed_value, self.process_method(*args_for_process)) File "/usr/local/lib/python2.7/site-packages/apache_beam/io/iobase.py", line 1041, in process self.writer = self.sink.open_writer(init_result, str(uuid.uuid4())) File "/usr/local/lib/python2.7/site-packages/apache_beam/options/value_provider.py", line 137, in _f return fnc(self, *args, **kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/io/filebasedsink.py", line 185, in open_writer return FileBasedSinkWriter(self, os.path.join(init_result, uid) + suffix) File "/usr/local/lib/python2.7/site-packages/apache_beam/io/filebasedsink.py", line 389, in __init__ self.temp_handle = self.sink.open(temp_shard_path) File "/usr/local/lib/python2.7/site-packages/apache_beam/io/textio.py", line 391, in open file_handle = super(_TextSink, self).open(temp_path) File "/usr/local/lib/python2.7/site-packages/apache_beam/options/value_provider.py", line 137, in _f return fnc(self, *args, **kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/io/filebasedsink.py", line 129, in open return FileSystems.create(temp_path, self.mime_type, self.compression_type) File "/usr/local/lib/python2.7/site-packages/apache_beam/io/filesystems.py", line 203, in create return file system.create(path, mime_type, compression_type) File "/usr/local/lib/python2.7/site-packages/apache_beam/io/localfilesystem.py", line 151, in create return self._path_open(path, 'wb', mime_type, compression_type) File "/usr/local/lib/python2.7/site-packages/apache_beam/io/localfilesystem.py", line 134, in _path_open raw_file = open(path, mode) RuntimeError: IOError: [Errno 2] No such file or directory: '/tmp/beam-temp-py-wordcount-direct-d3ca98a287bd11e997e842010a80006f/213ae725-add1-4ded-8e9b-ee9ac1ab6fd0.py-wordcount-direct' [while running 'write/Write/WriteImpl/WriteBundles']
There is also following exception earlier.
:sdks:python:portableWordCountBatch FAILED INFO:root:Using latest locally built Python SDK docker image. INFO:root:==================== <function lift_combiners at 0x7f63550fcf50> ==================== INFO:root:==================== <function expand_sdf at 0x7f63550f5050> ==================== [main] INFO org.apache.beam.runners.fnexecution.jobsubmission.JobServerDriver - ArtifactStagingService started on localhost:34803 [main] INFO org.apache.beam.runners.fnexecution.jobsubmission.JobServerDriver - Java ExpansionService started on localhost:59151 [main] INFO org.apache.beam.runners.fnexecution.jobsubmission.JobServerDriver - JobService started on localhost:57157 [grpc-default-executor-0] ERROR org.apache.beam.runners.fnexecution.jobsubmission.InMemoryJobService - Encountered Unexpected Exception for Invocation job_8dd6b809-0f47-4943-9d3e-4587db962ab9 org.apache.beam.vendor.grpc.v1p13p1.io.grpc.StatusException: NOT_FOUND at org.apache.beam.vendor.grpc.v1p13p1.io.grpc.Status.asException(Status.java:534) at org.apache.beam.runners.fnexecution.jobsubmission.InMemoryJobService.getInvocation(InMemoryJobService.java:341) at org.apache.beam.runners.fnexecution.jobsubmission.InMemoryJobService.getStateStream(InMemoryJobService.java:262) at org.apache.beam.model.jobmanagement.v1.JobServiceGrpc$MethodHandlers.invoke(JobServiceGrpc.java:770) at org.apache.beam.vendor.grpc.v1p13p1.io.grpc.stub.ServerCalls$UnaryServerCallHandler$UnaryServerCallListener.onHalfClose(ServerCalls.java:171) at org.apache.beam.vendor.grpc.v1p13p1.io.grpc.PartialForwardingServerCallListener.onHalfClose(PartialForwardingServerCallListener.java:35) at org.apache.beam.vendor.grpc.v1p13p1.io.grpc.ForwardingServerCallListener.onHalfClose(ForwardingServerCallListener.java:23) at org.apache.beam.vendor.grpc.v1p13p1.io.grpc.ForwardingServerCallListener$SimpleForwardingServerCallListener.onHalfClose(ForwardingServerCallListener.java:40) at org.apache.beam.vendor.grpc.v1p13p1.io.grpc.Contexts$ContextualizedServerCallListener.onHalfClose(Contexts.java:86) at org.apache.beam.vendor.grpc.v1p13p1.io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.halfClosed(ServerCallImpl.java:283) at org.apache.beam.vendor.grpc.v1p13p1.io.grpc.internal.ServerImpl$JumpToApplicationThreadServerStreamListener$1HalfClosed.runInContext(ServerImpl.java:707) at org.apache.beam.vendor.grpc.v1p13p1.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.apache.beam.vendor.grpc.v1p13p1.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Attachments
Issue Links
- links to