Details
-
Bug
-
Status: Open
-
P3
-
Resolution: Unresolved
-
2.13.0
-
None
-
None
-
Linux(Debian GNU/Linux 9) ( SMP x86_64 GNU/Linux)
Python 2.7.13
Content of requirements.txt file
------------------------------------------
apache-beam==2.13.0
apache-beam[gcp]
google-cloud-core==1.0.3
google-cloud-iam
google-cloud-bigquery==1.6.1
PyYAML==3.13
six==1.12.0
pyOpenSSL
httplib2==0.12.0
google-apitools
google-api-python-client
google-cloud-storage==1.17.0
Linux(Debian GNU/Linux 9) ( SMP x86_64 GNU/Linux) Python 2.7.13 Content of requirements.txt file ------------------------------------------ apache-beam==2.13.0 apache-beam[gcp] google-cloud-core==1.0.3 google-cloud-iam google-cloud-bigquery==1.6.1 PyYAML==3.13 six==1.12.0 pyOpenSSL httplib2==0.12.0 google-apitools google-api-python-client google-cloud-storage==1.17.0
Description
Python Dataflow runner failed with folrowing error
-----------------------------------------------------------------------------------------------------------------------
p.run().wait_until_finish()
File "/home/marappan/sanmgcppy2env/local/lib/python2.7/site-packages/apache_beam/pipeline.py", line 416, in run
pickler.dump_session(os.path.join(tmpdir, 'main_session.pickle'))
File "/home/marappan/sanmgcppy2env/local/lib/python2.7/site-packages/apache_beam/internal/pickler.py", line 274, in dump_session
dill.dump_session(file_path)
File "/home/marappan/sanmgcppy2env/local/lib/python2.7/site-packages/dill/_dill.py", line 393, in dump_session
pickler.dump(main)
File "/usr/lib/python2.7/pickle.py", line 224, in dump
self.save(obj)
File "/usr/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/home/marappan/sanmgcppy2env/local/lib/python2.7/site-packages/apache_beam/internal/pickler.py", line 149, in save_module
return old_save_module(pickler, obj)
File "/home/marappan/sanmgcppy2env/local/lib/python2.7/site-packages/dill/_dill.py", line 1269, in save_module
state=_main_dict)
File "/usr/lib/python2.7/pickle.py", line 425, in save_reduce
save(state)
File "/usr/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/home/marappan/sanmgcppy2env/local/lib/python2.7/site-packages/apache_beam/internal/pickler.py", line 198, in new_save_module_dict
return old_save_module_dict(pickler, obj)
File "/home/marappan/sanmgcppy2env/local/lib/python2.7/site-packages/dill/_dill.py", line 902, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "/usr/lib/python2.7/pickle.py", line 655, in save_dict
self._batch_setitems(obj.iteritems())
File "/usr/lib/python2.7/pickle.py", line 687, in _batch_setitems
save(v)
File "/usr/lib/python2.7/pickle.py", line 331, in save
self.save_reduce(obj=obj, *rv)
File "/usr/lib/python2.7/pickle.py", line 425, in save_reduce
save(state)
File "/usr/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/home/marappan/sanmgcppy2env/local/lib/python2.7/site-packages/apache_beam/internal/pickler.py", line 198, in new_save_module_dict
return old_save_module_dict(pickler, obj)
File "/home/marappan/sanmgcppy2env/local/lib/python2.7/site-packages/dill/_dill.py", line 902, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "/usr/lib/python2.7/pickle.py", line 655, in save_dict
self._batch_setitems(obj.iteritems())
File "/usr/lib/python2.7/pickle.py", line 687, in _batch_setitems
save(v)
File "/usr/lib/python2.7/pickle.py", line 331, in save
self.save_reduce(obj=obj, *rv)
File "/usr/lib/python2.7/pickle.py", line 425, in save_reduce
save(state)
File "/usr/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/home/marappan/sanmgcppy2env/local/lib/python2.7/site-packages/apache_beam/internal/pickler.py", line 198, in new_save_module_dict
return old_save_module_dict(pickler, obj)
File "/home/marappan/sanmgcppy2env/local/lib/python2.7/site-packages/dill/_dill.py", line 902, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "/usr/lib/python2.7/pickle.py", line 655, in save_dict
self._batch_setitems(obj.iteritems())
File "/usr/lib/python2.7/pickle.py", line 687, in _batch_setitems
save(v)
File "/usr/lib/python2.7/pickle.py", line 331, in save
self.save_reduce(obj=obj, *rv)
File "/usr/lib/python2.7/pickle.py", line 425, in save_reduce
save(state)
File "/usr/lib/python2.7/pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "/home/marappan/sanmgcppy2env/local/lib/python2.7/site-packages/apache_beam/internal/pickler.py", line 198, in new_save_module_dict
return old_save_module_dict(pickler, obj)
File "/usr/lib/python2.7/pickle.py", line 655, in save_dict
self._batch_setitems(obj.iteritems())
File "/usr/lib/python2.7/pickle.py", line 687, in _batch_setitems
save(v)
File "/usr/lib/python2.7/pickle.py", line 306, in save
rv = reduce(self.proto)
TypeError: can't pickle _cffi_backend.CDataGCP objects
-----------------------------------------------------------------------------------------------------------------------