Error Running Apache Beam with python via Dataflow runner - apache-beam

Question

Description
I am currently implementing a batch job with dataflow using apache beam, this works fine when i use direct runner. Changing this throws the below exception, not sure where the error is comming from.
Code Snippet
enter image description here
Error:
raceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/beam_mysql/connector/client.py", line 179, in enter
self.conn = mysql.connector.connect(**self._config)
File "/usr/local/lib/python3.9/site-packages/mysql/connector/pooling.py", line 286, in connect
return CMySQLConnection(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/mysql/connector/connection_cext.py", line 101, in init
self.connect(**kwargs)
File "/usr/local/lib/python3.9/site-packages/mysql/connector/abstracts.py", line 1095, in connect
self._open_connection()
File "/usr/local/lib/python3.9/site-packages/mysql/connector/connection_cext.py", line 268, in _open_connection
raise get_mysql_exception(
mysql.connector.errors.DatabaseError: 2003 (HY000): Can't connect to MySQL server on '127.0.0.1:41849' (111)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/apache_beam/runners/worker/sdk_worker.py", line 284, in _execute
response = task()
File "/usr/local/lib/python3.9/site-packages/apache_beam/runners/worker/sdk_worker.py", line 357, in
lambda: self.create_worker().do_instruction(request), request)
File "/usr/local/lib/python3.9/site-packages/apache_beam/runners/worker/sdk_worker.py", line 597, in do_instruction
return getattr(self, request_type)(
File "/usr/local/lib/python3.9/site-packages/apache_beam/runners/worker/sdk_worker.py", line 635, in process_bundle
bundle_processor.process_bundle(instruction_id))
File "/usr/local/lib/python3.9/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1003, in process_bundle
input_op_by_transform_id[element.transform_id].process_encoded(
File "/usr/local/lib/python3.9/site-packages/apache_beam/runners/worker/bundle_processor.py", line 227, in process_encoded
self.output(decoded_value)
File "apache_beam/runners/worker/operations.py", line 526, in apache_beam.runners.worker.operations.Operation.output
File "apache_beam/runners/worker/operations.py", line 528, in apache_beam.runners.worker.operations.Operation.output
File "apache_beam/runners/worker/operations.py", line 237, in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/worker/operations.py", line 240, in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/worker/operations.py", line 907, in apache_beam.runners.worker.operations.DoOperation.process
File "apache_beam/runners/worker/operations.py", line 908, in apache_beam.runners.worker.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1419, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1491, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1417, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 623, in apache_beam.runners.common.SimpleInvoker.invoke_process
File "apache_beam/runners/common.py", line 1581, in apache_beam.runners.common._OutputHandler.handle_process_outputs
File "apache_beam/runners/common.py", line 1694, in apache_beam.runners.common._OutputHandler._write_value_to_tag
File "apache_beam/runners/worker/operations.py", line 240, in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/worker/operations.py", line 907, in apache_beam.runners.worker.operations.DoOperation.process
File "apache_beam/runners/worker/operations.py", line 908, in apache_beam.runners.worker.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1419, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1491, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1417, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 623, in apache_beam.runners.common.SimpleInvoker.invoke_process
File "apache_beam/runners/common.py", line 1581, in apache_beam.runners.common._OutputHandler.handle_process_outputs
File "apache_beam/runners/common.py", line 1694, in apache_beam.runners.common._OutputHandler._write_value_to_tag
File "apache_beam/runners/worker/operations.py", line 240, in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/worker/operations.py", line 907, in apache_beam.runners.worker.operations.DoOperation.process
File "apache_beam/runners/worker/operations.py", line 908, in apache_beam.runners.worker.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1419, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1507, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1417, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 623, in apache_beam.runners.common.SimpleInvoker.invoke_process
File "apache_beam/runners/common.py", line 1571, in apache_beam.runners.common._OutputHandler.handle_process_outputs
File "/usr/local/lib/python3.9/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1454, in process
for part, size in self.restriction_provider.split_and_size(
File "/usr/local/lib/python3.9/site-packages/apache_beam/transforms/core.py", line 331, in split_and_size
for part in self.split(element, restriction):
File "/usr/local/lib/python3.9/site-packages/apache_beam/io/iobase.py", line 1641, in split
estimated_size = restriction.source().estimate_size()
File "/usr/local/lib/python3.9/site-packages/beam_mysql/connector/source.py", line 49, in estimate_size
return self._splitter.estimate_size()
File "/usr/local/lib/python3.9/site-packages/beam_mysql/connector/splitters.py", line 48, in estimate_size
return self.source.client.rough_counts_estimator(self.source.query)
File "/usr/local/lib/python3.9/site-packages/beam_mysql/connector/client.py", line 104, in rough_counts_estimator
with _MySQLConnection(self._config) as conn:
File "/usr/local/lib/python3.9/site-packages/beam_mysql/connector/client.py", line 182, in enter
raise MySQLClientError(f"Failed to connect mysql, Raise exception: {e}")
beam_mysql.connector.errors.MySQLClientError: Failed to connect mysql, Raise exception: 2003 (HY000): Can't connect to MySQL server on '127.0.0.1:36247' (111) [while running 'ref_AppliedPTransform_Read-From-Mysql-Read-SDFBoundedSourceReader-ParDo-SDFBoundedSourceDoFn-_7/SplitWithSizing-ptransform-38']

Answer 1

You have a connection problem to MySql :
Your traceback indicates :
mysql.connector.errors.DatabaseError: 2003 (HY000): Can't connect to MySQL server on '127.0.0.1:41849'
https://dev.mysql.com/doc/refman/8.0/en/can-not-connect-to-server.html#:~:text=The%20error%20(2003)%20Can',one%20configured%20on%20the%20server.
Check your connection to your MySql server please and also the parameters passed by your Beam job.