Created
April 19, 2021 22:04
-
-
Save almahmoud/800a57a8b002bb409fba67f4215c85e2 to your computer and use it in GitHub Desktop.
galaxy-db-race-condition
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| (Background on this error at: http://sqlalche.me/e/13/e3q8) | |
| galaxy.workflow.scheduling_manager ERROR 2021-04-19 22:00:19,000 [pN:job_handler_0,p:7,tN:WorkflowRequestMonitor.monitor_thread] An exception occured scheduling while scheduling workflows | |
| Traceback (most recent call last): | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1276, in _execute_context | |
| self.dialect.do_execute( | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 608, in do_execute | |
| cursor.execute(statement, parameters) | |
| psycopg2.errors.ReadOnlySqlTransaction: cannot execute UPDATE in a read-only transaction | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/galaxy/server/lib/galaxy/workflow/scheduling_manager.py", line 290, in __monitor | |
| self.invocation_grabber.grab_unhandled_items() | |
| File "/galaxy/server/lib/galaxy/jobs/handler.py", line 137, in grab_unhandled_items | |
| proxy = conn.execute(self._grab_query) | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1011, in execute | |
| return meth(self, multiparams, params) | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection | |
| return connection._execute_clauseelement(self, multiparams, params) | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1124, in _execute_clauseelement | |
| ret = self._execute_context( | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1316, in _execute_context | |
| self._handle_dbapi_exception( | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1510, in _handle_dbapi_exception | |
| util.raise_( | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 182, in raise_ | |
| raise exception | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1276, in _execute_context | |
| self.dialect.do_execute( | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 608, in do_execute | |
| cursor.execute(statement, parameters) | |
| sqlalchemy.exc.InternalError: (psycopg2.errors.ReadOnlySqlTransaction) cannot execute UPDATE in a read-only transaction | |
| [SQL: UPDATE workflow_invocation SET update_time=%(update_time)s, handler=%(handler)s WHERE workflow_invocation.id IN (SELECT workflow_invocation.id | |
| FROM workflow_invocation | |
| WHERE workflow_invocation.handler IN (%(handler_1)s, %(handler_2)s) AND workflow_invocation.state = %(state_1)s ORDER BY workflow_invocation.id FOR UPDATE SKIP LOCKED) RETURNING workflow_invocation.id] | |
| [parameters: {'update_time': datetime.datetime(2021, 4, 19, 22, 0, 18, 835538), 'handler': 'job_handler_0', 'handler_1': 'schedulers', 'handler_2': '_default_', 'state_1': 'new'}] | |
| (Background on this error at: http://sqlalche.me/e/13/2j85) | |
| galaxy.jobs.handler ERROR 2021-04-19 22:00:19,010 [pN:job_handler_0,p:7,tN:JobHandlerQueue.monitor_thread] Exception in monitor_step | |
| Traceback (most recent call last): | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1276, in _execute_context | |
| self.dialect.do_execute( | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 608, in do_execute | |
| cursor.execute(statement, parameters) | |
| psycopg2.errors.ReadOnlySqlTransaction: cannot execute UPDATE in a read-only transaction | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/galaxy/server/lib/galaxy/jobs/handler.py", line 305, in __monitor | |
| self.__monitor_step() | |
| File "/galaxy/server/lib/galaxy/jobs/handler.py", line 322, in __monitor_step | |
| self.job_grabber.grab_unhandled_items() | |
| File "/galaxy/server/lib/galaxy/jobs/handler.py", line 137, in grab_unhandled_items | |
| proxy = conn.execute(self._grab_query) | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1011, in execute | |
| return meth(self, multiparams, params) | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection | |
| return connection._execute_clauseelement(self, multiparams, params) | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1124, in _execute_clauseelement | |
| ret = self._execute_context( | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1316, in _execute_context | |
| self._handle_dbapi_exception( | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1510, in _handle_dbapi_exception | |
| util.raise_( | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 182, in raise_ | |
| raise exception | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1276, in _execute_context | |
| self.dialect.do_execute( | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 608, in do_execute | |
| cursor.execute(statement, parameters) | |
| sqlalchemy.exc.InternalError: (psycopg2.errors.ReadOnlySqlTransaction) cannot execute UPDATE in a read-only transaction | |
| [SQL: UPDATE job SET update_time=%(update_time)s, handler=%(handler)s WHERE job.id IN (SELECT job.id | |
| FROM job | |
| WHERE job.handler IN (%(handler_1)s) AND job.state = %(state_1)s ORDER BY job.id FOR UPDATE SKIP LOCKED) RETURNING job.id] | |
| [parameters: {'update_time': datetime.datetime(2021, 4, 19, 22, 0, 18, 880322), 'handler': 'job_handler_0', 'handler_1': '_default_', 'state_1': <states.NEW: 'new'>}] | |
| (Background on this error at: http://sqlalche.me/e/13/2j85) | |
| Exception in thread Thread-1: | |
| Traceback (most recent call last): | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1276, in _execute_context | |
| self.dialect.do_execute( | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 608, in do_execute | |
| cursor.execute(statement, parameters) | |
| psycopg2.errors.ReadOnlySqlTransaction: cannot execute SELECT FOR UPDATE in a read-only transaction | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner | |
| self.run() | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/kombu/mixins.py", line 172, in run | |
| for _ in self.consume(limit=None, **kwargs): | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/kombu/mixins.py", line 194, in consume | |
| conn.drain_events(timeout=safety_interval) | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/kombu/connection.py", line 318, in drain_events | |
| return self.transport.drain_events(self.connection, **kwargs) | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/kombu/transport/virtual/base.py", line 960, in drain_events | |
| get(self._deliver, timeout=timeout) | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/kombu/utils/scheduling.py", line 52, in get | |
| return self.fun(resource, callback, **kwargs) | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/kombu/transport/virtual/base.py", line 998, in _drain_channel | |
| return channel.drain_events(callback=callback, timeout=timeout) | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/kombu/transport/virtual/base.py", line 742, in drain_events | |
| return self._poll(self.cycle, callback, timeout=timeout) | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/kombu/transport/virtual/base.py", line 399, in _poll | |
| return cycle.get(callback) | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/kombu/utils/scheduling.py", line 52, in get | |
| return self.fun(resource, callback, **kwargs) | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/kombu/transport/virtual/base.py", line 402, in _get_and_deliver | |
| message = self._get(queue) | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/kombu/transport/sqlalchemy/__init__.py", line 116, in _get | |
| msg = self.session.query(self.message_cls) \ | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3429, in first | |
| ret = list(self[0:1]) | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3203, in __getitem__ | |
| return list(res) | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3535, in __iter__ | |
| return self._execute_and_instances(context) | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3560, in _execute_and_instances | |
| result = conn.execute(querycontext.statement, self._params) | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1011, in execute | |
| return meth(self, multiparams, params) | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection | |
| return connection._execute_clauseelement(self, multiparams, params) | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1124, in _execute_clauseelement | |
| ret = self._execute_context( | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1316, in _execute_context | |
| self._handle_dbapi_exception( | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1510, in _handle_dbapi_exception | |
| util.raise_( | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 182, in raise_ | |
| raise exception | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1276, in _execute_context | |
| self.dialect.do_execute( | |
| File "/galaxy/server/.venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 608, in do_execute | |
| cursor.execute(statement, parameters) | |
| sqlalchemy.exc.InternalError: (psycopg2.errors.ReadOnlySqlTransaction) cannot execute SELECT FOR UPDATE in a read-only transaction | |
| [SQL: SELECT kombu_message.timestamp AS kombu_message_timestamp, kombu_message.id AS kombu_message_id, kombu_message.visible AS kombu_message_visible, kombu_message.payload AS kombu_message_payload, kombu_message.version AS kombu_message_version, kombu_message.queue_id AS kombu_message_queue_id | |
| FROM kombu_message | |
| WHERE kombu_message.queue_id = %(queue_id_1)s AND kombu_message.visible != false ORDER BY kombu_message.timestamp, kombu_message.id | |
| LIMIT %(param_1)s FOR UPDATE] | |
| [parameters: {'queue_id_1': 92, 'param_1': 1}] |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment