Created
April 14, 2020 05:49
-
-
Save smly/ec26cff6dec17568d97006a573dda5f8 to your computer and use it in GitHub Desktop.
optgbm_multinode_rdb_error.log
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [I 2020-04-14 14:44:15,580] Using an existing study with name 'digits_optgbm' instead of creating a new one. | |
| [I 2020-04-14 14:44:15,596] Searching the best hyperparameters... | |
| [LightGBM] [Warning] feature_fraction is set=0.1, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.1 | |
| [LightGBM] [Warning] bagging_freq is set=10, subsample_freq=0 will be ignored. Current value: bagging_freq=10 | |
| [LightGBM] [Warning] lambda_l1 is set=6.0670460192270264e-05, reg_alpha=0.0 will be ignored. Current value: lambda_l1=6.0670460192270264e-05 | |
| [LightGBM] [Warning] lambda_l2 is set=0.06434253899747878, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.06434253899747878 | |
| [LightGBM] [Warning] bagging_fraction is set=0.7, subsample=1.0 will be ignored. Current value: bagging_fraction=0.7 | |
| [LightGBM] [Warning] min_data_in_leaf is set=64, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=64 | |
| [W 2020-04-14 14:44:17,804] Setting status of trial#7 as TrialState.FAIL because of the following error: DataError('(pymysql.err.DataError) (1406, "Data too long for column \'value_json\' at row 1")') | |
| Traceback (most recent call last): | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1244, in _execute_context | |
| cursor, statement, parameters, context | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 552, in do_execute | |
| cursor.execute(statement, parameters) | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/cursors.py", line 170, in execute | |
| result = self._query(query) | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/cursors.py", line 328, in _query | |
| conn.query(q) | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/connections.py", line 517, in query | |
| self._affected_rows = self._read_query_result(unbuffered=unbuffered) | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/connections.py", line 732, in _read_query_result | |
| result.read() | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/connections.py", line 1075, in read | |
| first_packet = self.connection._read_packet() | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/connections.py", line 684, in _read_packet | |
| packet.check_error() | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/protocol.py", line 220, in check_error | |
| err.raise_mysql_exception(self._data) | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/err.py", line 109, in raise_mysql_exception | |
| raise errorclass(errno, errval) | |
| pymysql.err.DataError: (1406, "Data too long for column 'value_json' at row 1") | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/home/smly/gitws/optuna/optuna/study.py", line 677, in _run_trial | |
| result = func(trial) | |
| File "/home/smly/gitws/optuna/optuna/integration/lightgbm_tuner/sklearn.py", line 318, in __call__ | |
| trial.study.set_user_attr("representations", representations) | |
| File "/home/smly/gitws/optuna/optuna/study.py", line 389, in set_user_attr | |
| self._storage.set_study_user_attr(self._study_id, key, value) | |
| File "/home/smly/gitws/optuna/optuna/storages/rdb/storage.py", line 226, in set_study_user_attr | |
| self._commit_with_integrity_check(session) | |
| File "/home/smly/gitws/optuna/optuna/storages/rdb/storage.py", line 875, in _commit_with_integrity_check | |
| session.commit() | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 1026, in commit | |
| self.transaction.commit() | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 493, in commit | |
| self._prepare_impl() | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 472, in _prepare_impl | |
| self.session.flush() | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 2451, in flush | |
| self._flush(objects) | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 2589, in _flush | |
| transaction.rollback(_capture_exception=True) | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/util/langhelpers.py", line 68, in __exit__ | |
| compat.reraise(exc_type, exc_value, exc_tb) | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 129, in reraise | |
| raise value | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 2549, in _flush | |
| flush_context.execute() | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/unitofwork.py", line 422, in execute | |
| rec.execute(self) | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/unitofwork.py", line 589, in execute | |
| uow, | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/persistence.py", line 245, in save_obj | |
| insert, | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/persistence.py", line 1120, in _emit_insert_statements | |
| statement, params | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 988, in execute | |
| return meth(self, multiparams, params) | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/sql/elements.py", line 287, in _execute_on_connection | |
| return connection._execute_clauseelement(self, multiparams, params) | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1107, in _execute_clauseelement | |
| distilled_params, | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1248, in _execute_context | |
| e, statement, parameters, cursor, context | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1466, in _handle_dbapi_exception | |
| util.raise_from_cause(sqlalchemy_exception, exc_info) | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 383, in raise_from_cause | |
| reraise(type(exception), exception, tb=exc_tb, cause=cause) | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 128, in reraise | |
| raise value.with_traceback(tb) | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1244, in _execute_context | |
| cursor, statement, parameters, context | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 552, in do_execute | |
| cursor.execute(statement, parameters) | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/cursors.py", line 170, in execute | |
| result = self._query(query) | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/cursors.py", line 328, in _query | |
| conn.query(q) | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/connections.py", line 517, in query | |
| self._affected_rows = self._read_query_result(unbuffered=unbuffered) | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/connections.py", line 732, in _read_query_result | |
| result.read() | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/connections.py", line 1075, in read | |
| first_packet = self.connection._read_packet() | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/connections.py", line 684, in _read_packet | |
| packet.check_error() | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/protocol.py", line 220, in check_error | |
| err.raise_mysql_exception(self._data) | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/err.py", line 109, in raise_mysql_exception | |
| raise errorclass(errno, errval) | |
| sqlalchemy.exc.DataError: (pymysql.err.DataError) (1406, "Data too long for column 'value_json' at row 1") | |
| [SQL: INSERT INTO study_user_attributes (study_id, `key`, value_json) VALUES (%(study_id)s, %(key)s, %(value_json)s)] | |
| [parameters: {'study_id': 5, 'key': 'representations', 'value_json': '["tree\\nversion=v2\\nnum_class=10\\nnum_tree_per_iteration=10\\nlabel_index=0\\nmax_feature_idx=63\\nobjective=multiclass num_class:10\\nfeature_nam ... (4070901 characters truncated) ... t_filename: ]\\n[machines: ]\\n[gpu_platform_id: -1]\\n[gpu_device_id: -1]\\n[gpu_use_dp: 0]\\n\\nend of parameters\\n\\npandas_categorical:null\\n"]'}] | |
| (Background on this error at: http://sqlalche.me/e/9h9h) | |
| Traceback (most recent call last): | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1244, in _execute_context | |
| cursor, statement, parameters, context | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 552, in do_execute | |
| cursor.execute(statement, parameters) | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/cursors.py", line 170, in execute | |
| result = self._query(query) | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/cursors.py", line 328, in _query | |
| conn.query(q) | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/connections.py", line 517, in query | |
| self._affected_rows = self._read_query_result(unbuffered=unbuffered) | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/connections.py", line 732, in _read_query_result | |
| result.read() | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/connections.py", line 1075, in read | |
| first_packet = self.connection._read_packet() | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/connections.py", line 684, in _read_packet | |
| packet.check_error() | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/protocol.py", line 220, in check_error | |
| err.raise_mysql_exception(self._data) | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/err.py", line 109, in raise_mysql_exception | |
| raise errorclass(errno, errval) | |
| pymysql.err.DataError: (1406, "Data too long for column 'value_json' at row 1") | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "/home/smly/gitws/optuna/optuna/study.py", line 677, in _run_trial | |
| result = func(trial) | |
| File "/home/smly/gitws/optuna/optuna/integration/lightgbm_tuner/sklearn.py", line 318, in __call__ | |
| trial.study.set_user_attr("representations", representations) | |
| File "/home/smly/gitws/optuna/optuna/study.py", line 389, in set_user_attr | |
| self._storage.set_study_user_attr(self._study_id, key, value) | |
| File "/home/smly/gitws/optuna/optuna/storages/rdb/storage.py", line 226, in set_study_user_attr | |
| self._commit_with_integrity_check(session) | |
| File "/home/smly/gitws/optuna/optuna/storages/rdb/storage.py", line 875, in _commit_with_integrity_check | |
| session.commit() | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 1026, in commit | |
| self.transaction.commit() | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 493, in commit | |
| self._prepare_impl() | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 472, in _prepare_impl | |
| self.session.flush() | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 2451, in flush | |
| self._flush(objects) | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 2589, in _flush | |
| transaction.rollback(_capture_exception=True) | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/util/langhelpers.py", line 68, in __exit__ | |
| compat.reraise(exc_type, exc_value, exc_tb) | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 129, in reraise | |
| raise value | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 2549, in _flush | |
| flush_context.execute() | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/unitofwork.py", line 422, in execute | |
| rec.execute(self) | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/unitofwork.py", line 589, in execute | |
| uow, | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/persistence.py", line 245, in save_obj | |
| insert, | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/persistence.py", line 1120, in _emit_insert_statements | |
| statement, params | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 988, in execute | |
| return meth(self, multiparams, params) | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/sql/elements.py", line 287, in _execute_on_connection | |
| return connection._execute_clauseelement(self, multiparams, params) | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1107, in _execute_clauseelement | |
| distilled_params, | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1248, in _execute_context | |
| e, statement, parameters, cursor, context | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1466, in _handle_dbapi_exception | |
| util.raise_from_cause(sqlalchemy_exception, exc_info) | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 383, in raise_from_cause | |
| reraise(type(exception), exception, tb=exc_tb, cause=cause) | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 128, in reraise | |
| raise value.with_traceback(tb) | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1244, in _execute_context | |
| cursor, statement, parameters, context | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 552, in do_execute | |
| cursor.execute(statement, parameters) | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/cursors.py", line 170, in execute | |
| result = self._query(query) | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/cursors.py", line 328, in _query | |
| conn.query(q) | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/connections.py", line 517, in query | |
| self._affected_rows = self._read_query_result(unbuffered=unbuffered) | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/connections.py", line 732, in _read_query_result | |
| result.read() | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/connections.py", line 1075, in read | |
| first_packet = self.connection._read_packet() | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/connections.py", line 684, in _read_packet | |
| packet.check_error() | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/protocol.py", line 220, in check_error | |
| err.raise_mysql_exception(self._data) | |
| File "/home/smly/a/lib/python3.7/site-packages/pymysql/err.py", line 109, in raise_mysql_exception | |
| raise errorclass(errno, errval) | |
| sqlalchemy.exc.DataError: (pymysql.err.DataError) (1406, "Data too long for column 'value_json' at row 1") | |
| [SQL: INSERT INTO study_user_attributes (study_id, `key`, value_json) VALUES (%(study_id)s, %(key)s, %(value_json)s)] | |
| [parameters: {'study_id': 5, 'key': 'representations', 'value_json': '["tree\\nversion=v2\\nnum_class=10\\nnum_tree_per_iteration=10\\nlabel_index=0\\nmax_feature_idx=63\\nobjective=multiclass num_class:10\\nfeature_nam ... (4070901 characters truncated) ... t_filename: ]\\n[machines: ]\\n[gpu_platform_id: -1]\\n[gpu_device_id: -1]\\n[gpu_use_dp: 0]\\n\\nend of parameters\\n\\npandas_categorical:null\\n"]'}] | |
| (Background on this error at: http://sqlalche.me/e/9h9h) | |
| During handling of the above exception, another exception occurred: | |
| Traceback (most recent call last): | |
| File "opt1.py", line 23, in <module> | |
| main() | |
| File "opt1.py", line 19, in main | |
| clf.fit(X, y, early_stopping_rounds=None) | |
| File "/home/smly/gitws/optuna/optuna/integration/lightgbm_tuner/sklearn.py", line 1013, in fit | |
| **fit_params | |
| File "/home/smly/gitws/optuna/optuna/integration/lightgbm_tuner/sklearn.py", line 712, in fit | |
| self.study_.optimize(objective, catch=(), n_trials=self.n_trials, timeout=self.timeout) | |
| File "/home/smly/gitws/optuna/optuna/study.py", line 331, in optimize | |
| func, n_trials, timeout, catch, callbacks, gc_after_trial, None | |
| File "/home/smly/gitws/optuna/optuna/study.py", line 626, in _optimize_sequential | |
| self._run_trial_and_callbacks(func, catch, callbacks, gc_after_trial) | |
| File "/home/smly/gitws/optuna/optuna/study.py", line 656, in _run_trial_and_callbacks | |
| trial = self._run_trial(func, catch, gc_after_trial) | |
| File "/home/smly/gitws/optuna/optuna/study.py", line 699, in _run_trial | |
| self._storage.set_trial_system_attr(trial_id, "fail_reason", message) | |
| File "/home/smly/gitws/optuna/optuna/storages/rdb/storage.py", line 642, in set_trial_system_attr | |
| self._set_trial_system_attr_without_commit(session, trial_id, key, value) | |
| File "/home/smly/gitws/optuna/optuna/storages/rdb/storage.py", line 649, in _set_trial_system_attr_without_commit | |
| trial = models.TrialModel.find_or_raise_by_id(trial_id, session) | |
| File "/home/smly/gitws/optuna/optuna/storages/rdb/models.py", line 220, in find_or_raise_by_id | |
| trial = cls.find_by_id(trial_id, session) | |
| File "/home/smly/gitws/optuna/optuna/storages/rdb/models.py", line 182, in find_by_id | |
| return query.one_or_none() | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line 3244, in one_or_none | |
| ret = list(self) | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line 3317, in __iter__ | |
| return self._execute_and_instances(context) | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line 3339, in _execute_and_instances | |
| querycontext, self._connection_from_session, close_with_result=True | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line 3354, in _get_bind_args | |
| mapper=self._bind_mapper(), clause=querycontext.statement, **kw | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line 3332, in _connection_from_session | |
| conn = self.session.connection(**kw) | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 1123, in connection | |
| execution_options=execution_options, | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 1129, in _connection_for_bind | |
| engine, execution_options | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 407, in _connection_for_bind | |
| self._assert_active() | |
| File "/home/smly/a/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 294, in _assert_active | |
| % self._rollback_exception | |
| sqlalchemy.exc.InvalidRequestError: This Session's transaction has been rolled back due to a previous exception during flush. To begin a new transaction with this Session, first issue Session.rollback(). Original exception was: (pymysql.err.DataError) (1406, "Data too long for column 'value_json' at row 1") | |
| [SQL: INSERT INTO study_user_attributes (study_id, `key`, value_json) VALUES (%(study_id)s, %(key)s, %(value_json)s)] | |
| [parameters: {'study_id': 5, 'key': 'representations', 'value_json': '["tree\\nversion=v2\\nnum_class=10\\nnum_tree_per_iteration=10\\nlabel_index=0\\nmax_feature_idx=63\\nobjective=multiclass num_class:10\\nfeature_nam ... (4070901 characters truncated) ... t_filename: ]\\n[machines: ]\\n[gpu_platform_id: -1]\\n[gpu_device_id: -1]\\n[gpu_use_dp: 0]\\n\\nend of parameters\\n\\npandas_categorical:null\\n"]'}] | |
| (Background on this error at: http://sqlalche.me/e/9h9h) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| CREATE DATABASE optuna_db; | |
| CREATE USER 'optuna_user'@'localhost' IDENTIFIED BY 'p4rW!w8@AUfJ5TkA'; | |
| GRANT CREATE,INSERT,UPDATE,SELECT ON optuna_db.* TO 'optuna_user'@'localhost'; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import optuna | |
| import optuna.integration.lightgbm as lgb | |
| from sklearn.datasets import load_digits | |
| def main(): | |
| X, y = load_digits(return_X_y=True) | |
| study = optuna.create_study( | |
| direction="minimize", | |
| study_name="digits_optgbm", | |
| storage=( | |
| "mysql+pymysql://optuna_user:" | |
| "p4rW%21w8%40AUfJ5TkA@localhost/optuna_db"), | |
| load_if_exists=True, | |
| ) | |
| clf = lgb.LGBMClassifier(n_estimators=100, random_state=0, study=study) | |
| clf.fit(X, y, early_stopping_rounds=None) | |
| if __name__ == "__main__": | |
| main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment