From 809ecf4bfd03afeecdba8dfb38f61a0dac1294ee Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 10 May 2023 23:22:50 -0500 Subject: [PATCH 1/7] Print error type to give more context ``` poetry run python -m synapse.app.homeserver -c homeserver.yaml ********************************************************************************** Error during initialisation: connection to server at "localhost" (::1), port 5432 failed: Connection refused Is the server running on that host and accepting TCP/IP connections? connection to server at "localhost" (127.0.0.1), port 5432 failed: Connection refused Is the server running on that host and accepting TCP/IP connections? There may be more information in the logs. ********************************************************************************** ``` ``` poetry run python -m synapse.app.homeserver -c homeserver.yaml ********************************************************************************** Error during initialisation: psycopg2.OperationalError: connection to server at "localhost" (::1), port 5432 failed: Connection refused Is the server running on that host and accepting TCP/IP connections? connection to server at "localhost" (127.0.0.1), port 5432 failed: Connection refused Is the server running on that host and accepting TCP/IP connections? There may be more information in the logs. ********************************************************************************** ``` ``` 2023-05-10 22:53:56,000 - synapse.app._base - 214 - ERROR - main - Exception during startup Traceback (most recent call last): File "/home/eric/Documents/github/element/synapse/synapse/app/homeserver.py", line 352, in setup hs.setup() File "/home/eric/Documents/github/element/synapse/synapse/server.py", line 337, in setup self.datastores = Databases(self.DATASTORE_CLASS, self) File "/home/eric/Documents/github/element/synapse/synapse/storage/databases/__init__.py", line 65, in __init__ with make_conn(database_config, engine, "startup") as db_conn: File "/home/eric/Documents/github/element/synapse/synapse/storage/database.py", line 161, in make_conn native_db_conn = engine.module.connect(**db_params) File "/home/eric/.cache/pypoetry/virtualenvs/matrix-synapse-xCtC9ulO-py3.10/lib/python3.10/site-packages/psycopg2/__init__.py", line 122, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: connection to server at "localhost" (::1), port 5432 failed: Connection refused Is the server running on that host and accepting TCP/IP connections? connection to server at "localhost" (127.0.0.1), port 5432 failed: Connection refused Is the server running on that host and accepting TCP/IP connections? ``` --- synapse/app/_base.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 7f83b34d89cb..e4ee75a32d33 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -208,12 +208,22 @@ def quit_with_error(error_string: str) -> NoReturn: sys.exit(1) +# Get the full class name like `psycopg2.OperationalError` instead of just +# `OperationalError` which a naive `type(e).__name__` would give. +# via https://stackoverflow.com/a/58045927/796832 +def get_full_class_name(obj) -> str: + module = obj.__class__.__module__ + if module is None or module == str.__class__.__module__: + return obj.__class__.__name__ + return module + "." + obj.__class__.__name__ + + def handle_startup_exception(e: Exception) -> NoReturn: # Exceptions that occur between setting up the logging and forking or starting # the reactor are written to the logs, followed by a summary to stderr. logger.exception("Exception during startup") quit_with_error( - f"Error during initialisation:\n {e}\nThere may be more information in the logs." + f"Error during initialisation:\n {get_full_class_name(e)}: {e}\nThere may be more information in the logs." ) From bb8b30c758793ae9c19347c70593c7ca8a2cc28b Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 10 May 2023 23:39:42 -0500 Subject: [PATCH 2/7] Print full startup exception --- synapse/app/_base.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index e4ee75a32d33..3eb400c4fa4e 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -19,6 +19,7 @@ import signal import socket import sys +from textwrap import indent import traceback import warnings from typing import ( @@ -208,22 +209,28 @@ def quit_with_error(error_string: str) -> NoReturn: sys.exit(1) -# Get the full class name like `psycopg2.OperationalError` instead of just -# `OperationalError` which a naive `type(e).__name__` would give. -# via https://stackoverflow.com/a/58045927/796832 -def get_full_class_name(obj) -> str: - module = obj.__class__.__module__ - if module is None or module == str.__class__.__module__: - return obj.__class__.__name__ - return module + "." + obj.__class__.__name__ +# via https://peps.python.org/pep-3134/#enhanced-reporting +def format_exception_chain(exc: Exception) -> str: + if exc.__cause__: + result = format_exception_chain(exc.__cause__) + result += "\nThe above exception was the direct cause..." + elif exc.__context__: + result = format_exception_chain(exc.__context__) + result += "\nDuring handling of the above exception, ..." + + return "".join(traceback.format_exception(exc)) def handle_startup_exception(e: Exception) -> NoReturn: # Exceptions that occur between setting up the logging and forking or starting # the reactor are written to the logs, followed by a summary to stderr. logger.exception("Exception during startup") + + error_string = format_exception_chain(e) + indendeted_error_string = indent(error_string, " ") + quit_with_error( - f"Error during initialisation:\n {get_full_class_name(e)}: {e}\nThere may be more information in the logs." + f"Error during initialisation:\n{indendeted_error_string}\nThere may be more information in the logs." ) From d524803da2c22e3a4b6c6140dd877d79ae6dadb9 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 10 May 2023 23:47:33 -0500 Subject: [PATCH 3/7] Add changelog --- changelog.d/15569.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/15569.feature diff --git a/changelog.d/15569.feature b/changelog.d/15569.feature new file mode 100644 index 000000000000..b58af8ad5513 --- /dev/null +++ b/changelog.d/15569.feature @@ -0,0 +1 @@ +Print full error and stack-trace of any exception that occurs during startup/initialization. From 47ad98761e36666362e73d17ff7030f6262aab4e Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 10 May 2023 23:49:32 -0500 Subject: [PATCH 4/7] Fix lint --- synapse/app/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 3eb400c4fa4e..f888114453cc 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -19,9 +19,9 @@ import signal import socket import sys -from textwrap import indent import traceback import warnings +from textwrap import indent from typing import ( TYPE_CHECKING, Any, From 0642ee183a02579a67b607321cbfc017356af3e6 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 11 May 2023 00:19:06 -0500 Subject: [PATCH 5/7] Fix type --- synapse/app/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index f888114453cc..4a25780b3fb5 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -210,7 +210,7 @@ def quit_with_error(error_string: str) -> NoReturn: # via https://peps.python.org/pep-3134/#enhanced-reporting -def format_exception_chain(exc: Exception) -> str: +def format_exception_chain(exc: BaseException) -> str: if exc.__cause__: result = format_exception_chain(exc.__cause__) result += "\nThe above exception was the direct cause..." From fcad2963b488dd03b5f4a5b2dcb7e85b41bfa53e Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 11 May 2023 10:10:18 -0500 Subject: [PATCH 6/7] All we need is traceback.format_exception --- synapse/app/_base.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 4a25780b3fb5..c3df3ca784cf 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -209,24 +209,12 @@ def quit_with_error(error_string: str) -> NoReturn: sys.exit(1) -# via https://peps.python.org/pep-3134/#enhanced-reporting -def format_exception_chain(exc: BaseException) -> str: - if exc.__cause__: - result = format_exception_chain(exc.__cause__) - result += "\nThe above exception was the direct cause..." - elif exc.__context__: - result = format_exception_chain(exc.__context__) - result += "\nDuring handling of the above exception, ..." - - return "".join(traceback.format_exception(exc)) - - def handle_startup_exception(e: Exception) -> NoReturn: # Exceptions that occur between setting up the logging and forking or starting # the reactor are written to the logs, followed by a summary to stderr. logger.exception("Exception during startup") - error_string = format_exception_chain(e) + error_string = "".join(traceback.format_exception(e)) indendeted_error_string = indent(error_string, " ") quit_with_error( From 08554a7c9629001edfb0df79138baf3f4841960f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 11 May 2023 10:12:19 -0500 Subject: [PATCH 7/7] Fix typo --- synapse/app/_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index c3df3ca784cf..4dfcf484faf3 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -215,10 +215,10 @@ def handle_startup_exception(e: Exception) -> NoReturn: logger.exception("Exception during startup") error_string = "".join(traceback.format_exception(e)) - indendeted_error_string = indent(error_string, " ") + indented_error_string = indent(error_string, " ") quit_with_error( - f"Error during initialisation:\n{indendeted_error_string}\nThere may be more information in the logs." + f"Error during initialisation:\n{indented_error_string}\nThere may be more information in the logs." )