### The problem
HA stops recording long term statistics. After restart of HA …the database file is marked as corrupted and a fresh database is created. All long-term statistics are consquently gone. Short-term statistics work fine and are current.
`PRAGMA schema.integrity_check;` on the old database file returns no errors.
When I go back to `core-2024.1.6` using the old db all long term statistics are back. The database file is not recreated.
### What version of Home Assistant Core has the issue?
core-2024.2.5
### What was the last working version of Home Assistant Core?
core-2024.1.6
### What type of installation are you running?
Home Assistant Container
### Integration causing the issue
_No response_
### Link to integration documentation on our website
_No response_
### Diagnostics information
_No response_
### Example YAML snippet
_No response_
### Anything in the logs that might be useful for us?
```txt
2024-03-05 16:28:22.422 ERROR (Recorder) [homeassistant.components.recorder.util] Error executing query: unsupported operand type(s) for +: 'NoneType' and 'float'
Traceback (most recent call last):
File "/usr/src/homeassistant/homeassistant/components/recorder/util.py", line 139, in session_scope
yield session
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 468, in compile_missing_statistics
modified_statistic_ids = _compile_statistics(
^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 542, in _compile_statistics
compiled: PlatformCompiledStatistics = platform_compile_statistics(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/sensor/recorder.py", line 466, in compile_statistics
last_stats = statistics.get_latest_short_term_statistics_with_session(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 1952, in get_latest_short_term_statistics_with_session
return _sorted_statistics_to_dict(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 2121, in _sorted_statistics_to_dict
"end": start_ts + table_duration_seconds,
~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~
TypeError: unsupported operand type(s) for +: 'NoneType' and 'float'
2024-03-05 16:28:22.439 ERROR (Recorder) [homeassistant.components.recorder.core] Error while processing event CompileMissingStatisticsTask(): unsupported operand type(s) for +: 'NoneType' and 'float'
Traceback (most recent call last):
File "/usr/src/homeassistant/homeassistant/components/recorder/core.py", line 906, in _guarded_process_one_task_or_event_or_recover
self._process_one_task_or_event_or_recover(task)
File "/usr/src/homeassistant/homeassistant/components/recorder/core.py", line 926, in _process_one_task_or_event_or_recover
return task.run(self)
^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/tasks.py", line 176, in run
if statistics.compile_missing_statistics(instance):
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/util.py", line 643, in wrapper
return job(instance, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 468, in compile_missing_statistics
modified_statistic_ids = _compile_statistics(
^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 542, in _compile_statistics
compiled: PlatformCompiledStatistics = platform_compile_statistics(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/sensor/recorder.py", line 466, in compile_statistics
last_stats = statistics.get_latest_short_term_statistics_with_session(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 1952, in get_latest_short_term_statistics_with_session
return _sorted_statistics_to_dict(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 2121, in _sorted_statistics_to_dict
"end": start_ts + table_duration_seconds,
~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~
TypeError: unsupported operand type(s) for +: 'NoneType' and 'float'
2024-03-05 16:28:22.495 ERROR (Recorder) [homeassistant.components.recorder.core] Unrecoverable sqlite3 database corruption detected: (sqlite3.IntegrityError) FOREIGN KEY constraint failed
[SQL: INSERT INTO states (entity_id, state, attributes, event_id, last_changed, last_changed_ts, last_updated, last_updated_ts, old_state_id, attributes_id, context_id, context_user_id, context_parent_id, origin_idx, context_id_bin, context_user_id_bin, context_parent_id_bin, metadata_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING state_id]
[parameters: (None, '25624.646', None, None, None, None, None, 1709652498.664474, 326200127, 135, None, None, None, 0, <memory at 0x7f726fcb80>, None, None, 29)]
(Background on this error at: https://sqlalche.me/e/20/gkpj)
Traceback (most recent call last):
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/engine/base.py", line 2113, in _exec_insertmany_context
dialect.do_execute(
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/engine/default.py", line 922, in do_execute
cursor.execute(statement, parameters)
sqlite3.IntegrityError: FOREIGN KEY constraint failed
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/src/homeassistant/homeassistant/components/recorder/core.py", line 925, in _process_one_task_or_event_or_recover
self._commit_event_session_or_retry()
File "/usr/src/homeassistant/homeassistant/components/recorder/core.py", line 1182, in _commit_event_session_or_retry
self._commit_event_session()
File "/usr/src/homeassistant/homeassistant/components/recorder/core.py", line 1202, in _commit_event_session
session.commit()
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/orm/session.py", line 1969, in commit
trans.commit(_to_root=True)
File "<string>", line 2, in commit
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/orm/state_changes.py", line 139, in _go
ret_value = fn(self, *arg, **kw)
^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/orm/session.py", line 1256, in commit
self._prepare_impl()
File "<string>", line 2, in _prepare_impl
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/orm/state_changes.py", line 139, in _go
ret_value = fn(self, *arg, **kw)
^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/orm/session.py", line 1231, in _prepare_impl
self.session.flush()
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/orm/session.py", line 4312, in flush
self._flush(objects)
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/orm/session.py", line 4447, in _flush
with util.safe_reraise():
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/util/langhelpers.py", line 146, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/orm/session.py", line 4408, in _flush
flush_context.execute()
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/orm/unitofwork.py", line 463, in execute
n.execute_aggregate(self, set_)
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/orm/unitofwork.py", line 759, in execute_aggregate
persistence.save_obj(
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/orm/persistence.py", line 93, in save_obj
_emit_insert_statements(
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/orm/persistence.py", line 1137, in _emit_insert_statements
result = connection.execute(
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/engine/base.py", line 1416, in execute
return meth(
^^^^^
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/sql/elements.py", line 517, in _execute_on_connection
return connection._execute_clauseelement(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/engine/base.py", line 1639, in _execute_clauseelement
ret = self._execute_context(
^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/engine/base.py", line 1843, in _execute_context
return self._exec_insertmany_context(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/engine/base.py", line 2121, in _exec_insertmany_context
self._handle_dbapi_exception(
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/engine/base.py", line 2344, in _handle_dbapi_exception
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/engine/base.py", line 2113, in _exec_insertmany_context
dialect.do_execute(
File "/usr/local/lib/python3.12/site-packages/sqlalchemy/engine/default.py", line 922, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.IntegrityError: (sqlite3.IntegrityError) FOREIGN KEY constraint failed
[SQL: INSERT INTO states (entity_id, state, attributes, event_id, last_changed, last_changed_ts, last_updated, last_updated_ts, old_state_id, attributes_id, context_id, context_user_id, context_parent_id, origin_idx, context_id_bin, context_user_id_bin, context_parent_id_bin, metadata_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING state_id]
[parameters: (None, '25624.646', None, None, None, None, None, 1709652498.664474, 326200127, 135, None, None, None, 0, <memory at 0x7f726fcb80>, None, None, 29)]
(Background on this error at: https://sqlalche.me/e/20/gkpj)
2024-03-05 16:28:22.559 ERROR (Recorder) [homeassistant.components.recorder.util] The system will rename the corrupt database file //config/home-assistant_v2.db to //config/home-assistant_v2.db.corrupt.2024-03-05T15:28:22.559828+00:00 in order to allow startup to proceed
```
### Additional information
Logs in `core-2024.1.6` with the same 'corrupt' database:
```
2024-03-05 17:20:35.474 WARNING (Recorder) [homeassistant.components.recorder.util] The system could not validate that the sqlite3 database at //config/home-assistant_v2.db was shutdown cleanly
2024-03-05 17:20:35.515 WARNING (Recorder) [homeassistant.components.recorder.util] Ended unfinished session (id=306 from 2024-03-05 15:27:47.545959)
2024-03-05 17:21:11.322 ERROR (Recorder) [homeassistant.components.recorder.util] Error executing query: unsupported operand type(s) for +: 'NoneType' and 'float'
Traceback (most recent call last):
File "/usr/src/homeassistant/homeassistant/components/recorder/util.py", line 139, in session_scope
yield session
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 454, in compile_missing_statistics
modified_statistic_ids = _compile_statistics(
^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 528, in _compile_statistics
compiled: PlatformCompiledStatistics = platform_compile_statistics(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/sensor/recorder.py", line 453, in compile_statistics
last_stats = statistics.get_latest_short_term_statistics_with_session(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 1938, in get_latest_short_term_statistics_with_session
return _sorted_statistics_to_dict(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 2107, in _sorted_statistics_to_dict
"end": start_ts + table_duration_seconds,
~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~
TypeError: unsupported operand type(s) for +: 'NoneType' and 'float'
2024-03-05 17:21:11.344 ERROR (Recorder) [homeassistant.components.recorder.core] Error while processing event CompileMissingStatisticsTask(): unsupported operand type(s) for +: 'NoneType' and 'float'
Traceback (most recent call last):
File "/usr/src/homeassistant/homeassistant/components/recorder/core.py", line 901, in _guarded_process_one_task_or_recover
self._process_one_task_or_recover(task)
File "/usr/src/homeassistant/homeassistant/components/recorder/core.py", line 912, in _process_one_task_or_recover
return task.run(self)
^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/tasks.py", line 177, in run
if statistics.compile_missing_statistics(instance):
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/util.py", line 625, in wrapper
return job(instance, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 454, in compile_missing_statistics
modified_statistic_ids = _compile_statistics(
^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 528, in _compile_statistics
compiled: PlatformCompiledStatistics = platform_compile_statistics(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/sensor/recorder.py", line 453, in compile_statistics
last_stats = statistics.get_latest_short_term_statistics_with_session(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 1938, in get_latest_short_term_statistics_with_session
return _sorted_statistics_to_dict(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 2107, in _sorted_statistics_to_dict
"end": start_ts + table_duration_seconds,
~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~
TypeError: unsupported operand type(s) for +: 'NoneType' and 'float'
2024-03-05 17:25:10.540 ERROR (Recorder) [homeassistant.components.recorder.util] Error executing query: unsupported operand type(s) for +: 'NoneType' and 'float'
Traceback (most recent call last):
File "/usr/src/homeassistant/homeassistant/components/recorder/util.py", line 139, in session_scope
yield session
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 477, in compile_statistics
modified_statistic_ids = _compile_statistics(
^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 528, in _compile_statistics
compiled: PlatformCompiledStatistics = platform_compile_statistics(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/sensor/recorder.py", line 453, in compile_statistics
last_stats = statistics.get_latest_short_term_statistics_with_session(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 1938, in get_latest_short_term_statistics_with_session
return _sorted_statistics_to_dict(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 2107, in _sorted_statistics_to_dict
"end": start_ts + table_duration_seconds,
~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~
TypeError: unsupported operand type(s) for +: 'NoneType' and 'float'
2024-03-05 17:25:10.548 ERROR (Recorder) [homeassistant.components.recorder.core] Error while processing event StatisticsTask(start=datetime.datetime(2024, 3, 5, 16, 20, tzinfo=datetime.timezone.utc), fire_events=True): unsupported operand type(s) for +: 'NoneType' and 'float'
Traceback (most recent call last):
File "/usr/src/homeassistant/homeassistant/components/recorder/core.py", line 901, in _guarded_process_one_task_or_recover
self._process_one_task_or_recover(task)
File "/usr/src/homeassistant/homeassistant/components/recorder/core.py", line 912, in _process_one_task_or_recover
return task.run(self)
^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/tasks.py", line 165, in run
if statistics.compile_statistics(instance, self.start, self.fire_events):
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/util.py", line 625, in wrapper
return job(instance, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 477, in compile_statistics
modified_statistic_ids = _compile_statistics(
^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 528, in _compile_statistics
compiled: PlatformCompiledStatistics = platform_compile_statistics(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/sensor/recorder.py", line 453, in compile_statistics
last_stats = statistics.get_latest_short_term_statistics_with_session(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 1938, in get_latest_short_term_statistics_with_session
return _sorted_statistics_to_dict(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/src/homeassistant/homeassistant/components/recorder/statistics.py", line 2107, in _sorted_statistics_to_dict
"end": start_ts + table_duration_seconds,
~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~
TypeError: unsupported operand type(s) for +: 'NoneType' and 'float'
```