Skip to content

Commit

Permalink
Use BIGINT SQL type for ID columns (home-assistant#121025)
Browse files Browse the repository at this point in the history
  • Loading branch information
@emontnemery
emontnemery authored Jul 5, 2024
1 parent fd815be commit 6eeb701
Show file tree
Hide file tree
Showing 5 changed files with 1,119 additions and 28 deletions.
1 change: 1 addition & 0 deletions homeassistant/components/recorder/core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1459,6 +1459,7 @@ def _setup_connection(self) -> None:
self.__dict__.pop("dialect_name", None)
sqlalchemy_event.listen(self.engine, "connect", self._setup_recorder_connection)

migration.pre_migrate_schema(self.engine)
Base.metadata.create_all(self.engine)
self._get_session = scoped_session(sessionmaker(bind=self.engine, future=True))
_LOGGER.debug("Connected to recorder database")
Expand Down
91 changes: 70 additions & 21 deletions homeassistant/components/recorder/db_schema.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,11 @@ class Base(DeclarativeBase):
"""Base class for tables."""


SCHEMA_VERSION = 43
class LegacyBase(DeclarativeBase):
"""Base class for tables, used for schema migration."""


SCHEMA_VERSION = 44

_LOGGER = logging.getLogger(__name__)

Expand Down Expand Up @@ -187,6 +191,9 @@ def result_processor(self, dialect, coltype): # type: ignore[no-untyped-def]
return None


# Although all integers are same in SQLite, it does not allow an identity column to be BIGINT
# https://sqlite.org/forum/info/2dfa968a702e1506e885cb06d92157d492108b22bf39459506ab9f7125bca7fd
ID_TYPE = BigInteger().with_variant(sqlite.INTEGER, "sqlite")
# For MariaDB and MySQL we can use an unsigned integer type since it will fit 2**32
# for sqlite and postgresql we use a bigint
UINT_32_TYPE = BigInteger().with_variant(
Expand Down Expand Up @@ -217,6 +224,7 @@ def result_processor(self, dialect, coltype): # type: ignore[no-untyped-def]
UNUSED_LEGACY_DATETIME_COLUMN = UnusedDateTime(timezone=True)
UNUSED_LEGACY_INTEGER_COLUMN = SmallInteger()
DOUBLE_PRECISION_TYPE_SQL = "DOUBLE PRECISION"
BIG_INTEGER_SQL = "BIGINT"
CONTEXT_BINARY_TYPE = LargeBinary(CONTEXT_ID_BIN_MAX_LENGTH).with_variant(
NativeLargeBinary(CONTEXT_ID_BIN_MAX_LENGTH), "mysql", "mariadb", "sqlite"
)
Expand Down Expand Up @@ -258,7 +266,7 @@ class Events(Base):
_DEFAULT_TABLE_ARGS,
)
__tablename__ = TABLE_EVENTS
event_id: Mapped[int] = mapped_column(Integer, Identity(), primary_key=True)
event_id: Mapped[int] = mapped_column(ID_TYPE, Identity(), primary_key=True)
event_type: Mapped[str | None] = mapped_column(UNUSED_LEGACY_COLUMN)
event_data: Mapped[str | None] = mapped_column(UNUSED_LEGACY_COLUMN)
origin: Mapped[str | None] = mapped_column(UNUSED_LEGACY_COLUMN)
Expand All @@ -269,13 +277,13 @@ class Events(Base):
context_user_id: Mapped[str | None] = mapped_column(UNUSED_LEGACY_COLUMN)
context_parent_id: Mapped[str | None] = mapped_column(UNUSED_LEGACY_COLUMN)
data_id: Mapped[int | None] = mapped_column(
Integer, ForeignKey("event_data.data_id"), index=True
ID_TYPE, ForeignKey("event_data.data_id"), index=True
)
context_id_bin: Mapped[bytes | None] = mapped_column(CONTEXT_BINARY_TYPE)
context_user_id_bin: Mapped[bytes | None] = mapped_column(CONTEXT_BINARY_TYPE)
context_parent_id_bin: Mapped[bytes | None] = mapped_column(CONTEXT_BINARY_TYPE)
event_type_id: Mapped[int | None] = mapped_column(
Integer, ForeignKey("event_types.event_type_id")
ID_TYPE, ForeignKey("event_types.event_type_id")
)
event_data_rel: Mapped[EventData | None] = relationship("EventData")
event_type_rel: Mapped[EventTypes | None] = relationship("EventTypes")
Expand Down Expand Up @@ -347,7 +355,7 @@ class EventData(Base):

__table_args__ = (_DEFAULT_TABLE_ARGS,)
__tablename__ = TABLE_EVENT_DATA
data_id: Mapped[int] = mapped_column(Integer, Identity(), primary_key=True)
data_id: Mapped[int] = mapped_column(ID_TYPE, Identity(), primary_key=True)
hash: Mapped[int | None] = mapped_column(UINT_32_TYPE, index=True)
# Note that this is not named attributes to avoid confusion with the states table
shared_data: Mapped[str | None] = mapped_column(
Expand Down Expand Up @@ -403,7 +411,7 @@ class EventTypes(Base):

__table_args__ = (_DEFAULT_TABLE_ARGS,)
__tablename__ = TABLE_EVENT_TYPES
event_type_id: Mapped[int] = mapped_column(Integer, Identity(), primary_key=True)
event_type_id: Mapped[int] = mapped_column(ID_TYPE, Identity(), primary_key=True)
event_type: Mapped[str | None] = mapped_column(
String(MAX_LENGTH_EVENT_EVENT_TYPE), index=True, unique=True
)
Expand Down Expand Up @@ -433,7 +441,7 @@ class States(Base):
_DEFAULT_TABLE_ARGS,
)
__tablename__ = TABLE_STATES
state_id: Mapped[int] = mapped_column(Integer, Identity(), primary_key=True)
state_id: Mapped[int] = mapped_column(ID_TYPE, Identity(), primary_key=True)
entity_id: Mapped[str | None] = mapped_column(UNUSED_LEGACY_COLUMN)
state: Mapped[str | None] = mapped_column(String(MAX_LENGTH_STATE_STATE))
attributes: Mapped[str | None] = mapped_column(UNUSED_LEGACY_COLUMN)
Expand All @@ -446,10 +454,10 @@ class States(Base):
TIMESTAMP_TYPE, default=time.time, index=True
)
old_state_id: Mapped[int | None] = mapped_column(
Integer, ForeignKey("states.state_id"), index=True
ID_TYPE, ForeignKey("states.state_id"), index=True
)
attributes_id: Mapped[int | None] = mapped_column(
Integer, ForeignKey("state_attributes.attributes_id"), index=True
ID_TYPE, ForeignKey("state_attributes.attributes_id"), index=True
)
context_id: Mapped[str | None] = mapped_column(UNUSED_LEGACY_COLUMN)
context_user_id: Mapped[str | None] = mapped_column(UNUSED_LEGACY_COLUMN)
Expand All @@ -463,7 +471,7 @@ class States(Base):
context_user_id_bin: Mapped[bytes | None] = mapped_column(CONTEXT_BINARY_TYPE)
context_parent_id_bin: Mapped[bytes | None] = mapped_column(CONTEXT_BINARY_TYPE)
metadata_id: Mapped[int | None] = mapped_column(
Integer, ForeignKey("states_meta.metadata_id")
ID_TYPE, ForeignKey("states_meta.metadata_id")
)
states_meta_rel: Mapped[StatesMeta | None] = relationship("StatesMeta")

Expand Down Expand Up @@ -573,7 +581,7 @@ class StateAttributes(Base):

__table_args__ = (_DEFAULT_TABLE_ARGS,)
__tablename__ = TABLE_STATE_ATTRIBUTES
attributes_id: Mapped[int] = mapped_column(Integer, Identity(), primary_key=True)
attributes_id: Mapped[int] = mapped_column(ID_TYPE, Identity(), primary_key=True)
hash: Mapped[int | None] = mapped_column(UINT_32_TYPE, index=True)
# Note that this is not named attributes to avoid confusion with the states table
shared_attrs: Mapped[str | None] = mapped_column(
Expand Down Expand Up @@ -647,7 +655,7 @@ class StatesMeta(Base):

__table_args__ = (_DEFAULT_TABLE_ARGS,)
__tablename__ = TABLE_STATES_META
metadata_id: Mapped[int] = mapped_column(Integer, Identity(), primary_key=True)
metadata_id: Mapped[int] = mapped_column(ID_TYPE, Identity(), primary_key=True)
entity_id: Mapped[str | None] = mapped_column(
String(MAX_LENGTH_STATE_ENTITY_ID), index=True, unique=True
)
Expand All @@ -664,11 +672,11 @@ def __repr__(self) -> str:
class StatisticsBase:
"""Statistics base class."""

id: Mapped[int] = mapped_column(Integer, Identity(), primary_key=True)
id: Mapped[int] = mapped_column(ID_TYPE, Identity(), primary_key=True)
created: Mapped[datetime | None] = mapped_column(UNUSED_LEGACY_DATETIME_COLUMN)
created_ts: Mapped[float | None] = mapped_column(TIMESTAMP_TYPE, default=time.time)
metadata_id: Mapped[int | None] = mapped_column(
Integer,
ID_TYPE,
ForeignKey(f"{TABLE_STATISTICS_META}.id", ondelete="CASCADE"),
)
start: Mapped[datetime | None] = mapped_column(UNUSED_LEGACY_DATETIME_COLUMN)
Expand Down Expand Up @@ -738,11 +746,17 @@ class Statistics(Base, StatisticsBase):
__tablename__ = TABLE_STATISTICS


class StatisticsShortTerm(Base, StatisticsBase):
class _StatisticsShortTerm(StatisticsBase):
"""Short term statistics."""

duration = timedelta(minutes=5)

__tablename__ = TABLE_STATISTICS_SHORT_TERM


class StatisticsShortTerm(Base, _StatisticsShortTerm):
"""Short term statistics."""

__table_args__ = (
# Used for fetching statistics for a certain entity at a specific time
Index(
Expand All @@ -753,15 +767,35 @@ class StatisticsShortTerm(Base, StatisticsBase):
),
_DEFAULT_TABLE_ARGS,
)
__tablename__ = TABLE_STATISTICS_SHORT_TERM


class StatisticsMeta(Base):
class LegacyStatisticsShortTerm(LegacyBase, _StatisticsShortTerm):
"""Short term statistics with 32-bit index, used for schema migration."""

__table_args__ = (
# Used for fetching statistics for a certain entity at a specific time
Index(
"ix_statistics_short_term_statistic_id_start_ts",
"metadata_id",
"start_ts",
unique=True,
),
_DEFAULT_TABLE_ARGS,
)

metadata_id: Mapped[int | None] = mapped_column(
Integer,
ForeignKey(f"{TABLE_STATISTICS_META}.id", ondelete="CASCADE"),
use_existing_column=True,
)


class _StatisticsMeta:
"""Statistics meta data."""

__table_args__ = (_DEFAULT_TABLE_ARGS,)
__tablename__ = TABLE_STATISTICS_META
id: Mapped[int] = mapped_column(Integer, Identity(), primary_key=True)
id: Mapped[int] = mapped_column(ID_TYPE, Identity(), primary_key=True)
statistic_id: Mapped[str | None] = mapped_column(
String(255), index=True, unique=True
)
Expand All @@ -777,6 +811,21 @@ def from_meta(meta: StatisticMetaData) -> StatisticsMeta:
return StatisticsMeta(**meta)


class StatisticsMeta(Base, _StatisticsMeta):
"""Statistics meta data."""


class LegacyStatisticsMeta(LegacyBase, _StatisticsMeta):
"""Statistics meta data with 32-bit index, used for schema migration."""

id: Mapped[int] = mapped_column(
Integer,
Identity(),
primary_key=True,
use_existing_column=True,
)


class RecorderRuns(Base):
"""Representation of recorder run."""

Expand All @@ -785,7 +834,7 @@ class RecorderRuns(Base):
_DEFAULT_TABLE_ARGS,
)
__tablename__ = TABLE_RECORDER_RUNS
run_id: Mapped[int] = mapped_column(Integer, Identity(), primary_key=True)
run_id: Mapped[int] = mapped_column(ID_TYPE, Identity(), primary_key=True)
start: Mapped[datetime] = mapped_column(DATETIME_TYPE, default=dt_util.utcnow)
end: Mapped[datetime | None] = mapped_column(DATETIME_TYPE)
closed_incorrect: Mapped[bool] = mapped_column(Boolean, default=False)
Expand Down Expand Up @@ -824,7 +873,7 @@ class SchemaChanges(Base):
__tablename__ = TABLE_SCHEMA_CHANGES
__table_args__ = (_DEFAULT_TABLE_ARGS,)

change_id: Mapped[int] = mapped_column(Integer, Identity(), primary_key=True)
change_id: Mapped[int] = mapped_column(ID_TYPE, Identity(), primary_key=True)
schema_version: Mapped[int | None] = mapped_column(Integer)
changed: Mapped[datetime] = mapped_column(DATETIME_TYPE, default=dt_util.utcnow)

Expand All @@ -844,7 +893,7 @@ class StatisticsRuns(Base):
__tablename__ = TABLE_STATISTICS_RUNS
__table_args__ = (_DEFAULT_TABLE_ARGS,)

run_id: Mapped[int] = mapped_column(Integer, Identity(), primary_key=True)
run_id: Mapped[int] = mapped_column(ID_TYPE, Identity(), primary_key=True)
start: Mapped[datetime] = mapped_column(DATETIME_TYPE, index=True)

def __repr__(self) -> str:
Expand Down
Loading

0 comments on commit 6eeb701

Please sign in to comment.