forked from platypush/platypush
Added support for automatic database migrations.
Added Alembic environment and `run_db_migrations` logic to the entities engine so database schema changes can be processed as soon as the application is started.
This commit is contained in:
parent
ff9b76477d
commit
78cee5d9b0
7 changed files with 286 additions and 5 deletions
|
@ -1,12 +1,16 @@
|
|||
import logging
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
import types
|
||||
from datetime import datetime
|
||||
import pkgutil
|
||||
from typing import Callable, Dict, Final, Optional, Set, Type, Tuple, Any
|
||||
|
||||
import pkgutil
|
||||
|
||||
from dateutil.tz import tzutc
|
||||
from sqlalchemy import (
|
||||
Boolean,
|
||||
|
@ -23,6 +27,7 @@ from sqlalchemy import (
|
|||
from sqlalchemy.orm import ColumnProperty, backref, relationship
|
||||
from sqlalchemy.orm.exc import ObjectDeletedError
|
||||
|
||||
import platypush
|
||||
from platypush.common.db import Base
|
||||
from platypush.message import JSONAble, Message
|
||||
|
||||
|
@ -170,7 +175,10 @@ if 'entity' not in Base.metadata:
|
|||
return normalized_name
|
||||
except ObjectDeletedError as e:
|
||||
logger.warning(
|
||||
f'Could not access column "{col.key}" for entity ID "{self.id}": {e}'
|
||||
'Could not access column "%s" for entity ID "{%s}": {%s}',
|
||||
col.key,
|
||||
self.id,
|
||||
e,
|
||||
)
|
||||
return None
|
||||
|
||||
|
@ -267,9 +275,9 @@ def _discover_entity_types():
|
|||
isinstance(e, (ImportError, ModuleNotFoundError))
|
||||
and modname[len(__package__) + 1 :] in _import_error_ignored_modules
|
||||
):
|
||||
logger.debug(f'Could not import module {modname}')
|
||||
logger.debug('Could not import module %s', modname)
|
||||
else:
|
||||
logger.warning(f'Could not import module {modname}')
|
||||
logger.warning('Could not import module %s', modname)
|
||||
logger.exception(e)
|
||||
|
||||
continue
|
||||
|
@ -292,7 +300,31 @@ def init_entities_db():
|
|||
"""
|
||||
from platypush.context import get_plugin
|
||||
|
||||
run_db_migrations()
|
||||
_discover_entity_types()
|
||||
db = get_plugin('db')
|
||||
assert db
|
||||
db.create_all(db.get_engine(), Base)
|
||||
|
||||
|
||||
def run_db_migrations():
|
||||
"""
|
||||
Run the database migrations upon engine initialization.
|
||||
"""
|
||||
logger.info('Running database migrations')
|
||||
alembic_ini = os.path.join(
|
||||
os.path.dirname(inspect.getabsfile(platypush)), 'migrations', 'alembic.ini'
|
||||
)
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
'-m',
|
||||
'alembic',
|
||||
'-c',
|
||||
alembic_ini,
|
||||
'upgrade',
|
||||
'head',
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
|
106
platypush/migrations/alembic.ini
Normal file
106
platypush/migrations/alembic.ini
Normal file
|
@ -0,0 +1,106 @@
|
|||
# A generic, single database configuration.
|
||||
|
||||
[alembic]
|
||||
# path to migration scripts
|
||||
script_location = %(here)s/alembic
|
||||
|
||||
# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
|
||||
# Uncomment the line below if you want the files to be prepended with date and time
|
||||
# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
|
||||
# for all available tokens
|
||||
# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
|
||||
|
||||
# sys.path path, will be prepended to sys.path if present.
|
||||
# defaults to the current working directory.
|
||||
prepend_sys_path = .
|
||||
|
||||
# timezone to use when rendering the date within the migration file
|
||||
# as well as the filename.
|
||||
# If specified, requires the python-dateutil library that can be
|
||||
# installed by adding `alembic[tz]` to the pip requirements
|
||||
# string value is passed to dateutil.tz.gettz()
|
||||
# leave blank for localtime
|
||||
# timezone =
|
||||
|
||||
# max length of characters to apply to the
|
||||
# "slug" field
|
||||
# truncate_slug_length = 40
|
||||
|
||||
# set to 'true' to run the environment during
|
||||
# the 'revision' command, regardless of autogenerate
|
||||
# revision_environment = false
|
||||
|
||||
# set to 'true' to allow .pyc and .pyo files without
|
||||
# a source .py file to be detected as revisions in the
|
||||
# versions/ directory
|
||||
# sourceless = false
|
||||
|
||||
# version location specification; This defaults
|
||||
# to alembic/versions. When using multiple version
|
||||
# directories, initial revisions must be specified with --version-path.
|
||||
# The path separator used here should be the separator specified by "version_path_separator" below.
|
||||
version_locations = %(here)s/alembic/versions
|
||||
|
||||
# version path separator; As mentioned above, this is the character used to split
|
||||
# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
|
||||
# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
|
||||
# Valid values for version_path_separator are:
|
||||
#
|
||||
# version_path_separator = :
|
||||
# version_path_separator = ;
|
||||
# version_path_separator = space
|
||||
# Use os.pathsep. Default configuration used for new projects.
|
||||
version_path_separator = os
|
||||
|
||||
# the output encoding used when revision files
|
||||
# are written from script.py.mako
|
||||
# output_encoding = utf-8
|
||||
|
||||
sqlalchemy.url = %(DB_ENGINE)s
|
||||
|
||||
|
||||
[post_write_hooks]
|
||||
# post_write_hooks defines scripts or Python functions that are run
|
||||
# on newly generated revision scripts. See the documentation for further
|
||||
# detail and examples
|
||||
|
||||
# format using "black" - use the console_scripts runner, against the "black" entrypoint
|
||||
# hooks = black
|
||||
# black.type = console_scripts
|
||||
# black.entrypoint = black
|
||||
# black.options = -l 79 REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Logging configuration
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
92
platypush/migrations/alembic/env.py
Normal file
92
platypush/migrations/alembic/env.py
Normal file
|
@ -0,0 +1,92 @@
|
|||
from logging.config import fileConfig
|
||||
|
||||
from sqlalchemy import engine_from_config
|
||||
from sqlalchemy import pool
|
||||
|
||||
from alembic import context
|
||||
|
||||
from platypush.config import Config
|
||||
from platypush.common.db import Base
|
||||
|
||||
# this is the Alembic Config object, which provides
|
||||
# access to the values within the .ini file in use.
|
||||
config = context.config
|
||||
|
||||
# Interpret the config file for Python logging.
|
||||
# This line sets up loggers basically.
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
# add your model's MetaData object here
|
||||
# for 'autogenerate' support
|
||||
# from myapp import mymodel
|
||||
# target_metadata = mymodel.Base.metadata
|
||||
target_metadata = Base.metadata
|
||||
|
||||
# other values from the config, defined by the needs of env.py,
|
||||
# can be acquired:
|
||||
# my_important_option = config.get_main_option("my_important_option")
|
||||
# ... etc.
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
"""Run migrations in 'offline' mode.
|
||||
|
||||
This configures the context with just a URL
|
||||
and not an Engine, though an Engine is acceptable
|
||||
here as well. By skipping the Engine creation
|
||||
we don't even need a DBAPI to be available.
|
||||
|
||||
Calls to context.execute() here emit the given string to the
|
||||
script output.
|
||||
|
||||
"""
|
||||
url = config.get_main_option("sqlalchemy.url")
|
||||
context.configure(
|
||||
url=url,
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
"""Run migrations in 'online' mode.
|
||||
|
||||
In this scenario we need to create an Engine
|
||||
and associate a connection with the context.
|
||||
|
||||
"""
|
||||
connectable = engine_from_config(
|
||||
config.get_section(config.config_ini_section, {}),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
|
||||
with connectable.connect() as connection:
|
||||
context.configure(connection=connection, target_metadata=target_metadata)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def set_db_engine():
|
||||
db_conf = Config.get('db')
|
||||
assert db_conf, 'Could not retrieve the database configuration'
|
||||
engine = db_conf['engine']
|
||||
assert engine, 'No database engine configured'
|
||||
|
||||
config = context.config
|
||||
section = config.config_ini_section
|
||||
config.set_section_option(section, 'DB_ENGINE', engine)
|
||||
|
||||
|
||||
set_db_engine()
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
24
platypush/migrations/alembic/script.py.mako
Normal file
24
platypush/migrations/alembic/script.py.mako
Normal file
|
@ -0,0 +1,24 @@
|
|||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = ${repr(up_revision)}
|
||||
down_revision = ${repr(down_revision)}
|
||||
branch_labels = ${repr(branch_labels)}
|
||||
depends_on = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
|
@ -0,0 +1,22 @@
|
|||
"""Base alembic version
|
||||
|
||||
Revision ID: d030953a871d
|
||||
Revises:
|
||||
Create Date: 2023-04-28 22:32:49.460118
|
||||
|
||||
"""
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'd030953a871d'
|
||||
down_revision = None
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
|
@ -13,7 +13,8 @@ python-dateutil
|
|||
tz
|
||||
frozendict
|
||||
requests
|
||||
sqlalchemy<2.0.0
|
||||
sqlalchemy
|
||||
alembic
|
||||
bcrypt
|
||||
rsa
|
||||
zeroconf>=0.27.0
|
||||
|
|
4
setup.py
4
setup.py
|
@ -38,6 +38,9 @@ setup(
|
|||
url="https://platypush.tech",
|
||||
packages=find_packages(),
|
||||
include_package_data=True,
|
||||
package_data={
|
||||
'platypush': ['alembic.ini', 'alembic/*', 'alembic/**/*'],
|
||||
},
|
||||
entry_points={
|
||||
'console_scripts': [
|
||||
'platypush=platypush:main',
|
||||
|
@ -58,6 +61,7 @@ setup(
|
|||
'requests',
|
||||
'croniter',
|
||||
'sqlalchemy',
|
||||
'alembic',
|
||||
'websockets',
|
||||
'websocket-client',
|
||||
'wheel',
|
||||
|
|
Loading…
Reference in a new issue