forked from platypush/platypush
Added support for automatic database migrations.
Added Alembic environment and `run_db_migrations` logic to the entities engine so database schema changes can be processed as soon as the application is started.
This commit is contained in:
parent
ff9b76477d
commit
78cee5d9b0
7 changed files with 286 additions and 5 deletions
|
@ -1,12 +1,16 @@
|
||||||
import logging
|
import logging
|
||||||
import inspect
|
import inspect
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
import pathlib
|
import pathlib
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
import types
|
import types
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
import pkgutil
|
|
||||||
from typing import Callable, Dict, Final, Optional, Set, Type, Tuple, Any
|
from typing import Callable, Dict, Final, Optional, Set, Type, Tuple, Any
|
||||||
|
|
||||||
|
import pkgutil
|
||||||
|
|
||||||
from dateutil.tz import tzutc
|
from dateutil.tz import tzutc
|
||||||
from sqlalchemy import (
|
from sqlalchemy import (
|
||||||
Boolean,
|
Boolean,
|
||||||
|
@ -23,6 +27,7 @@ from sqlalchemy import (
|
||||||
from sqlalchemy.orm import ColumnProperty, backref, relationship
|
from sqlalchemy.orm import ColumnProperty, backref, relationship
|
||||||
from sqlalchemy.orm.exc import ObjectDeletedError
|
from sqlalchemy.orm.exc import ObjectDeletedError
|
||||||
|
|
||||||
|
import platypush
|
||||||
from platypush.common.db import Base
|
from platypush.common.db import Base
|
||||||
from platypush.message import JSONAble, Message
|
from platypush.message import JSONAble, Message
|
||||||
|
|
||||||
|
@ -170,7 +175,10 @@ if 'entity' not in Base.metadata:
|
||||||
return normalized_name
|
return normalized_name
|
||||||
except ObjectDeletedError as e:
|
except ObjectDeletedError as e:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
f'Could not access column "{col.key}" for entity ID "{self.id}": {e}'
|
'Could not access column "%s" for entity ID "{%s}": {%s}',
|
||||||
|
col.key,
|
||||||
|
self.id,
|
||||||
|
e,
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
@ -267,9 +275,9 @@ def _discover_entity_types():
|
||||||
isinstance(e, (ImportError, ModuleNotFoundError))
|
isinstance(e, (ImportError, ModuleNotFoundError))
|
||||||
and modname[len(__package__) + 1 :] in _import_error_ignored_modules
|
and modname[len(__package__) + 1 :] in _import_error_ignored_modules
|
||||||
):
|
):
|
||||||
logger.debug(f'Could not import module {modname}')
|
logger.debug('Could not import module %s', modname)
|
||||||
else:
|
else:
|
||||||
logger.warning(f'Could not import module {modname}')
|
logger.warning('Could not import module %s', modname)
|
||||||
logger.exception(e)
|
logger.exception(e)
|
||||||
|
|
||||||
continue
|
continue
|
||||||
|
@ -292,7 +300,31 @@ def init_entities_db():
|
||||||
"""
|
"""
|
||||||
from platypush.context import get_plugin
|
from platypush.context import get_plugin
|
||||||
|
|
||||||
|
run_db_migrations()
|
||||||
_discover_entity_types()
|
_discover_entity_types()
|
||||||
db = get_plugin('db')
|
db = get_plugin('db')
|
||||||
assert db
|
assert db
|
||||||
db.create_all(db.get_engine(), Base)
|
db.create_all(db.get_engine(), Base)
|
||||||
|
|
||||||
|
|
||||||
|
def run_db_migrations():
|
||||||
|
"""
|
||||||
|
Run the database migrations upon engine initialization.
|
||||||
|
"""
|
||||||
|
logger.info('Running database migrations')
|
||||||
|
alembic_ini = os.path.join(
|
||||||
|
os.path.dirname(inspect.getabsfile(platypush)), 'migrations', 'alembic.ini'
|
||||||
|
)
|
||||||
|
|
||||||
|
subprocess.run(
|
||||||
|
[
|
||||||
|
sys.executable,
|
||||||
|
'-m',
|
||||||
|
'alembic',
|
||||||
|
'-c',
|
||||||
|
alembic_ini,
|
||||||
|
'upgrade',
|
||||||
|
'head',
|
||||||
|
],
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
|
106
platypush/migrations/alembic.ini
Normal file
106
platypush/migrations/alembic.ini
Normal file
|
@ -0,0 +1,106 @@
|
||||||
|
# A generic, single database configuration.
|
||||||
|
|
||||||
|
[alembic]
|
||||||
|
# path to migration scripts
|
||||||
|
script_location = %(here)s/alembic
|
||||||
|
|
||||||
|
# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
|
||||||
|
# Uncomment the line below if you want the files to be prepended with date and time
|
||||||
|
# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
|
||||||
|
# for all available tokens
|
||||||
|
# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
|
||||||
|
|
||||||
|
# sys.path path, will be prepended to sys.path if present.
|
||||||
|
# defaults to the current working directory.
|
||||||
|
prepend_sys_path = .
|
||||||
|
|
||||||
|
# timezone to use when rendering the date within the migration file
|
||||||
|
# as well as the filename.
|
||||||
|
# If specified, requires the python-dateutil library that can be
|
||||||
|
# installed by adding `alembic[tz]` to the pip requirements
|
||||||
|
# string value is passed to dateutil.tz.gettz()
|
||||||
|
# leave blank for localtime
|
||||||
|
# timezone =
|
||||||
|
|
||||||
|
# max length of characters to apply to the
|
||||||
|
# "slug" field
|
||||||
|
# truncate_slug_length = 40
|
||||||
|
|
||||||
|
# set to 'true' to run the environment during
|
||||||
|
# the 'revision' command, regardless of autogenerate
|
||||||
|
# revision_environment = false
|
||||||
|
|
||||||
|
# set to 'true' to allow .pyc and .pyo files without
|
||||||
|
# a source .py file to be detected as revisions in the
|
||||||
|
# versions/ directory
|
||||||
|
# sourceless = false
|
||||||
|
|
||||||
|
# version location specification; This defaults
|
||||||
|
# to alembic/versions. When using multiple version
|
||||||
|
# directories, initial revisions must be specified with --version-path.
|
||||||
|
# The path separator used here should be the separator specified by "version_path_separator" below.
|
||||||
|
version_locations = %(here)s/alembic/versions
|
||||||
|
|
||||||
|
# version path separator; As mentioned above, this is the character used to split
|
||||||
|
# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
|
||||||
|
# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
|
||||||
|
# Valid values for version_path_separator are:
|
||||||
|
#
|
||||||
|
# version_path_separator = :
|
||||||
|
# version_path_separator = ;
|
||||||
|
# version_path_separator = space
|
||||||
|
# Use os.pathsep. Default configuration used for new projects.
|
||||||
|
version_path_separator = os
|
||||||
|
|
||||||
|
# the output encoding used when revision files
|
||||||
|
# are written from script.py.mako
|
||||||
|
# output_encoding = utf-8
|
||||||
|
|
||||||
|
sqlalchemy.url = %(DB_ENGINE)s
|
||||||
|
|
||||||
|
|
||||||
|
[post_write_hooks]
|
||||||
|
# post_write_hooks defines scripts or Python functions that are run
|
||||||
|
# on newly generated revision scripts. See the documentation for further
|
||||||
|
# detail and examples
|
||||||
|
|
||||||
|
# format using "black" - use the console_scripts runner, against the "black" entrypoint
|
||||||
|
# hooks = black
|
||||||
|
# black.type = console_scripts
|
||||||
|
# black.entrypoint = black
|
||||||
|
# black.options = -l 79 REVISION_SCRIPT_FILENAME
|
||||||
|
|
||||||
|
# Logging configuration
|
||||||
|
[loggers]
|
||||||
|
keys = root,sqlalchemy,alembic
|
||||||
|
|
||||||
|
[handlers]
|
||||||
|
keys = console
|
||||||
|
|
||||||
|
[formatters]
|
||||||
|
keys = generic
|
||||||
|
|
||||||
|
[logger_root]
|
||||||
|
level = WARN
|
||||||
|
handlers = console
|
||||||
|
qualname =
|
||||||
|
|
||||||
|
[logger_sqlalchemy]
|
||||||
|
level = WARN
|
||||||
|
handlers =
|
||||||
|
qualname = sqlalchemy.engine
|
||||||
|
|
||||||
|
[logger_alembic]
|
||||||
|
level = INFO
|
||||||
|
handlers =
|
||||||
|
qualname = alembic
|
||||||
|
|
||||||
|
[handler_console]
|
||||||
|
class = StreamHandler
|
||||||
|
args = (sys.stderr,)
|
||||||
|
level = NOTSET
|
||||||
|
formatter = generic
|
||||||
|
|
||||||
|
[formatter_generic]
|
||||||
|
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||||
|
datefmt = %H:%M:%S
|
92
platypush/migrations/alembic/env.py
Normal file
92
platypush/migrations/alembic/env.py
Normal file
|
@ -0,0 +1,92 @@
|
||||||
|
from logging.config import fileConfig
|
||||||
|
|
||||||
|
from sqlalchemy import engine_from_config
|
||||||
|
from sqlalchemy import pool
|
||||||
|
|
||||||
|
from alembic import context
|
||||||
|
|
||||||
|
from platypush.config import Config
|
||||||
|
from platypush.common.db import Base
|
||||||
|
|
||||||
|
# this is the Alembic Config object, which provides
|
||||||
|
# access to the values within the .ini file in use.
|
||||||
|
config = context.config
|
||||||
|
|
||||||
|
# Interpret the config file for Python logging.
|
||||||
|
# This line sets up loggers basically.
|
||||||
|
if config.config_file_name is not None:
|
||||||
|
fileConfig(config.config_file_name)
|
||||||
|
|
||||||
|
# add your model's MetaData object here
|
||||||
|
# for 'autogenerate' support
|
||||||
|
# from myapp import mymodel
|
||||||
|
# target_metadata = mymodel.Base.metadata
|
||||||
|
target_metadata = Base.metadata
|
||||||
|
|
||||||
|
# other values from the config, defined by the needs of env.py,
|
||||||
|
# can be acquired:
|
||||||
|
# my_important_option = config.get_main_option("my_important_option")
|
||||||
|
# ... etc.
|
||||||
|
|
||||||
|
|
||||||
|
def run_migrations_offline() -> None:
|
||||||
|
"""Run migrations in 'offline' mode.
|
||||||
|
|
||||||
|
This configures the context with just a URL
|
||||||
|
and not an Engine, though an Engine is acceptable
|
||||||
|
here as well. By skipping the Engine creation
|
||||||
|
we don't even need a DBAPI to be available.
|
||||||
|
|
||||||
|
Calls to context.execute() here emit the given string to the
|
||||||
|
script output.
|
||||||
|
|
||||||
|
"""
|
||||||
|
url = config.get_main_option("sqlalchemy.url")
|
||||||
|
context.configure(
|
||||||
|
url=url,
|
||||||
|
target_metadata=target_metadata,
|
||||||
|
literal_binds=True,
|
||||||
|
dialect_opts={"paramstyle": "named"},
|
||||||
|
)
|
||||||
|
|
||||||
|
with context.begin_transaction():
|
||||||
|
context.run_migrations()
|
||||||
|
|
||||||
|
|
||||||
|
def run_migrations_online() -> None:
|
||||||
|
"""Run migrations in 'online' mode.
|
||||||
|
|
||||||
|
In this scenario we need to create an Engine
|
||||||
|
and associate a connection with the context.
|
||||||
|
|
||||||
|
"""
|
||||||
|
connectable = engine_from_config(
|
||||||
|
config.get_section(config.config_ini_section, {}),
|
||||||
|
prefix="sqlalchemy.",
|
||||||
|
poolclass=pool.NullPool,
|
||||||
|
)
|
||||||
|
|
||||||
|
with connectable.connect() as connection:
|
||||||
|
context.configure(connection=connection, target_metadata=target_metadata)
|
||||||
|
|
||||||
|
with context.begin_transaction():
|
||||||
|
context.run_migrations()
|
||||||
|
|
||||||
|
|
||||||
|
def set_db_engine():
|
||||||
|
db_conf = Config.get('db')
|
||||||
|
assert db_conf, 'Could not retrieve the database configuration'
|
||||||
|
engine = db_conf['engine']
|
||||||
|
assert engine, 'No database engine configured'
|
||||||
|
|
||||||
|
config = context.config
|
||||||
|
section = config.config_ini_section
|
||||||
|
config.set_section_option(section, 'DB_ENGINE', engine)
|
||||||
|
|
||||||
|
|
||||||
|
set_db_engine()
|
||||||
|
|
||||||
|
if context.is_offline_mode():
|
||||||
|
run_migrations_offline()
|
||||||
|
else:
|
||||||
|
run_migrations_online()
|
24
platypush/migrations/alembic/script.py.mako
Normal file
24
platypush/migrations/alembic/script.py.mako
Normal file
|
@ -0,0 +1,24 @@
|
||||||
|
"""${message}
|
||||||
|
|
||||||
|
Revision ID: ${up_revision}
|
||||||
|
Revises: ${down_revision | comma,n}
|
||||||
|
Create Date: ${create_date}
|
||||||
|
|
||||||
|
"""
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
${imports if imports else ""}
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = ${repr(up_revision)}
|
||||||
|
down_revision = ${repr(down_revision)}
|
||||||
|
branch_labels = ${repr(branch_labels)}
|
||||||
|
depends_on = ${repr(depends_on)}
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
${upgrades if upgrades else "pass"}
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
${downgrades if downgrades else "pass"}
|
|
@ -0,0 +1,22 @@
|
||||||
|
"""Base alembic version
|
||||||
|
|
||||||
|
Revision ID: d030953a871d
|
||||||
|
Revises:
|
||||||
|
Create Date: 2023-04-28 22:32:49.460118
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = 'd030953a871d'
|
||||||
|
down_revision = None
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
pass
|
|
@ -13,7 +13,8 @@ python-dateutil
|
||||||
tz
|
tz
|
||||||
frozendict
|
frozendict
|
||||||
requests
|
requests
|
||||||
sqlalchemy<2.0.0
|
sqlalchemy
|
||||||
|
alembic
|
||||||
bcrypt
|
bcrypt
|
||||||
rsa
|
rsa
|
||||||
zeroconf>=0.27.0
|
zeroconf>=0.27.0
|
||||||
|
|
4
setup.py
4
setup.py
|
@ -38,6 +38,9 @@ setup(
|
||||||
url="https://platypush.tech",
|
url="https://platypush.tech",
|
||||||
packages=find_packages(),
|
packages=find_packages(),
|
||||||
include_package_data=True,
|
include_package_data=True,
|
||||||
|
package_data={
|
||||||
|
'platypush': ['alembic.ini', 'alembic/*', 'alembic/**/*'],
|
||||||
|
},
|
||||||
entry_points={
|
entry_points={
|
||||||
'console_scripts': [
|
'console_scripts': [
|
||||||
'platypush=platypush:main',
|
'platypush=platypush:main',
|
||||||
|
@ -58,6 +61,7 @@ setup(
|
||||||
'requests',
|
'requests',
|
||||||
'croniter',
|
'croniter',
|
||||||
'sqlalchemy',
|
'sqlalchemy',
|
||||||
|
'alembic',
|
||||||
'websockets',
|
'websockets',
|
||||||
'websocket-client',
|
'websocket-client',
|
||||||
'wheel',
|
'wheel',
|
||||||
|
|
Loading…
Reference in a new issue