mirror of
https://github.com/simonw/sqlite-utils.git
synced 2026-09-20 15:34:08 +02:00
parent
8f0c06e188
commit
09c7005f15
10 changed files with 811 additions and 3 deletions
|
|
@ -4,6 +4,13 @@
|
||||||
Changelog
|
Changelog
|
||||||
===========
|
===========
|
||||||
|
|
||||||
|
.. _v_unreleased:
|
||||||
|
|
||||||
|
Unreleased
|
||||||
|
----------
|
||||||
|
|
||||||
|
- New ``sqlite-utils migrate`` command for applying Python database migrations, incorporating functionality that was previously provided by the separate `sqlite-migrate <https://github.com/simonw/sqlite-migrate>`__ plugin. Define migration sets using the new :class:`sqlite_utils.Migrations` class and apply them using ``sqlite-utils migrate database.db migrations.py`` or the :ref:`migrations Python API <migrations_python>`. See :ref:`migrations` for details. (:issue:`752`)
|
||||||
|
|
||||||
.. _v3_39:
|
.. _v3_39:
|
||||||
|
|
||||||
3.39 (2025-11-24)
|
3.39 (2025-11-24)
|
||||||
|
|
@ -182,7 +189,7 @@ This release introduces a new :ref:`plugin system <plugins>`. Read more about th
|
||||||
- Conversion functions passed to :ref:`table.convert(...) <python_api_convert>` can now return lists or dictionaries, which will be inserted into the database as JSON strings. (:issue:`495`)
|
- Conversion functions passed to :ref:`table.convert(...) <python_api_convert>` can now return lists or dictionaries, which will be inserted into the database as JSON strings. (:issue:`495`)
|
||||||
- ``sqlite-utils install`` and ``sqlite-utils uninstall`` commands for installing packages into the same virtual environment as ``sqlite-utils``, :ref:`described here <cli_install>`. (:issue:`483`)
|
- ``sqlite-utils install`` and ``sqlite-utils uninstall`` commands for installing packages into the same virtual environment as ``sqlite-utils``, :ref:`described here <cli_install>`. (:issue:`483`)
|
||||||
- New :ref:`sqlite_utils.utils.flatten() <reference_utils_flatten>` utility function. (:issue:`500`)
|
- New :ref:`sqlite_utils.utils.flatten() <reference_utils_flatten>` utility function. (:issue:`500`)
|
||||||
- Documentation on :ref:`using Just <contributing_just>` to run tests, linters and build documentation.
|
- Documentation on :ref:`using Just <contributing_just>` to run tests, linters and build documentation.
|
||||||
- Documentation now covers the :ref:`release_process` for this package.
|
- Documentation now covers the :ref:`release_process` for this package.
|
||||||
|
|
||||||
.. _v3_29:
|
.. _v3_29:
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,7 @@ This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command.
|
||||||
"query", "memory", "insert", "upsert", "bulk", "search", "transform", "extract",
|
"query", "memory", "insert", "upsert", "bulk", "search", "transform", "extract",
|
||||||
"schema", "insert-files", "analyze-tables", "convert", "tables", "views", "rows",
|
"schema", "insert-files", "analyze-tables", "convert", "tables", "views", "rows",
|
||||||
"triggers", "indexes", "create-database", "create-table", "create-index",
|
"triggers", "indexes", "create-database", "create-table", "create-index",
|
||||||
"enable-fts", "populate-fts", "rebuild-fts", "disable-fts"
|
"migrate", "enable-fts", "populate-fts", "rebuild-fts", "disable-fts"
|
||||||
]
|
]
|
||||||
refs = {
|
refs = {
|
||||||
"query": "cli_query",
|
"query": "cli_query",
|
||||||
|
|
@ -49,6 +49,7 @@ This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command.
|
||||||
"enable-wal": "cli_wal",
|
"enable-wal": "cli_wal",
|
||||||
"enable-counts": "cli_enable_counts",
|
"enable-counts": "cli_enable_counts",
|
||||||
"bulk": "cli_bulk",
|
"bulk": "cli_bulk",
|
||||||
|
"migrate": "cli_migrate",
|
||||||
"create-database": "cli_create_database",
|
"create-database": "cli_create_database",
|
||||||
"create-table": "cli_create_table",
|
"create-table": "cli_create_table",
|
||||||
"drop-table": "cli_drop_table",
|
"drop-table": "cli_drop_table",
|
||||||
|
|
@ -965,6 +966,40 @@ See :ref:`cli_create_index`.
|
||||||
-h, --help Show this message and exit.
|
-h, --help Show this message and exit.
|
||||||
|
|
||||||
|
|
||||||
|
.. _cli_ref_migrate:
|
||||||
|
|
||||||
|
migrate
|
||||||
|
=======
|
||||||
|
|
||||||
|
See :ref:`cli_migrate`.
|
||||||
|
|
||||||
|
::
|
||||||
|
|
||||||
|
Usage: sqlite-utils migrate [OPTIONS] DB_PATH [MIGRATIONS]...
|
||||||
|
|
||||||
|
Apply pending database migrations.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
|
||||||
|
sqlite-utils migrate database.db
|
||||||
|
|
||||||
|
This will find the migrations.py file in the current directory or
|
||||||
|
subdirectories and apply any pending migrations.
|
||||||
|
|
||||||
|
Or pass paths to one or more migrations.py files directly:
|
||||||
|
|
||||||
|
sqlite-utils migrate database.db path/to/migrations.py
|
||||||
|
|
||||||
|
Pass --list to see a list of applied and pending migrations without applying
|
||||||
|
them.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--stop-before TEXT Stop before applying this migration
|
||||||
|
--list List migrations without running them
|
||||||
|
-v, --verbose Show verbose output
|
||||||
|
-h, --help Show this message and exit.
|
||||||
|
|
||||||
|
|
||||||
.. _cli_ref_enable_fts:
|
.. _cli_ref_enable_fts:
|
||||||
|
|
||||||
enable-fts
|
enable-fts
|
||||||
|
|
|
||||||
23
docs/cli.rst
23
docs/cli.rst
|
|
@ -1058,6 +1058,29 @@ That will look for SpatiaLite in a set of predictable locations. To load it from
|
||||||
|
|
||||||
sqlite-utils create-database empty.db --init-spatialite --load-extension /path/to/spatialite.so
|
sqlite-utils create-database empty.db --init-spatialite --load-extension /path/to/spatialite.so
|
||||||
|
|
||||||
|
.. _cli_migrate:
|
||||||
|
|
||||||
|
Running migrations
|
||||||
|
==================
|
||||||
|
|
||||||
|
The ``migrate`` command applies pending Python migrations to a database. For the full migration file format and Python API, see :ref:`migrations`.
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sqlite-utils migrate creatures.db path/to/migrations.py
|
||||||
|
|
||||||
|
If you omit the migration path it will search the current directory and subdirectories for files called ``migrations.py``:
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sqlite-utils migrate creatures.db
|
||||||
|
|
||||||
|
Use ``--list`` to list applied and pending migrations without running them:
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sqlite-utils migrate creatures.db --list
|
||||||
|
|
||||||
.. _cli_inserting_data:
|
.. _cli_inserting_data:
|
||||||
|
|
||||||
Inserting JSON data
|
Inserting JSON data
|
||||||
|
|
|
||||||
|
|
@ -36,6 +36,7 @@ Contents
|
||||||
installation
|
installation
|
||||||
cli
|
cli
|
||||||
python-api
|
python-api
|
||||||
|
migrations
|
||||||
plugins
|
plugins
|
||||||
reference
|
reference
|
||||||
cli-reference
|
cli-reference
|
||||||
|
|
|
||||||
164
docs/migrations.rst
Normal file
164
docs/migrations.rst
Normal file
|
|
@ -0,0 +1,164 @@
|
||||||
|
.. _migrations:
|
||||||
|
|
||||||
|
====================
|
||||||
|
Database migrations
|
||||||
|
====================
|
||||||
|
|
||||||
|
``sqlite-utils`` includes a small migration system for applying repeatable changes to SQLite database files.
|
||||||
|
|
||||||
|
A migration is a Python function that receives a :class:`sqlite_utils.Database` instance. Migrations are grouped into named sets using the :class:`sqlite_utils.Migrations` class, and each applied migration is recorded in the ``_sqlite_migrations`` table in that database.
|
||||||
|
|
||||||
|
Applying migrations in Python
|
||||||
|
=============================
|
||||||
|
|
||||||
|
Create a :class:`sqlite_utils.Migrations` object, decorate migration functions with it and call ``.apply(db)`` against a :class:`sqlite_utils.Database` instance:
|
||||||
|
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
from sqlite_utils import Database, Migrations
|
||||||
|
|
||||||
|
migrations = Migrations("creatures")
|
||||||
|
|
||||||
|
@migrations()
|
||||||
|
def create_table(db):
|
||||||
|
db["creatures"].create(
|
||||||
|
{"id": int, "name": str, "species": str},
|
||||||
|
pk="id",
|
||||||
|
)
|
||||||
|
|
||||||
|
@migrations()
|
||||||
|
def add_weight(db):
|
||||||
|
db["creatures"].add_column("weight", float)
|
||||||
|
|
||||||
|
db = Database("creatures.db")
|
||||||
|
migrations.apply(db)
|
||||||
|
|
||||||
|
Running ``migrations.apply(db)`` repeatedly is safe. Migrations that already have a matching ``migration_set`` and ``name`` row in ``_sqlite_migrations`` will be skipped.
|
||||||
|
|
||||||
|
The name passed to ``Migrations("creatures")`` identifies that set of migrations. Use a name that is unique for your project, since multiple migration sets can be applied to the same database.
|
||||||
|
|
||||||
|
Migration functions are applied in the order their decorators run. The function name is used as the migration name unless you pass one explicitly:
|
||||||
|
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
@migrations(name="001_create_table")
|
||||||
|
def create_table(db):
|
||||||
|
db["creatures"].create({"id": int, "name": str}, pk="id")
|
||||||
|
|
||||||
|
You can also stop before a named migration:
|
||||||
|
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
migrations.apply(db, stop_before="add_weight")
|
||||||
|
|
||||||
|
Migration files
|
||||||
|
===============
|
||||||
|
|
||||||
|
The ``sqlite-utils migrate`` command looks for migration sets in Python files, usually named ``migrations.py``. A migration file should define one or more :class:`sqlite_utils.Migrations` objects:
|
||||||
|
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
from sqlite_utils import Migrations
|
||||||
|
|
||||||
|
migrations = Migrations("creatures")
|
||||||
|
|
||||||
|
@migrations()
|
||||||
|
def create_table(db):
|
||||||
|
db["creatures"].create(
|
||||||
|
{"id": int, "name": str, "species": str},
|
||||||
|
pk="id",
|
||||||
|
)
|
||||||
|
|
||||||
|
@migrations()
|
||||||
|
def add_weight(db):
|
||||||
|
db["creatures"].add_column("weight", float)
|
||||||
|
|
||||||
|
Applying migrations using the CLI
|
||||||
|
=================================
|
||||||
|
|
||||||
|
Run migrations using the ``sqlite-utils migrate`` command:
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sqlite-utils migrate creatures.db path/to/migrations.py
|
||||||
|
|
||||||
|
The first argument is the database file. The remaining arguments can be paths to migration files or directories containing migration files.
|
||||||
|
|
||||||
|
If you omit migration paths, ``sqlite-utils`` searches the current directory and subdirectories for files called ``migrations.py``:
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sqlite-utils migrate creatures.db
|
||||||
|
|
||||||
|
You can also pass a directory. Every ``migrations.py`` file in that directory tree will be considered:
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sqlite-utils migrate creatures.db path/to/project/
|
||||||
|
|
||||||
|
Running the command repeatedly is safe. Migrations that already have a matching ``migration_set`` and ``name`` row in ``_sqlite_migrations`` will be skipped.
|
||||||
|
|
||||||
|
Listing migrations
|
||||||
|
==================
|
||||||
|
|
||||||
|
Use ``--list`` to show applied and pending migrations without running them:
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sqlite-utils migrate creatures.db --list
|
||||||
|
|
||||||
|
Example output:
|
||||||
|
|
||||||
|
.. code-block:: output
|
||||||
|
|
||||||
|
Migrations for: creatures
|
||||||
|
|
||||||
|
Applied:
|
||||||
|
create_table - 2026-06-09 17:23:12.048092+00:00
|
||||||
|
add_weight - 2026-06-09 17:23:12.051249+00:00
|
||||||
|
|
||||||
|
Pending:
|
||||||
|
add_age
|
||||||
|
|
||||||
|
Stopping before a migration
|
||||||
|
===========================
|
||||||
|
|
||||||
|
When applying a single migration file, you can stop before a named migration:
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sqlite-utils migrate creatures.db path/to/migrations.py --stop-before add_weight
|
||||||
|
|
||||||
|
This applies any pending migrations before ``add_weight`` and leaves ``add_weight`` and later migrations pending.
|
||||||
|
|
||||||
|
Verbose output
|
||||||
|
==============
|
||||||
|
|
||||||
|
Use ``--verbose`` or ``-v`` to show the schema before and after migrations are applied, plus a unified diff when the schema changes:
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
sqlite-utils migrate creatures.db --verbose
|
||||||
|
|
||||||
|
Migrating from sqlite-migrate
|
||||||
|
=============================
|
||||||
|
|
||||||
|
This system uses the same migration table format as the separate ``sqlite-migrate`` package. To use existing migration files directly with ``sqlite-utils``, update their import from ``sqlite_migrate`` to ``sqlite_utils``:
|
||||||
|
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
from sqlite_utils import Migrations
|
||||||
|
|
||||||
|
migration = Migrations("creatures")
|
||||||
|
|
||||||
|
@migration()
|
||||||
|
def create_table(db):
|
||||||
|
db["creatures"].create({"id": int, "name": str}, pk="id")
|
||||||
|
|
||||||
|
Python API
|
||||||
|
==========
|
||||||
|
|
||||||
|
.. autoclass:: sqlite_utils.migrations.Migrations
|
||||||
|
:members:
|
||||||
|
:undoc-members:
|
||||||
|
:exclude-members: _Migration, _AppliedMigration
|
||||||
|
|
@ -2,5 +2,6 @@ from .utils import suggest_column_types
|
||||||
from .hookspecs import hookimpl
|
from .hookspecs import hookimpl
|
||||||
from .hookspecs import hookspec
|
from .hookspecs import hookspec
|
||||||
from .db import Database
|
from .db import Database
|
||||||
|
from .migrations import Migrations
|
||||||
|
|
||||||
__all__ = ["Database", "suggest_column_types", "hookimpl", "hookspec"]
|
__all__ = ["Database", "Migrations", "suggest_column_types", "hookimpl", "hookspec"]
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import base64
|
import base64
|
||||||
|
import difflib
|
||||||
from typing import Any
|
from typing import Any
|
||||||
import click
|
import click
|
||||||
from click_default_group import DefaultGroup
|
from click_default_group import DefaultGroup
|
||||||
|
|
@ -3252,6 +3253,125 @@ def create_spatial_index(db_path, table, column_name, load_extension):
|
||||||
db.table(table).create_spatial_index(column_name)
|
db.table(table).create_spatial_index(column_name)
|
||||||
|
|
||||||
|
|
||||||
|
def _find_migration_files(migrations):
|
||||||
|
if not migrations:
|
||||||
|
migrations = [pathlib.Path(".").resolve()]
|
||||||
|
files = set()
|
||||||
|
for path_str in migrations:
|
||||||
|
path = pathlib.Path(path_str)
|
||||||
|
if path.is_dir():
|
||||||
|
files.update(path.rglob("migrations.py"))
|
||||||
|
else:
|
||||||
|
files.add(path)
|
||||||
|
return sorted(files)
|
||||||
|
|
||||||
|
|
||||||
|
def _compatible_migration_set(obj):
|
||||||
|
return isinstance(obj, sqlite_utils.Migrations) or all(
|
||||||
|
hasattr(obj, attr) for attr in ("name", "applied", "pending", "apply")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _load_migration_sets(files):
|
||||||
|
migration_sets = []
|
||||||
|
for filepath in files:
|
||||||
|
code = filepath.read_text()
|
||||||
|
namespace = {
|
||||||
|
"__file__": str(filepath),
|
||||||
|
"__name__": "__sqlite_utils_migration__",
|
||||||
|
}
|
||||||
|
exec(code, namespace)
|
||||||
|
migration_sets.extend(
|
||||||
|
obj for obj in namespace.values() if _compatible_migration_set(obj)
|
||||||
|
)
|
||||||
|
return migration_sets
|
||||||
|
|
||||||
|
|
||||||
|
def _display_migration_list(db, migration_sets):
|
||||||
|
for migration_set in migration_sets:
|
||||||
|
click.echo("Migrations for: {}".format(migration_set.name))
|
||||||
|
click.echo()
|
||||||
|
click.echo(" Applied:")
|
||||||
|
for migration in migration_set.applied(db):
|
||||||
|
click.echo(" {} - {}".format(migration.name, migration.applied_at))
|
||||||
|
click.echo()
|
||||||
|
click.echo(" Pending:")
|
||||||
|
output = False
|
||||||
|
for migration in migration_set.pending(db):
|
||||||
|
output = True
|
||||||
|
click.echo(" {}".format(migration.name))
|
||||||
|
if not output:
|
||||||
|
click.echo(" (none)")
|
||||||
|
click.echo()
|
||||||
|
|
||||||
|
|
||||||
|
@click.command()
|
||||||
|
@click.argument(
|
||||||
|
"db_path", type=click.Path(dir_okay=False, readable=True, writable=True)
|
||||||
|
)
|
||||||
|
@click.argument("migrations", type=click.Path(dir_okay=True, exists=True), nargs=-1)
|
||||||
|
@click.option("--stop-before", help="Stop before applying this migration")
|
||||||
|
@click.option(
|
||||||
|
"list_", "--list", is_flag=True, help="List migrations without running them"
|
||||||
|
)
|
||||||
|
@click.option("-v", "--verbose", is_flag=True, help="Show verbose output")
|
||||||
|
def migrate(db_path, migrations, stop_before, list_, verbose):
|
||||||
|
"""
|
||||||
|
Apply pending database migrations.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
|
||||||
|
sqlite-utils migrate database.db
|
||||||
|
|
||||||
|
This will find the migrations.py file in the current directory
|
||||||
|
or subdirectories and apply any pending migrations.
|
||||||
|
|
||||||
|
Or pass paths to one or more migrations.py files directly:
|
||||||
|
|
||||||
|
sqlite-utils migrate database.db path/to/migrations.py
|
||||||
|
|
||||||
|
Pass --list to see a list of applied and pending migrations
|
||||||
|
without applying them.
|
||||||
|
"""
|
||||||
|
files = _find_migration_files(migrations)
|
||||||
|
migration_sets = _load_migration_sets(files)
|
||||||
|
if not migration_sets:
|
||||||
|
raise click.ClickException("No migrations.py files found")
|
||||||
|
|
||||||
|
if stop_before and len(migration_sets) > 1:
|
||||||
|
raise click.ClickException(
|
||||||
|
"--stop-before can only be used with a single migrations.py file"
|
||||||
|
)
|
||||||
|
|
||||||
|
db = sqlite_utils.Database(db_path)
|
||||||
|
_register_db_for_cleanup(db)
|
||||||
|
|
||||||
|
if list_:
|
||||||
|
_display_migration_list(db, migration_sets)
|
||||||
|
return
|
||||||
|
|
||||||
|
prev_schema = db.schema
|
||||||
|
if verbose:
|
||||||
|
click.echo("Migrating {}".format(db_path))
|
||||||
|
click.echo("\nSchema before:\n")
|
||||||
|
click.echo(textwrap.indent(prev_schema, " ") or " (empty)")
|
||||||
|
click.echo()
|
||||||
|
for migration_set in migration_sets:
|
||||||
|
migration_set.apply(db, stop_before=stop_before)
|
||||||
|
if verbose:
|
||||||
|
click.echo("Schema after:\n")
|
||||||
|
post_schema = db.schema
|
||||||
|
if post_schema == prev_schema:
|
||||||
|
click.echo(" (unchanged)")
|
||||||
|
else:
|
||||||
|
click.echo(textwrap.indent(post_schema, " "))
|
||||||
|
click.echo("\nSchema diff:\n")
|
||||||
|
diff = list(
|
||||||
|
difflib.unified_diff(prev_schema.splitlines(), post_schema.splitlines())
|
||||||
|
)
|
||||||
|
click.echo("\n".join(diff[3:]))
|
||||||
|
|
||||||
|
|
||||||
@cli.command(name="plugins")
|
@cli.command(name="plugins")
|
||||||
def plugins_list():
|
def plugins_list():
|
||||||
"List installed plugins"
|
"List installed plugins"
|
||||||
|
|
@ -3259,6 +3379,7 @@ def plugins_list():
|
||||||
|
|
||||||
|
|
||||||
pm.hook.register_commands(cli=cli)
|
pm.hook.register_commands(cli=cli)
|
||||||
|
cli.add_command(migrate)
|
||||||
|
|
||||||
|
|
||||||
def _render_common(title, values):
|
def _render_common(title, values):
|
||||||
|
|
|
||||||
119
sqlite_utils/migrations.py
Normal file
119
sqlite_utils/migrations.py
Normal file
|
|
@ -0,0 +1,119 @@
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import datetime
|
||||||
|
from typing import Callable, cast, TYPE_CHECKING
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from sqlite_utils.db import Database, Table
|
||||||
|
|
||||||
|
|
||||||
|
class Migrations:
|
||||||
|
migrations_table = "_sqlite_migrations"
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _Migration:
|
||||||
|
name: str
|
||||||
|
fn: Callable
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _AppliedMigration:
|
||||||
|
name: str
|
||||||
|
applied_at: datetime.datetime
|
||||||
|
|
||||||
|
def __init__(self, name: str):
|
||||||
|
"""
|
||||||
|
:param name: The name of the migration set. This should be unique.
|
||||||
|
"""
|
||||||
|
self.name = name
|
||||||
|
self._migrations: list[Migrations._Migration] = []
|
||||||
|
|
||||||
|
def __call__(self, *, name: str | None = None) -> Callable:
|
||||||
|
"""
|
||||||
|
:param name: The name to use for this migration - if not provided,
|
||||||
|
the name of the function will be used.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def inner(func: Callable) -> Callable:
|
||||||
|
self._migrations.append(
|
||||||
|
self._Migration(name or getattr(func, "__name__"), func)
|
||||||
|
)
|
||||||
|
return func
|
||||||
|
|
||||||
|
return inner
|
||||||
|
|
||||||
|
def pending(self, db: "Database") -> list["Migrations._Migration"]:
|
||||||
|
"""
|
||||||
|
Return a list of pending migrations.
|
||||||
|
"""
|
||||||
|
self.ensure_migrations_table(db)
|
||||||
|
already_applied = {
|
||||||
|
r["name"]
|
||||||
|
for r in db[self.migrations_table].rows_where(
|
||||||
|
"migration_set = ?", [self.name]
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return [
|
||||||
|
migration
|
||||||
|
for migration in self._migrations
|
||||||
|
if migration.name not in already_applied
|
||||||
|
]
|
||||||
|
|
||||||
|
def applied(self, db: "Database") -> list["Migrations._AppliedMigration"]:
|
||||||
|
"""
|
||||||
|
Return a list of applied migrations.
|
||||||
|
"""
|
||||||
|
self.ensure_migrations_table(db)
|
||||||
|
return [
|
||||||
|
self._AppliedMigration(name=row["name"], applied_at=row["applied_at"])
|
||||||
|
for row in db[self.migrations_table].rows_where(
|
||||||
|
"migration_set = ?", [self.name]
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
def apply(self, db: "Database", *, stop_before: str | None = None):
|
||||||
|
"""
|
||||||
|
Apply any pending migrations to the database.
|
||||||
|
"""
|
||||||
|
self.ensure_migrations_table(db)
|
||||||
|
for migration in self.pending(db):
|
||||||
|
name = migration.name
|
||||||
|
if name == stop_before:
|
||||||
|
return
|
||||||
|
migration.fn(db)
|
||||||
|
_table(db, self.migrations_table).insert(
|
||||||
|
{
|
||||||
|
"migration_set": self.name,
|
||||||
|
"name": name,
|
||||||
|
"applied_at": str(datetime.datetime.now(datetime.timezone.utc)),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
def ensure_migrations_table(self, db: "Database"):
|
||||||
|
"""
|
||||||
|
Ensure the _sqlite_migrations table exists and has the correct schema.
|
||||||
|
"""
|
||||||
|
table = _table(db, self.migrations_table)
|
||||||
|
if not table.exists():
|
||||||
|
table.create(
|
||||||
|
{
|
||||||
|
"id": int,
|
||||||
|
"migration_set": str,
|
||||||
|
"name": str,
|
||||||
|
"applied_at": str,
|
||||||
|
},
|
||||||
|
pk="id",
|
||||||
|
)
|
||||||
|
table.create_index(["migration_set", "name"], unique=True)
|
||||||
|
elif table.pks != ["id"]:
|
||||||
|
table.transform(pk="id")
|
||||||
|
unique_indexes = {tuple(index.columns) for index in table.indexes}
|
||||||
|
if ("migration_set", "name") not in unique_indexes:
|
||||||
|
table.create_index(["migration_set", "name"], unique=True)
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return "<Migrations '{}': [{}]>".format(
|
||||||
|
self.name, ", ".join(m.name for m in self._migrations)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _table(db: "Database", name: str) -> "Table":
|
||||||
|
return cast("Table", db[name])
|
||||||
227
tests/test_cli_migrate.py
Normal file
227
tests/test_cli_migrate.py
Normal file
|
|
@ -0,0 +1,227 @@
|
||||||
|
import pathlib
|
||||||
|
|
||||||
|
from click.testing import CliRunner
|
||||||
|
import pytest
|
||||||
|
import sqlite_utils
|
||||||
|
import sqlite_utils.cli
|
||||||
|
|
||||||
|
TWO_MIGRATIONS = """
|
||||||
|
from sqlite_utils import Migrations
|
||||||
|
|
||||||
|
m = Migrations("hello")
|
||||||
|
|
||||||
|
@m()
|
||||||
|
def foo(db):
|
||||||
|
db["foo"].insert({"hello": "world"})
|
||||||
|
|
||||||
|
@m()
|
||||||
|
def bar(db):
|
||||||
|
db["bar"].insert({"hello": "world"})
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def two_migrations(tmpdir):
|
||||||
|
path = pathlib.Path(tmpdir)
|
||||||
|
(path / "foo").mkdir()
|
||||||
|
migrations_py = path / "foo" / "migrations.py"
|
||||||
|
migrations_py.write_text(TWO_MIGRATIONS, "utf-8")
|
||||||
|
return path, migrations_py
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("arg", ("TMPDIR", "TMPDIR/foo/migrations.py", "TMPDIR/foo/"))
|
||||||
|
def test_basic(two_migrations, arg):
|
||||||
|
path, _ = two_migrations
|
||||||
|
db_path = str(path / "test.db")
|
||||||
|
|
||||||
|
runner = CliRunner()
|
||||||
|
|
||||||
|
def _list():
|
||||||
|
list_result = runner.invoke(
|
||||||
|
sqlite_utils.cli.cli,
|
||||||
|
["migrate", db_path, "--list", arg.replace("TMPDIR", str(path))],
|
||||||
|
)
|
||||||
|
assert list_result.exit_code == 0
|
||||||
|
return list_result.output
|
||||||
|
|
||||||
|
assert _list() == (
|
||||||
|
"Migrations for: hello\n\n"
|
||||||
|
" Applied:\n\n"
|
||||||
|
" Pending:\n"
|
||||||
|
" foo\n"
|
||||||
|
" bar\n\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
result = runner.invoke(
|
||||||
|
sqlite_utils.cli.cli, ["migrate", db_path, arg.replace("TMPDIR", str(path))]
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
|
||||||
|
list_output = _list()
|
||||||
|
assert "Migrations for: hello\n\n Applied:\n " in list_output
|
||||||
|
prior_to_pending = list_output.split(" Pending")[0]
|
||||||
|
assert " foo" in prior_to_pending
|
||||||
|
assert " bar" in prior_to_pending
|
||||||
|
assert " Pending:\n (none)" in list_output
|
||||||
|
|
||||||
|
db = sqlite_utils.Database(db_path)
|
||||||
|
assert db["foo"].exists()
|
||||||
|
assert db["bar"].exists()
|
||||||
|
assert db["_sqlite_migrations"].exists()
|
||||||
|
rows = list(db["_sqlite_migrations"].rows)
|
||||||
|
assert len(rows) == 2
|
||||||
|
assert rows[0]["name"] == "foo"
|
||||||
|
assert rows[1]["name"] == "bar"
|
||||||
|
|
||||||
|
|
||||||
|
def test_list_same_migration_names_in_different_sets(capsys):
|
||||||
|
applied = sqlite_utils.Migrations("applied")
|
||||||
|
|
||||||
|
@applied(name="foo")
|
||||||
|
def applied_foo(db):
|
||||||
|
db["applied"].insert({"hello": "world"})
|
||||||
|
|
||||||
|
pending = sqlite_utils.Migrations("pending")
|
||||||
|
|
||||||
|
@pending(name="foo")
|
||||||
|
def pending_foo(db):
|
||||||
|
db["pending"].insert({"hello": "world"})
|
||||||
|
|
||||||
|
db = sqlite_utils.Database(memory=True)
|
||||||
|
applied.apply(db)
|
||||||
|
|
||||||
|
sqlite_utils.cli._display_migration_list(db, [applied, pending])
|
||||||
|
|
||||||
|
output = capsys.readouterr().out
|
||||||
|
assert (
|
||||||
|
"Migrations for: pending\n\n" " Applied:\n\n" " Pending:\n" " foo\n\n"
|
||||||
|
) in output
|
||||||
|
|
||||||
|
|
||||||
|
def test_verbose(tmpdir):
|
||||||
|
path = pathlib.Path(tmpdir)
|
||||||
|
(path / "foo").mkdir()
|
||||||
|
migrations_py = path / "foo" / "migrations.py"
|
||||||
|
migrations_py.write_text(
|
||||||
|
"""
|
||||||
|
from sqlite_utils import Migrations
|
||||||
|
|
||||||
|
m = Migrations("hello")
|
||||||
|
|
||||||
|
@m()
|
||||||
|
def foo(db):
|
||||||
|
db["dogs"].insert({"id": 1, "name": "Cleo"})
|
||||||
|
""",
|
||||||
|
"utf-8",
|
||||||
|
)
|
||||||
|
db_path = str(path / "test.db")
|
||||||
|
runner = CliRunner()
|
||||||
|
result = runner.invoke(
|
||||||
|
sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py)]
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0
|
||||||
|
|
||||||
|
result = runner.invoke(
|
||||||
|
sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py), "--verbose"]
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0
|
||||||
|
expected = """
|
||||||
|
Schema before:
|
||||||
|
|
||||||
|
CREATE TABLE "_sqlite_migrations" (
|
||||||
|
"id" INTEGER PRIMARY KEY,
|
||||||
|
"migration_set" TEXT,
|
||||||
|
"name" TEXT,
|
||||||
|
"applied_at" TEXT
|
||||||
|
);
|
||||||
|
CREATE UNIQUE INDEX "idx__sqlite_migrations_migration_set_name"
|
||||||
|
ON "_sqlite_migrations" ("migration_set", "name");
|
||||||
|
CREATE TABLE "dogs" (
|
||||||
|
"id" INTEGER,
|
||||||
|
"name" TEXT
|
||||||
|
);
|
||||||
|
|
||||||
|
Schema after:
|
||||||
|
|
||||||
|
(unchanged)
|
||||||
|
""".strip()
|
||||||
|
assert expected in result.output
|
||||||
|
|
||||||
|
new_migration = """
|
||||||
|
@m()
|
||||||
|
def bar(db):
|
||||||
|
db["dogs"].add_column("age", int)
|
||||||
|
db["dogs"].add_column("weight", float)
|
||||||
|
db["dogs"].transform()
|
||||||
|
"""
|
||||||
|
migrations_py.write_text(migrations_py.read_text("utf-8") + new_migration)
|
||||||
|
|
||||||
|
result = runner.invoke(
|
||||||
|
sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py), "--verbose"]
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0
|
||||||
|
expected_diff = """
|
||||||
|
Schema diff:
|
||||||
|
|
||||||
|
ON "_sqlite_migrations" ("migration_set", "name");
|
||||||
|
CREATE TABLE "dogs" (
|
||||||
|
"id" INTEGER,
|
||||||
|
- "name" TEXT
|
||||||
|
+ "name" TEXT,
|
||||||
|
+ "age" INTEGER,
|
||||||
|
+ "weight" REAL
|
||||||
|
);
|
||||||
|
""".strip()
|
||||||
|
assert expected_diff in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_stop_before(two_migrations):
|
||||||
|
path, _ = two_migrations
|
||||||
|
db_path = str(path / "test.db")
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
sqlite_utils.cli.cli,
|
||||||
|
[
|
||||||
|
"migrate",
|
||||||
|
db_path,
|
||||||
|
str(path / "foo" / "migrations.py"),
|
||||||
|
"--stop-before",
|
||||||
|
"bar",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0
|
||||||
|
db = sqlite_utils.Database(db_path)
|
||||||
|
assert db["foo"].exists()
|
||||||
|
assert not db["bar"].exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_stop_before_error(two_migrations):
|
||||||
|
path, _ = two_migrations
|
||||||
|
db_path = str(path / "test.db")
|
||||||
|
(path / "foo" / "migrations2.py").write_text(
|
||||||
|
"""
|
||||||
|
from sqlite_utils import Migrations
|
||||||
|
|
||||||
|
m = Migrations("hello2")
|
||||||
|
|
||||||
|
@m()
|
||||||
|
def foo(db):
|
||||||
|
db["foo"].insert({"hello": "world"})
|
||||||
|
""",
|
||||||
|
"utf-8",
|
||||||
|
)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
sqlite_utils.cli.cli,
|
||||||
|
[
|
||||||
|
"migrate",
|
||||||
|
db_path,
|
||||||
|
str(path / "foo" / "migrations.py"),
|
||||||
|
str(path / "foo" / "migrations2.py"),
|
||||||
|
"--stop-before",
|
||||||
|
"foo",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 1
|
||||||
|
assert (
|
||||||
|
"--stop-before can only be used with a single migrations.py file"
|
||||||
|
in result.output
|
||||||
|
)
|
||||||
110
tests/test_migrations.py
Normal file
110
tests/test_migrations.py
Normal file
|
|
@ -0,0 +1,110 @@
|
||||||
|
import pytest
|
||||||
|
import sqlite_utils
|
||||||
|
from sqlite_utils import Migrations
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def migrations():
|
||||||
|
migrations = Migrations("test")
|
||||||
|
|
||||||
|
@migrations()
|
||||||
|
def m001(db):
|
||||||
|
db["dogs"].insert({"name": "Cleo"})
|
||||||
|
|
||||||
|
@migrations()
|
||||||
|
def m002(db):
|
||||||
|
db["cats"].create({"name": str})
|
||||||
|
db.query("insert into dogs (name) values ('Pancakes')")
|
||||||
|
|
||||||
|
return migrations
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def migrations_not_ordered_alphabetically():
|
||||||
|
# Names order alphabetically in the wrong direction but this
|
||||||
|
# should still be applied correctly.
|
||||||
|
migrations = Migrations("test")
|
||||||
|
|
||||||
|
@migrations()
|
||||||
|
def m002(db):
|
||||||
|
db["dogs"].insert({"name": "Cleo"})
|
||||||
|
|
||||||
|
@migrations()
|
||||||
|
def m001(db):
|
||||||
|
db["cats"].create({"name": str})
|
||||||
|
db.query("insert into dogs (name) values ('Pancakes')")
|
||||||
|
|
||||||
|
return migrations
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def migrations2():
|
||||||
|
migrations = Migrations("test2")
|
||||||
|
|
||||||
|
@migrations()
|
||||||
|
def m001(db):
|
||||||
|
db["dogs2"].insert({"name": "Cleo"})
|
||||||
|
|
||||||
|
return migrations
|
||||||
|
|
||||||
|
|
||||||
|
def test_basic(migrations):
|
||||||
|
db = sqlite_utils.Database(memory=True)
|
||||||
|
assert db.table_names() == []
|
||||||
|
migrations.apply(db)
|
||||||
|
assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_stop_before(migrations):
|
||||||
|
db = sqlite_utils.Database(memory=True)
|
||||||
|
assert db.table_names() == []
|
||||||
|
migrations.apply(db, stop_before="m002")
|
||||||
|
assert set(db.table_names()) == {"_sqlite_migrations", "dogs"}
|
||||||
|
migrations.apply(db)
|
||||||
|
assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_two_migration_sets(migrations, migrations2):
|
||||||
|
db = sqlite_utils.Database(memory=True)
|
||||||
|
assert db.table_names() == []
|
||||||
|
migrations.apply(db)
|
||||||
|
migrations2.apply(db)
|
||||||
|
assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats", "dogs2"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_order_does_not_matter(migrations, migrations_not_ordered_alphabetically):
|
||||||
|
db1 = sqlite_utils.Database(memory=True)
|
||||||
|
db2 = sqlite_utils.Database(memory=True)
|
||||||
|
migrations.apply(db1)
|
||||||
|
migrations_not_ordered_alphabetically.apply(db2)
|
||||||
|
assert db1.schema == db2.schema
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"create_table,pk",
|
||||||
|
(
|
||||||
|
(
|
||||||
|
{
|
||||||
|
"migration_set": str,
|
||||||
|
"name": str,
|
||||||
|
"applied_at": str,
|
||||||
|
},
|
||||||
|
"name",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
{
|
||||||
|
"migration_set": str,
|
||||||
|
"name": str,
|
||||||
|
"applied_at": str,
|
||||||
|
},
|
||||||
|
("migration_set", "name"),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
def test_upgrades_sqlite_migrations(migrations, create_table, pk):
|
||||||
|
db = sqlite_utils.Database(memory=True)
|
||||||
|
db["_sqlite_migrations"].create(create_table, pk=pk)
|
||||||
|
assert db.table_names() == ["_sqlite_migrations"]
|
||||||
|
assert db["_sqlite_migrations"].pks == ([pk] if isinstance(pk, str) else list(pk))
|
||||||
|
migrations.apply(db)
|
||||||
|
assert db["_sqlite_migrations"].pks == ["id"]
|
||||||
Loading…
Add table
Add a link
Reference in a new issue