From b95b6b63f6ca92895c07efe947ad4b7dba7f35b1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 10 Aug 2021 09:57:59 -0700 Subject: [PATCH] More docstrings plus configured autodoc, refs #311 --- docs/Makefile | 2 +- docs/conf.py | 3 +- docs/index.rst | 1 + docs/reference.rst | 55 +++++++++++++++++++++ sqlite_utils/db.py | 117 ++++++++++++++++++++++++++++++++++++++------- 5 files changed, 158 insertions(+), 20 deletions(-) create mode 100644 docs/reference.rst diff --git a/docs/Makefile b/docs/Makefile index a279768..5578ae3 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -20,4 +20,4 @@ help: @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) livehtml: - sphinx-autobuild -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0) + sphinx-autobuild -a -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0) --watch ../sqlite_utils diff --git a/docs/conf.py b/docs/conf.py index b4c7f44..1f5a158 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,7 +30,8 @@ from subprocess import Popen, PIPE # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["sphinx.ext.extlinks"] +extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc"] +autodoc_member_order = "bysource" extlinks = { "issue": ("https://github.com/simonw/sqlite-utils/issues/%s", "#"), diff --git a/docs/index.rst b/docs/index.rst index 93b0bc0..581f306 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -32,6 +32,7 @@ Contents installation cli python-api + reference contributing changelog diff --git a/docs/reference.rst b/docs/reference.rst new file mode 100644 index 0000000..7871270 --- /dev/null +++ b/docs/reference.rst @@ -0,0 +1,55 @@ +=========== + Reference +=========== + +.. contents:: :local: + +.. _reference_db_database: + +sqlit_utils.db.Database +======================= + +.. autoclass:: sqlite_utils.db.Database + :members: + :undoc-members: + :show-inheritance: + :special-members: __getitem__ + :exclude-members: use_counts_table + +.. _reference_db_queryable: + +sqlit_utils.db.Queryable +======================== + +.. autoclass:: sqlite_utils.db.Queryable + :members: + :undoc-members: + +.. _reference_db_table: + +sqlit_utils.db.Table +==================== + +.. autoclass:: sqlite_utils.db.Table + :members: + :undoc-members: + :show-inheritance: + +.. _reference_db_view: + +sqlit_utils.db.View +=================== + +.. autoclass:: sqlite_utils.db.View + :members: + :undoc-members: + :show-inheritance: + +.. _reference_db_other: + +Other +===== + +.. autoclass:: sqlite_utils.db.Column + +.. autoclass:: sqlite_utils.db.ColumnDetails diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index a21acc4..814bc22 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -21,7 +21,7 @@ import re from sqlite_fts4 import rank_bm25 # type: ignore import sys import textwrap -from typing import Generator, Iterable, Union, Optional, List +from typing import Callable, Dict, Generator, Iterable, Union, Optional, List, Tuple import uuid SQLITE_MAX_VARS = 999 @@ -49,7 +49,7 @@ USING\s+(?P\w+) # e.g. USING FTS5 try: import pandas as pd # type: ignore except ImportError: - pd = None + pd = None # type: ignore try: import numpy as np # type: ignore @@ -59,6 +59,9 @@ except ImportError: Column = namedtuple( "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk") ) +Column.__doc__ = ( + "Describes a SQLite column, returned by the :attr:`.Table.columns` property" +) ColumnDetails = namedtuple( "ColumnDetails", ( @@ -72,6 +75,7 @@ ColumnDetails = namedtuple( "least_common", ), ) +ColumnDetails.__doc__ = "Column analytics" ForeignKey = namedtuple( "ForeignKey", ("table", "column", "other_table", "other_column") ) @@ -133,26 +137,32 @@ if pd: class AlterError(Exception): + "Error altering table" pass class NoObviousTable(Exception): + "Could not tell which table this operation refers to" pass class BadPrimaryKey(Exception): + "Table does not have a single obvious primary key" pass class NotFoundError(Exception): + "Record not found" pass class PrimaryKeyRequired(Exception): + "Primary key needs to be specified" pass class InvalidColumns(Exception): + "Specified columns do not exist" pass @@ -176,17 +186,32 @@ CREATE TABLE IF NOT EXISTS [{}]( class Database: + """ + Wrapper for a SQLite database connection that adds a variety of useful utility methods. + + - ``filename_or_conn`` - String path to a file, or a ``pathlib.Path`` object, or a + ``sqlite3`` connection + - ``memory`` - set to ``True`` to create an in-memory database + - ``recreate`` - set to ``True`` to delete and recreate a file database (**dangerous**) + - ``recursive_triggers`` - defaults to ``True``, which sets ``PRAGMA recursive_triggers=on;`` - + set to ``False`` to avoid setting this pragma + - ``tracer``` - set a tracer function (``print`` works for this) which will be called with + ``sql, parameters`` every time a SQL query is executed + - ``use_counts_table`` - set to ``True`` to use a cached counts table, if available. See + :ref:`python_api_cached_table_counts`. + """ + _counts_table_name = "_counts" use_counts_table = False def __init__( self, filename_or_conn=None, - memory=False, - recreate=False, - recursive_triggers=True, - tracer=None, - use_counts_table=False, + memory: bool = False, + recreate: bool = False, + recursive_triggers: bool = True, + tracer: Callable = None, + use_counts_table: bool = False, ): assert (filename_or_conn is not None and not memory) or ( filename_or_conn is None and memory @@ -203,11 +228,24 @@ class Database: self._tracer = tracer if recursive_triggers: self.execute("PRAGMA recursive_triggers=on;") - self._registered_functions = set() + self._registered_functions: set = set() self.use_counts_table = use_counts_table @contextlib.contextmanager - def tracer(self, tracer=None): + def tracer(self, tracer: Callable = None): + """ + Context manager to temporarily set a tracer function - all executed SQL queries will + be passed to this. + + The tracer function should accept two arguments: ``sql`` and ``parameters`` + + Example usage:: + + with db.tracer(print): + db["creatures"].insert({"name": "Cleo"}) + + See :ref:`python_api_tracing`. + """ prev_tracer = self._tracer self._tracer = tracer or print try: @@ -215,13 +253,31 @@ class Database: finally: self._tracer = prev_tracer - def __getitem__(self, table_name): + def __getitem__(self, table_name: str): + """ + ``db[table_name]`` returns a :class:`.Table` object for the table with the specified name. + If the table does not exist yet it will be created the first time data is inserted into it. + """ return self.table(table_name) def __repr__(self): return "".format(self.conn) - def register_function(self, fn=None, deterministic=None, replace=False): + def register_function( + self, fn: Callable, deterministic: bool = False, replace: bool = False + ): + """ + ``fn`` will be made available as a function within SQL, with the same name and number + of arguments. Can be used as a decorator:: + + @db.register + def upper(value): + return str(value).upper() + + - ``deterministic`` - set ``True`` for functions that always returns the same output for a given input + - ``replace`` - set ``True`` to replace an existing function with the same name - otherwise throw an error + """ + def register(fn): name = fn.__name__ arity = len(inspect.signature(fn).parameters) @@ -240,9 +296,15 @@ class Database: register(fn) def register_fts4_bm25(self): + "Register the ``rank_bm25(match_info)`` function used for calculating relevance with SQLite FTS4" self.register_function(rank_bm25, deterministic=True) - def attach(self, alias, filepath): + def attach(self, alias: str, filepath: Union[str, pathlib.Path]): + """ + Attach another SQLite database file to this connection with the specified alias, equivalent to:: + + ATTACH DATABASE 'filepath.db' AS alias + """ attach_sql = """ ATTACH DATABASE '{}' AS [{}]; """.format( @@ -251,6 +313,7 @@ class Database: self.execute(attach_sql) def execute(self, sql, parameters=None): + "Execute SQL query and return a ``sqlite3.Cursor``" if self._tracer: self._tracer(sql, parameters) if parameters is not None: @@ -259,15 +322,18 @@ class Database: return self.conn.execute(sql) def executescript(self, sql): + "Execute multiple SQL statements separated by ; and return the ``sqlite3.Cursor``" if self._tracer: self._tracer(sql, None) return self.conn.executescript(sql) def table(self, table_name, **kwargs): + "Return a table object, optionally configured with default options" klass = View if table_name in self.view_names() else Table return klass(self, table_name, **kwargs) def quote(self, value): + "Apply SQLite string quoting to a value, including wrappping it in single quotes" # Normally we would use .execute(sql, [params]) for escaping, but # occasionally that isn't available - most notable when we need # to include a "... DEFAULT 'value'" in a column definition. @@ -278,6 +344,7 @@ class Database: ).fetchone()[0] def table_names(self, fts4=False, fts5=False): + "A list of string table names in this database" where = ["type = 'table'"] if fts4: where.append("sql like '%USING FTS4%'") @@ -287,6 +354,7 @@ class Database: return [r[0] for r in self.execute(sql).fetchall()] def view_names(self): + "A list of string view names in this database" return [ r[0] for r in self.execute( @@ -295,15 +363,18 @@ class Database: ] @property - def tables(self): + def tables(self) -> List[Table]: + "A list of Table objects in this database" return [self[name] for name in self.table_names()] @property - def views(self): + def views(self) -> List[str]: + "A list of View objects in this database" return [self[name] for name in self.view_names()] @property - def triggers(self): + def triggers(self) -> List[Trigger]: + "A list of ``(name, table_name, sql)`` tuples representing triggers in this database" return [ Trigger(*r) for r in self.execute( @@ -313,11 +384,12 @@ class Database: @property def triggers_dict(self): - "Returns {trigger_name: sql} dictionary" + "A {trigger_name: sql} dictionary for triggers in this database" return {trigger.name: trigger.sql for trigger in self.triggers} @property def schema(self): + "SQL schema of this database" sqls = [] for row in self.execute( "select sql from sqlite_master where sql is not null" @@ -330,13 +402,16 @@ class Database: @property def journal_mode(self): + "Current journal_mode of this database" return self.execute("PRAGMA journal_mode;").fetchone()[0] def enable_wal(self): + "Set journal_mode to 'wal' to enable Write-Ahead Log mode" if self.journal_mode != "wal": self.execute("PRAGMA journal_mode=wal;") def disable_wal(self): + "Set journal_mode to 'delete' to disable Write-Ahead Log mode" if self.journal_mode != "delete": self.execute("PRAGMA journal_mode=delete;") @@ -597,7 +672,13 @@ class Database: candidates.append(table.name) return candidates - def add_foreign_keys(self, foreign_keys): + def add_foreign_keys(self, foreign_keys: Iterable[Tuple[str, str, str, str]]): + """ + Add multiple foreign keys at once. + + ``foreign_keys`` should be a list of ``(table, column, other_table, other_column)`` + tuples, see :ref:`python_api_add_foreign_keys`. + """ # foreign_keys is a list of explicit 4-tuples assert all( len(fk) == 4 and isinstance(fk, (list, tuple)) for fk in foreign_keys @@ -633,7 +714,7 @@ class Database: ) # Construct SQL for use with "UPDATE sqlite_master SET sql = ? WHERE name = ?" - table_sql = {} + table_sql: Dict[str, str] = {} for table, column, other_table, other_column in foreign_keys_to_create: old_sql = table_sql.get(table, self[table].schema) extra_sql = ",\n FOREIGN KEY([{column}]) REFERENCES [{other_table}]([{other_column}])\n".format(