From a9fd1e785f42a34f3dc63d7532e22ca3f52941dc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Jul 2019 12:22:43 -0700 Subject: [PATCH 01/12] Initial implementation of table.get(...) --- sqlite_utils/db.py | 27 +++++++++++++++++++++++++++ tests/test_get.py | 14 ++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 9b179ac..d7fe44a 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -78,6 +78,9 @@ class NoObviousTable(Exception): class BadPrimaryKey(Exception): pass +class NotFoundError(Exception): + pass + class Database: def __init__(self, filename_or_conn): @@ -382,6 +385,30 @@ class Table: def pks(self): return [column.name for column in self.columns if column.is_pk] + def get(self, pk_values): + if not isinstance(pk_values, (list, tuple)): + pk_values = [pk_values] + pks = self.pks + pk_names = [] + if len(pks) == 0: + # rowid table + pk_names = ["rowid"] + last_pk = pk_values[0] + elif len(pks) == 1: + pk_names = [pks[0]] + last_pk = pk_values[0] + elif len(pks) > 1: + pk_names = pks + last_pk = pk_values + wheres = ["[{}] = ?".format(pk_name) for pk_name in pk_names] + rows = self.rows_where(' and '.join(wheres), pk_values) + try: + row = list(rows)[0] + self.last_pk = last_pk + return row + except IndexError: + raise NotFoundError + @property def foreign_keys(self): fks = [] diff --git a/tests/test_get.py b/tests/test_get.py index 75ddaef..bd79490 100644 --- a/tests/test_get.py +++ b/tests/test_get.py @@ -1,6 +1,20 @@ import pytest +def test_get_rowid(fresh_db): + dogs = fresh_db["dogs"] + cleo = {"name": "Cleo", "age": 4} + row_id = dogs.insert(cleo).last_rowid + assert cleo == dogs.get(row_id) + + +def test_get_primary_key(fresh_db): + dogs = fresh_db["dogs"] + cleo = {"name": "Cleo", "age": 4, "id": 5} + row_id = dogs.insert(cleo, pk="id").last_pk + assert cleo == dogs.get(5) + + @pytest.mark.parametrize( "where,where_args,expected_ids", [ From 9ff905937f3fd63e114f9ff5b2ee6f2b59256460 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Jul 2019 12:23:31 -0700 Subject: [PATCH 02/12] Use pip install -e .[docs] for documentation dependencies --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0a40728..368522c 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,10 @@ setup( packages=find_packages(exclude="tests"), install_requires=["click", "click-default-group", "tabulate"], setup_requires=["pytest-runner"], - extras_require={"test": ["pytest", "black"]}, + extras_require={ + "test": ["pytest", "black"], + "docs": ["sphinx_rtd_theme", "sphinx-autobuild"], + }, entry_points=""" [console_scripts] sqlite-utils=sqlite_utils.cli:cli From 0fcb1e4839e1313af0a386098996549eeed29890 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Jul 2019 12:24:45 -0700 Subject: [PATCH 03/12] Sphinx docs now pull version info from git tag Based on http://dreamiurg.net/2011/10/03/using-git-to-get-version-information/ --- docs/conf.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 90aed1d..33c4153 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# + +from subprocess import Popen, PIPE + # This file is execfile()d with the current directory set to its # containing dir. # @@ -52,9 +54,15 @@ author = "Simon Willison" # built documents. # # The short X.Y version. -version = "" -# The full version, including alpha/beta/rc tags. -release = "" +pipe = Popen("git describe --tags --always", stdout=PIPE, shell=True) +git_version = pipe.stdout.read().decode("utf8") + +if git_version: + version = git_version + release = git_version +else: + version = "" + release = "" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 90edd0d8174e23fc1e07d23d9c8d6a628c4d12d2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Jul 2019 12:45:00 -0700 Subject: [PATCH 04/12] Show version in docs Thanks, @nedbat https://twitter.com/nedbat/status/1150490436114534400 --- docs/conf.py | 2 +- docs/index.rst | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 33c4153..8429616 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,7 +58,7 @@ pipe = Popen("git describe --tags --always", stdout=PIPE, shell=True) git_version = pipe.stdout.read().decode("utf8") if git_version: - version = git_version + version = git_version.rsplit("-", 1)[0] release = git_version else: version = "" diff --git a/docs/index.rst b/docs/index.rst index 2a8d4c5..f50d76e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,6 +1,6 @@ -============== - sqlite-utils -============== +======================= + sqlite-utils |version| +======================= *Python utility functions for manipulating SQLite databases* From d5dc92876ec97cb94eae2db0b8169612e15d98f7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Jul 2019 20:42:16 -0700 Subject: [PATCH 05/12] create_table now handles compound primary keys, closes #36 --- sqlite_utils/db.py | 35 +++++++++++++++++++++++------------ tests/test_create.py | 22 ++++++++++++++++++++++ 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d7fe44a..4f2b906 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -78,6 +78,7 @@ class NoObviousTable(Exception): class BadPrimaryKey(Exception): pass + class NotFoundError(Exception): pass @@ -208,11 +209,15 @@ class Database: raise AlterError( "No such column: {}.{}".format(fk.other_table, fk.other_column) ) - extra = "" + column_defs = [] + # ensure pk is a tuple + single_pk = None + if isinstance(pk, str): + single_pk = pk for column_name, column_type in column_items: column_extras = [] - if pk == column_name: + if column_name == single_pk: column_extras.append("PRIMARY KEY") if column_name in not_null: column_extras.append("NOT NULL") @@ -236,12 +241,17 @@ class Database: else "", ) ) + extra_pk = "" + if single_pk is None and pk and len(pk) > 1: + extra_pk = ",\n PRIMARY KEY ({pks})".format( + pks=", ".join(["[{}]".format(p) for p in pk]) + ) columns_sql = ",\n".join(column_defs) sql = """CREATE TABLE [{table}] ( -{columns_sql} -){extra}; +{columns_sql}{extra_pk} +); """.format( - table=name, columns_sql=columns_sql, extra=extra + table=name, columns_sql=columns_sql, extra_pk=extra_pk ) self.conn.execute(sql) return self[name] @@ -401,7 +411,7 @@ class Table: pk_names = pks last_pk = pk_values wheres = ["[{}] = ?".format(pk_name) for pk_name in pk_names] - rows = self.rows_where(' and '.join(wheres), pk_values) + rows = self.rows_where(" and ".join(wheres), pk_values) try: row = list(rows)[0] self.last_pk = last_pk @@ -813,12 +823,13 @@ class Table: self.last_pk = None # self.last_rowid will be 0 if a "INSERT OR IGNORE" happened if (hash_id or pk) and self.last_rowid: - self.last_pk = self.db.conn.execute( - "select [{}] from [{}] where rowid = ?".format( - hash_id or pk, self.name - ), - (self.last_rowid,), - ).fetchone()[0] + row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] + if hash_id: + self.last_pk = row[hash_id] + elif isinstance(pk, str): + self.last_pk = row[pk] + else: + self.last_pk = tuple(row[p] for p in pk) return self def upsert( diff --git a/tests/test_create.py b/tests/test_create.py index 5e76d0c..91ad169 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -55,6 +55,21 @@ def test_create_table(fresh_db): ) == table.schema +def test_create_table_compound_primary_key(fresh_db): + table = fresh_db.create_table( + "test_table", {"id1": str, "id2": str, "value": int}, pk=("id1", "id2") + ) + assert ( + "CREATE TABLE [test_table] (\n" + " [id1] TEXT,\n" + " [id2] TEXT,\n" + " [value] INTEGER,\n" + " PRIMARY KEY ([id1], [id2])\n" + ")" + ) == table.schema + assert ["id1", "id2"] == table.pks + + def test_create_table_with_bad_defaults(fresh_db): with pytest.raises(AssertionError): fresh_db.create_table( @@ -122,6 +137,13 @@ def test_create_table_from_example(fresh_db, example, expected_columns): ] +def test_create_table_from_example_with_compound_primary_keys(fresh_db): + record = {"name": "Zhang", "group": "staff", "employee_id": 2} + table = fresh_db["people"].insert(record, pk=("group", "employee_id")) + assert ["group", "employee_id"] == table.pks + assert record == table.get(("staff", 2)) + + def test_create_table_column_order(fresh_db): fresh_db["table"].insert( collections.OrderedDict( From f47b5a9e1954bc6aefcca4a822b24f289c25f1af Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Jul 2019 20:57:57 -0700 Subject: [PATCH 06/12] CLI now accepts multiple --pk for compound primary keys --- sqlite_utils/cli.py | 4 +++- tests/test_cli.py | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 1c8661f..34228e4 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -302,7 +302,7 @@ def insert_upsert_options(fn): ), click.argument("table"), click.argument("json_file", type=click.File(), required=True), - click.option("--pk", help="Column to use as the primary key, e.g. id"), + click.option("--pk", help="Column to use as the primary key, e.g. id", multiple=True), click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"), click.option("-c", "--csv", is_flag=True, help="Expect CSV"), click.option( @@ -348,6 +348,8 @@ def insert_upsert_implementation( if nl and csv: click.echo("Use just one of --nl and --csv", err=True) return + if pk and len(pk) == 1: + pk = pk[0] if csv: reader = csv_std.reader(json_file) headers = next(reader) diff --git a/tests/test_cli.py b/tests/test_cli.py index 47afbb0..5526d4d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -427,6 +427,28 @@ def test_insert_multiple_with_primary_key(db_path, tmpdir): assert ["id"] == db["dogs"].pks +def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): + json_path = str(tmpdir / "dogs.json") + dogs = [{"breed": "mixed", "id": i, "name": "Cleo {}".format(i), "age": i + 3} for i in range(1, 21)] + open(json_path, "w").write(json.dumps(dogs)) + result = CliRunner().invoke( + cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--pk", "breed"] + ) + assert 0 == result.exit_code + db = Database(db_path) + assert dogs == db.execute_returning_dicts("select * from dogs order by breed, id") + assert {"breed", "id"} == set(db["dogs"].pks) + assert ( + "CREATE TABLE [dogs] (\n" + " [breed] TEXT,\n" + " [id] INTEGER,\n" + " [name] TEXT,\n" + " [age] INTEGER,\n" + " PRIMARY KEY ([id], [breed])\n" + ")" + ) == db["dogs"].schema + + def test_insert_not_null_default(db_path, tmpdir): json_path = str(tmpdir / "dogs.json") dogs = [ From be9ff6494002a16351938dac4ddd4a8a631c75aa Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Jul 2019 21:00:12 -0700 Subject: [PATCH 07/12] Docs for compound primary keys with CLI --- docs/cli.rst | 2 ++ sqlite_utils/cli.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index db1954c..cb9574c 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -200,6 +200,8 @@ Here's the simplest possible example:: To specify a column as the primary key, use ``--pk=column_name``. +To create a compound primary key across more than one column, use ``--pk`` multiple times. + If you feed it a JSON list it will insert multiple records. For example, if ``dogs.json`` looks like this:: [ diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 34228e4..c902295 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -302,7 +302,7 @@ def insert_upsert_options(fn): ), click.argument("table"), click.argument("json_file", type=click.File(), required=True), - click.option("--pk", help="Column to use as the primary key, e.g. id", multiple=True), + click.option("--pk", help="Columns to use as the primary key, e.g. id", multiple=True), click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"), click.option("-c", "--csv", is_flag=True, help="Expect CSV"), click.option( From e51bd09a35d5c14f6aa5bdcc76262c14b5838d84 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Jul 2019 21:03:57 -0700 Subject: [PATCH 08/12] Python library docs for compound primary keys --- docs/python-api.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index 0cc5007..1575349 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -147,6 +147,24 @@ The first argument here is a dictionary specifying the columns you would like to This method takes optional arguments ``pk=``, ``column_order=``, ``foreign_keys=``, ``not_null=set()`` and ``defaults=dict()`` - explained below. +.. _python_api_compound_primary_keys: + +Compound primary keys +--------------------- + +If you want to create a table with a compound primary key that spans multiple columns, you can do so by passing a tuple of column names to any of the methods that accept a ``pk=`` parameter. For example: + +.. code-block:: python + + db["cats"].create({ + "id": int, + "breed": str, + "name": str, + "weight": float, + }, pk=("breed", "id")) + +This also works for the ``.insert()``, ``.insert_all()``, ``.upsert()`` and ``.upsert_all()`` methods. + .. _python_api_foreign_keys: Specifying foreign keys From 71a3bc26c97afbd890de0c78ece0a1bec8773e39 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Jul 2019 21:04:43 -0700 Subject: [PATCH 09/12] Applied Black --- sqlite_utils/cli.py | 4 +++- tests/test_cli.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c902295..d53a77c 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -302,7 +302,9 @@ def insert_upsert_options(fn): ), click.argument("table"), click.argument("json_file", type=click.File(), required=True), - click.option("--pk", help="Columns to use as the primary key, e.g. id", multiple=True), + click.option( + "--pk", help="Columns to use as the primary key, e.g. id", multiple=True + ), click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"), click.option("-c", "--csv", is_flag=True, help="Expect CSV"), click.option( diff --git a/tests/test_cli.py b/tests/test_cli.py index 5526d4d..0e01814 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -429,7 +429,10 @@ def test_insert_multiple_with_primary_key(db_path, tmpdir): def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): json_path = str(tmpdir / "dogs.json") - dogs = [{"breed": "mixed", "id": i, "name": "Cleo {}".format(i), "age": i + 3} for i in range(1, 21)] + dogs = [ + {"breed": "mixed", "id": i, "name": "Cleo {}".format(i), "age": i + 3} + for i in range(1, 21) + ] open(json_path, "w").write(json.dumps(dogs)) result = CliRunner().invoke( cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--pk", "breed"] From 1445a8ecb576f9d803a426971663afe83801ee9d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Jul 2019 21:21:26 -0700 Subject: [PATCH 10/12] Tests for .get() NotFoundError --- sqlite_utils/db.py | 7 +++++++ tests/test_get.py | 18 +++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 4f2b906..d40951e 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -410,6 +410,13 @@ class Table: elif len(pks) > 1: pk_names = pks last_pk = pk_values + if len(pk_names) != len(pk_values): + raise NotFoundError( + "Need {} primary key value{}".format( + len(pk_names), "" if len(pk_names) == 1 else "s" + ) + ) + wheres = ["[{}] = ?".format(pk_name) for pk_name in pk_names] rows = self.rows_where(" and ".join(wheres), pk_values) try: diff --git a/tests/test_get.py b/tests/test_get.py index bd79490..ae5ffdf 100644 --- a/tests/test_get.py +++ b/tests/test_get.py @@ -1,4 +1,5 @@ import pytest +from sqlite_utils.db import NotFoundError def test_get_rowid(fresh_db): @@ -11,10 +12,25 @@ def test_get_rowid(fresh_db): def test_get_primary_key(fresh_db): dogs = fresh_db["dogs"] cleo = {"name": "Cleo", "age": 4, "id": 5} - row_id = dogs.insert(cleo, pk="id").last_pk + last_pk = dogs.insert(cleo, pk="id").last_pk + assert 5 == last_pk assert cleo == dogs.get(5) +@pytest.mark.parametrize( + "argument,expected_msg", + [(100, None), (None, None), ((1, 2), "Need 1 primary key value"), ("2", None)], +) +def test_get_not_found(argument, expected_msg, fresh_db): + fresh_db["dogs"].insert( + {"id": 1, "name": "Cleo", "age": 4, "is_good": True}, pk="id" + ) + with pytest.raises(NotFoundError) as excinfo: + fresh_db["dogs"].get(argument) + if expected_msg is not None: + assert expected_msg == excinfo.value.args[0] + + @pytest.mark.parametrize( "where,where_args,expected_ids", [ From b5a5df6d0ed47f33f6e1b4873948ead9a7c71060 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Jul 2019 21:24:42 -0700 Subject: [PATCH 11/12] Documentation for .get() --- docs/python-api.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index 1575349..287bdcb 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -72,6 +72,21 @@ You can filter rows by a WHERE clause using ``.rows_where(where, where_args)``:: ... print(row) {'id': 1, 'age': 4, 'name': 'Cleo'} +.. _python_api_get: + +Retrieving a specific record +============================ + +You can retrieve a record by its primary key using ``table.get()``:: + + >>> db = sqlite_utils.Database("dogs.db") + >>> print(db["dogs"].get(1)) + {'id': 1, 'age': 4, 'name': 'Cleo'} + +If the table has a compound primary key you can pass in the primary key values as a tuple:: + + >>> db["compound_dogs"].get(("mixed", 3)) + Creating tables =============== From 3f1e6cf12e347e98189eb963a9e64180f30b3a40 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Jul 2019 21:34:49 -0700 Subject: [PATCH 12/12] Release 1.5 --- docs/changelog.rst | 12 ++++++++++++ setup.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index dca16c9..8c49b60 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,18 @@ Changelog =========== +.. _v1_5: + +1.5 (2019-07-14) +---------------- + +- Support for compound primary keys (`#36 `__) + + - Configure these using the CLI tool by passing ``--pk`` multiple times + - In Python, pass a tuple of columns to the ``pk=(..., ...)`` argument: see :ref:`python_api_compound_primary_keys` + +- New ``table.get()`` method for retrieving a record by its primary key (`#39 `__) - :ref:`documentation ` + .. _v1_4_1: 1.4.1 (2019-07-14) diff --git a/setup.py b/setup.py index 368522c..077bb20 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "1.4.1" +VERSION = "1.5" def get_long_description():