From 17cbbb1f7f230b39650afac62dd16476626001b5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 30 Nov 2020 16:28:02 -0800 Subject: [PATCH] generated_columns table in fixtures.py, closes #1119 --- datasette/utils/__init__.py | 4 +- datasette/utils/sqlite.py | 8 ++++ tests/fixtures.py | 19 ++++++++- tests/test_api.py | 51 ++++++++++++----------- tests/test_internals_database.py | 70 +++++++++++++++++--------------- tests/test_plugins.py | 6 +-- 6 files changed, 93 insertions(+), 65 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index b951539d..2576090a 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -19,7 +19,7 @@ import urllib import numbers import yaml from .shutil_backport import copytree -from .sqlite import sqlite3, sqlite_version +from .sqlite import sqlite3, sqlite_version, supports_table_xinfo from ..plugins import pm @@ -561,7 +561,7 @@ def table_columns(conn, table): def table_column_details(conn, table): - if sqlite_version() >= (3, 26, 0): + if supports_table_xinfo(): # table_xinfo was added in 3.26.0 return [ Column(*r) diff --git a/datasette/utils/sqlite.py b/datasette/utils/sqlite.py index 9a043ccd..c8522f35 100644 --- a/datasette/utils/sqlite.py +++ b/datasette/utils/sqlite.py @@ -26,3 +26,11 @@ def _sqlite_version(): .split("."), ) ) + + +def supports_table_xinfo(): + return sqlite_version() >= (3, 26, 0) + + +def supports_generated_columns(): + return sqlite_version() >= (3, 31, 0) diff --git a/tests/fixtures.py b/tests/fixtures.py index b0c98f39..b52a531f 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,5 +1,5 @@ from datasette.app import Datasette -from datasette.utils.sqlite import sqlite3 +from datasette.utils.sqlite import sqlite3, sqlite_version, supports_generated_columns from datasette.utils.testing import TestClient import click import contextlib @@ -116,6 +116,8 @@ def make_app_client( immutables = [] conn = sqlite3.connect(filepath) conn.executescript(TABLES) + if supports_generated_columns(): + conn.executescript(GENERATED_COLUMNS_SQL) for sql, params in TABLE_PARAMETERIZED_SQL: with conn: conn.execute(sql, params) @@ -699,6 +701,18 @@ INSERT INTO "searchable_fts" (rowid, text1, text2) SELECT rowid, text1, text2 FROM searchable; """ +GENERATED_COLUMNS_SQL = """ +CREATE TABLE generated_columns ( + body TEXT, + id INT GENERATED ALWAYS AS (json_extract(body, '$.number')) STORED, + consideration INT GENERATED ALWAYS AS (json_extract(body, '$.string')) STORED +); +INSERT INTO generated_columns (body) VALUES ('{ + "number": 1, + "string": "This is a string" +}'); +""" + def assert_permissions_checked(datasette, actions): # actions is a list of "action" or (action, resource) tuples @@ -754,6 +768,9 @@ def cli(db_filename, metadata, plugins_path, recreate): for sql, params in TABLE_PARAMETERIZED_SQL: with conn: conn.execute(sql, params) + if supports_generated_columns(): + with conn: + conn.executescript(GENERATED_COLUMNS_SQL) print(f"Test tables written to {db_filename}") if metadata: open(metadata, "w").write(json.dumps(METADATA, indent=4)) diff --git a/tests/test_api.py b/tests/test_api.py index 5676622e..f82a8fe9 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,7 +1,7 @@ from datasette.app import Datasette from datasette.plugins import DEFAULT_PLUGINS from datasette.utils import detect_json1 -from datasette.utils.sqlite import sqlite3, sqlite_version +from datasette.utils.sqlite import sqlite3, sqlite_version, supports_table_xinfo from datasette.version import __version__ from .fixtures import ( # noqa app_client, @@ -19,6 +19,7 @@ from .fixtures import ( # noqa generate_compound_rows, generate_sortable_rows, make_app_client, + supports_generated_columns, EXPECTED_PLUGINS, METADATA, ) @@ -35,7 +36,7 @@ def test_homepage(app_client): assert response.json.keys() == {"fixtures": 0}.keys() d = response.json["fixtures"] assert d["name"] == "fixtures" - assert d["tables_count"] == 24 + assert d["tables_count"] == 25 if supports_generated_columns() else 24 assert len(d["tables_and_views_truncated"]) == 5 assert d["tables_and_views_more"] is True # 4 hidden FTS tables + no_primary_key (hidden in metadata) @@ -268,6 +269,22 @@ def test_database_page(app_client): }, "private": False, }, + ] + ( + [ + { + "columns": ["body", "id", "consideration"], + "count": 1, + "foreign_keys": {"incoming": [], "outgoing": []}, + "fts_table": None, + "hidden": False, + "name": "generated_columns", + "primary_keys": [], + "private": False, + } + ] + if supports_generated_columns() + else [] + ) + [ { "name": "infinity", "columns": ["value"], @@ -527,7 +544,7 @@ def test_database_page(app_client): "docid", "__langid", ] - if sqlite_version() >= (3, 26, 0) + if supports_table_xinfo() else [] ), "primary_keys": [], @@ -1934,31 +1951,13 @@ def test_paginate_using_link_header(app_client, qs): sqlite_version() < (3, 31, 0), reason="generated columns were added in SQLite 3.31.0", ) -@pytest.mark.asyncio -async def test_generated_columns_are_visible_in_datasette(tmp_path_factory): - db_directory = tmp_path_factory.mktemp("dbs") - db_path = db_directory / "test.db" - conn = sqlite3.connect(str(db_path)) - conn.executescript( - """ - CREATE TABLE deeds ( - body TEXT, - id INT GENERATED ALWAYS AS (json_extract(body, '$.id')) STORED, - consideration INT GENERATED ALWAYS AS (json_extract(body, '$.consideration')) STORED - ); - INSERT INTO deeds (body) VALUES ('{ - "id": 1, - "consideration": "This is the consideration" - }'); - """ - ) - datasette = Datasette([db_path]) - response = await datasette.client.get("/test/deeds.json?_shape=array") +async def test_generated_columns_are_visible_in_datasette(app_client): + response = app_client.get("/test/generated_columns.json?_shape=array") assert response.json() == [ { "rowid": 1, - "body": '{\n "id": 1,\n "consideration": "This is the consideration"\n }', - "id": 1, - "consideration": "This is the consideration", + "body": '{\n "number": 1,\n "string": "This is a string"\n }', + "number": 1, + "string": "This is a string", } ] diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index e50cf20e..49b8a1b3 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -2,7 +2,7 @@ Tests for the datasette.database.Database class """ from datasette.database import Database, Results, MultipleValues -from datasette.utils.sqlite import sqlite3 +from datasette.utils.sqlite import sqlite3, supports_generated_columns from datasette.utils import Column from .fixtures import app_client import pytest @@ -340,38 +340,42 @@ async def test_get_all_foreign_keys(db): @pytest.mark.asyncio async def test_table_names(db): table_names = await db.table_names() - assert table_names == [ - "simple_primary_key", - "primary_key_multiple_columns", - "primary_key_multiple_columns_explicit_label", - "compound_primary_key", - "compound_three_primary_keys", - "foreign_key_references", - "sortable", - "no_primary_key", - "123_starts_with_digits", - "Table With Space In Name", - "table/with/slashes.csv", - "complex_foreign_keys", - "custom_foreign_key_label", - "units", - "tags", - "searchable", - "searchable_tags", - "searchable_fts", - "searchable_fts_segments", - "searchable_fts_segdir", - "searchable_fts_docsize", - "searchable_fts_stat", - "select", - "infinity", - "facet_cities", - "facetable", - "binary_data", - "roadside_attractions", - "attraction_characteristic", - "roadside_attraction_characteristics", - ] + assert ( + table_names + == [ + "simple_primary_key", + "primary_key_multiple_columns", + "primary_key_multiple_columns_explicit_label", + "compound_primary_key", + "compound_three_primary_keys", + "foreign_key_references", + "sortable", + "no_primary_key", + "123_starts_with_digits", + "Table With Space In Name", + "table/with/slashes.csv", + "complex_foreign_keys", + "custom_foreign_key_label", + "units", + "tags", + "searchable", + "searchable_tags", + "searchable_fts", + "searchable_fts_segments", + "searchable_fts_segdir", + "searchable_fts_docsize", + "searchable_fts_stat", + "select", + "infinity", + "facet_cities", + "facetable", + "binary_data", + "roadside_attractions", + "attraction_characteristic", + "roadside_attraction_characteristics", + ] + + (["generated_columns"] if supports_generated_columns() else []) + ) @pytest.mark.asyncio diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 4554cfd4..dab5ef68 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -413,8 +413,7 @@ def test_hook_register_output_renderer_all_parameters(app_client): # Lots of 'at 0x103a4a690' in here - replace those so we can do # an easy comparison body = at_memory_re.sub(" at 0xXXX", response.text) - assert { - "1+1": 2, + assert json.loads(body) == { "datasette": "", "columns": [ "pk", @@ -451,7 +450,8 @@ def test_hook_register_output_renderer_all_parameters(app_client): "table": "facetable", "request": "", "view_name": "table", - } == json.loads(body) + "1+1": 2, + } # Test that query_name is set correctly query_response = app_client.get("/fixtures/pragma_cache_size.testall") assert "pragma_cache_size" == json.loads(query_response.body)["query_name"]