?_extra= support and TableView refactor to table_view

* Implemented ?_extra= option for JSON views, refs #262
* New dependency: asyncinject
* Remove now-obsolete TableView class
This commit is contained in:
Simon Willison 2023-03-22 15:49:39 -07:00 committed by GitHub
commit d97e82df3c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 1593 additions and 1085 deletions

View file

@ -896,9 +896,11 @@ def test_config_cache_size(app_client_larger_cache_size):
def test_config_force_https_urls():
with make_app_client(settings={"force_https_urls": True}) as client:
response = client.get("/fixtures/facetable.json?_size=3&_facet=state")
response = client.get(
"/fixtures/facetable.json?_size=3&_facet=state&_extra=next_url,suggested_facets"
)
assert response.json["next_url"].startswith("https://")
assert response.json["facet_results"]["state"]["results"][0][
assert response.json["facet_results"]["results"]["state"]["results"][0][
"toggle_url"
].startswith("https://")
assert response.json["suggested_facets"][0]["toggle_url"].startswith("https://")
@ -981,7 +983,9 @@ def test_common_prefix_database_names(app_client_conflicting_database_names):
def test_inspect_file_used_for_count(app_client_immutable_and_inspect_file):
response = app_client_immutable_and_inspect_file.get("/fixtures/sortable.json")
response = app_client_immutable_and_inspect_file.get(
"/fixtures/sortable.json?_extra=count"
)
assert response.json["count"] == 100

View file

@ -419,7 +419,7 @@ async def test_array_facet_handle_duplicate_tags():
)
response = await ds.client.get("/test_array_facet/otters.json?_facet_array=tags")
assert response.json()["facet_results"]["tags"] == {
assert response.json()["facet_results"]["results"]["tags"] == {
"name": "tags",
"type": "array",
"results": [
@ -517,13 +517,13 @@ async def test_json_array_with_blanks_and_nulls():
await db.execute_write("create table foo(json_column text)")
for value in ('["a", "b", "c"]', '["a", "b"]', "", None):
await db.execute_write("insert into foo (json_column) values (?)", [value])
response = await ds.client.get("/test_json_array/foo.json")
response = await ds.client.get("/test_json_array/foo.json?_extra=suggested_facets")
data = response.json()
assert data["suggested_facets"] == [
{
"name": "json_column",
"type": "array",
"toggle_url": "http://localhost/test_json_array/foo.json?_facet_array=json_column",
"toggle_url": "http://localhost/test_json_array/foo.json?_extra=suggested_facets&_facet_array=json_column",
}
]
@ -539,27 +539,29 @@ async def test_facet_size():
"insert into neighbourhoods (city, neighbourhood) values (?, ?)",
["City {}".format(i), "Neighbourhood {}".format(j)],
)
response = await ds.client.get("/test_facet_size/neighbourhoods.json")
response = await ds.client.get(
"/test_facet_size/neighbourhoods.json?_extra=suggested_facets"
)
data = response.json()
assert data["suggested_facets"] == [
{
"name": "neighbourhood",
"toggle_url": "http://localhost/test_facet_size/neighbourhoods.json?_facet=neighbourhood",
"toggle_url": "http://localhost/test_facet_size/neighbourhoods.json?_extra=suggested_facets&_facet=neighbourhood",
}
]
# Bump up _facet_size= to suggest city too
response2 = await ds.client.get(
"/test_facet_size/neighbourhoods.json?_facet_size=50"
"/test_facet_size/neighbourhoods.json?_facet_size=50&_extra=suggested_facets"
)
data2 = response2.json()
assert sorted(data2["suggested_facets"], key=lambda f: f["name"]) == [
{
"name": "city",
"toggle_url": "http://localhost/test_facet_size/neighbourhoods.json?_facet_size=50&_facet=city",
"toggle_url": "http://localhost/test_facet_size/neighbourhoods.json?_facet_size=50&_extra=suggested_facets&_facet=city",
},
{
"name": "neighbourhood",
"toggle_url": "http://localhost/test_facet_size/neighbourhoods.json?_facet_size=50&_facet=neighbourhood",
"toggle_url": "http://localhost/test_facet_size/neighbourhoods.json?_facet_size=50&_extra=suggested_facets&_facet=neighbourhood",
},
]
# Facet by city should return expected number of results
@ -567,20 +569,20 @@ async def test_facet_size():
"/test_facet_size/neighbourhoods.json?_facet_size=50&_facet=city"
)
data3 = response3.json()
assert len(data3["facet_results"]["city"]["results"]) == 50
assert len(data3["facet_results"]["results"]["city"]["results"]) == 50
# Reduce max_returned_rows and check that it's respected
ds._settings["max_returned_rows"] = 20
response4 = await ds.client.get(
"/test_facet_size/neighbourhoods.json?_facet_size=50&_facet=city"
)
data4 = response4.json()
assert len(data4["facet_results"]["city"]["results"]) == 20
assert len(data4["facet_results"]["results"]["city"]["results"]) == 20
# Test _facet_size=max
response5 = await ds.client.get(
"/test_facet_size/neighbourhoods.json?_facet_size=max&_facet=city"
)
data5 = response5.json()
assert len(data5["facet_results"]["city"]["results"]) == 20
assert len(data5["facet_results"]["results"]["city"]["results"]) == 20
# Now try messing with facet_size in the table metadata
orig_metadata = ds._metadata_local
try:
@ -593,7 +595,7 @@ async def test_facet_size():
"/test_facet_size/neighbourhoods.json?_facet=city"
)
data6 = response6.json()
assert len(data6["facet_results"]["city"]["results"]) == 6
assert len(data6["facet_results"]["results"]["city"]["results"]) == 6
# Setting it to max bumps it up to 50 again
ds._metadata_local["databases"]["test_facet_size"]["tables"]["neighbourhoods"][
"facet_size"
@ -601,7 +603,7 @@ async def test_facet_size():
data7 = (
await ds.client.get("/test_facet_size/neighbourhoods.json?_facet=city")
).json()
assert len(data7["facet_results"]["city"]["results"]) == 20
assert len(data7["facet_results"]["results"]["city"]["results"]) == 20
finally:
ds._metadata_local = orig_metadata
@ -635,7 +637,7 @@ async def test_conflicting_facet_names_json(ds_client):
"/fixtures/facetable.json?_facet=created&_facet_date=created"
"&_facet=tags&_facet_array=tags"
)
assert set(response.json()["facet_results"].keys()) == {
assert set(response.json()["facet_results"]["results"].keys()) == {
"created",
"tags",
"created_2",

View file

@ -82,13 +82,11 @@ async def test_through_filters_from_request(ds_client):
request = Request.fake(
'/?_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}'
)
filter_args = await (
through_filters(
request=request,
datasette=ds_client.ds,
table="roadside_attractions",
database="fixtures",
)
filter_args = await through_filters(
request=request,
datasette=ds_client.ds,
table="roadside_attractions",
database="fixtures",
)()
assert filter_args.where_clauses == [
"pk in (select attraction_id from roadside_attraction_characteristics where characteristic_id = :p0)"
@ -105,13 +103,11 @@ async def test_through_filters_from_request(ds_client):
request = Request.fake(
'/?_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}'
)
filter_args = await (
through_filters(
request=request,
datasette=ds_client.ds,
table="roadside_attractions",
database="fixtures",
)
filter_args = await through_filters(
request=request,
datasette=ds_client.ds,
table="roadside_attractions",
database="fixtures",
)()
assert filter_args.where_clauses == [
"pk in (select attraction_id from roadside_attraction_characteristics where characteristic_id = :p0)"
@ -127,12 +123,10 @@ async def test_through_filters_from_request(ds_client):
async def test_where_filters_from_request(ds_client):
await ds_client.ds.invoke_startup()
request = Request.fake("/?_where=pk+>+3")
filter_args = await (
where_filters(
request=request,
datasette=ds_client.ds,
database="fixtures",
)
filter_args = await where_filters(
request=request,
datasette=ds_client.ds,
database="fixtures",
)()
assert filter_args.where_clauses == ["pk > 3"]
assert filter_args.params == {}
@ -145,13 +139,11 @@ async def test_where_filters_from_request(ds_client):
@pytest.mark.asyncio
async def test_search_filters_from_request(ds_client):
request = Request.fake("/?_search=bobcat")
filter_args = await (
search_filters(
request=request,
datasette=ds_client.ds,
database="fixtures",
table="searchable",
)
filter_args = await search_filters(
request=request,
datasette=ds_client.ds,
database="fixtures",
table="searchable",
)()
assert filter_args.where_clauses == [
"rowid in (select rowid from searchable_fts where searchable_fts match escape_fts(:search))"

View file

@ -8,6 +8,7 @@ from pathlib import Path
# this resolves to "./ext", which is enough for SQLite to calculate the rest
COMPILED_EXTENSION_PATH = str(Path(__file__).parent / "ext")
# See if ext.c has been compiled, based off the different possible suffixes.
def has_compiled_ext():
for ext in ["dylib", "so", "dll"]:
@ -20,7 +21,6 @@ def has_compiled_ext():
@pytest.mark.asyncio
@pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c")
async def test_load_extension_default_entrypoint():
# The default entrypoint only loads a() and NOT b() or c(), so those
# should fail.
ds = Datasette(sqlite_extensions=[COMPILED_EXTENSION_PATH])
@ -41,7 +41,6 @@ async def test_load_extension_default_entrypoint():
@pytest.mark.asyncio
@pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c")
async def test_load_extension_multiple_entrypoints():
# Load in the default entrypoint and the other 2 custom entrypoints, now
# all a(), b(), and c() should run successfully.
ds = Datasette(

View file

@ -595,42 +595,42 @@ def test_hook_publish_subcommand():
@pytest.mark.asyncio
async def test_hook_register_facet_classes(ds_client):
response = await ds_client.get(
"/fixtures/compound_three_primary_keys.json?_dummy_facet=1"
"/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_extra=suggested_facets"
)
assert [
assert response.json()["suggested_facets"] == [
{
"name": "pk1",
"toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_facet_dummy=pk1",
"toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_extra=suggested_facets&_facet_dummy=pk1",
"type": "dummy",
},
{
"name": "pk2",
"toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_facet_dummy=pk2",
"toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_extra=suggested_facets&_facet_dummy=pk2",
"type": "dummy",
},
{
"name": "pk3",
"toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_facet_dummy=pk3",
"toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_extra=suggested_facets&_facet_dummy=pk3",
"type": "dummy",
},
{
"name": "content",
"toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_facet_dummy=content",
"toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_extra=suggested_facets&_facet_dummy=content",
"type": "dummy",
},
{
"name": "pk1",
"toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_facet=pk1",
"toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_extra=suggested_facets&_facet=pk1",
},
{
"name": "pk2",
"toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_facet=pk2",
"toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_extra=suggested_facets&_facet=pk2",
},
{
"name": "pk3",
"toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_facet=pk3",
"toggle_url": "http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_extra=suggested_facets&_facet=pk3",
},
] == response.json()["suggested_facets"]
]
@pytest.mark.asyncio

View file

@ -11,7 +11,7 @@ def routes():
@pytest.mark.parametrize(
"path,expected_class,expected_matches",
"path,expected_name,expected_matches",
(
("/", "IndexView", {"format": None}),
("/foo", "DatabaseView", {"format": None, "database": "foo"}),
@ -20,17 +20,17 @@ def routes():
("/foo.humbug", "DatabaseView", {"format": "humbug", "database": "foo"}),
(
"/foo/humbug",
"TableView",
"table_view",
{"database": "foo", "table": "humbug", "format": None},
),
(
"/foo/humbug.json",
"TableView",
"table_view",
{"database": "foo", "table": "humbug", "format": "json"},
),
(
"/foo/humbug.blah",
"TableView",
"table_view",
{"database": "foo", "table": "humbug", "format": "blah"},
),
(
@ -47,12 +47,14 @@ def routes():
("/-/metadata", "JsonDataView", {"format": None}),
),
)
def test_routes(routes, path, expected_class, expected_matches):
def test_routes(routes, path, expected_name, expected_matches):
match, view = resolve_routes(routes, path)
if expected_class is None:
if expected_name is None:
assert match is None
else:
assert view.view_class.__name__ == expected_class
assert (
view.__name__ == expected_name or view.view_class.__name__ == expected_name
)
assert match.groupdict() == expected_matches

View file

@ -15,7 +15,7 @@ import urllib
@pytest.mark.asyncio
async def test_table_json(ds_client):
response = await ds_client.get("/fixtures/simple_primary_key.json?_shape=objects")
response = await ds_client.get("/fixtures/simple_primary_key.json?_extra=query")
assert response.status_code == 200
data = response.json()
assert (
@ -198,6 +198,10 @@ async def test_paginate_tables_and_views(
fetched = []
count = 0
while path:
if "?" in path:
path += "&_extra=next_url"
else:
path += "?_extra=next_url"
response = await ds_client.get(path)
assert response.status_code == 200
count += 1
@ -230,7 +234,9 @@ async def test_validate_page_size(ds_client, path, expected_error):
@pytest.mark.asyncio
async def test_page_size_zero(ds_client):
"""For _size=0 we return the counts, empty rows and no continuation token"""
response = await ds_client.get("/fixtures/no_primary_key.json?_size=0")
response = await ds_client.get(
"/fixtures/no_primary_key.json?_size=0&_extra=count,next_url"
)
assert response.status_code == 200
assert [] == response.json()["rows"]
assert 201 == response.json()["count"]
@ -241,7 +247,7 @@ async def test_page_size_zero(ds_client):
@pytest.mark.asyncio
async def test_paginate_compound_keys(ds_client):
fetched = []
path = "/fixtures/compound_three_primary_keys.json?_shape=objects"
path = "/fixtures/compound_three_primary_keys.json?_shape=objects&_extra=next_url"
page = 0
while path:
page += 1
@ -262,9 +268,7 @@ async def test_paginate_compound_keys(ds_client):
@pytest.mark.asyncio
async def test_paginate_compound_keys_with_extra_filters(ds_client):
fetched = []
path = (
"/fixtures/compound_three_primary_keys.json?content__contains=d&_shape=objects"
)
path = "/fixtures/compound_three_primary_keys.json?content__contains=d&_shape=objects&_extra=next_url"
page = 0
while path:
page += 1
@ -315,7 +319,7 @@ async def test_paginate_compound_keys_with_extra_filters(ds_client):
],
)
async def test_sortable(ds_client, query_string, sort_key, human_description_en):
path = f"/fixtures/sortable.json?_shape=objects&{query_string}"
path = f"/fixtures/sortable.json?_shape=objects&_extra=human_description_en,next_url&{query_string}"
fetched = []
page = 0
while path:
@ -338,6 +342,7 @@ async def test_sortable_and_filtered(ds_client):
path = (
"/fixtures/sortable.json"
"?content__contains=d&_sort_desc=sortable&_shape=objects"
"&_extra=human_description_en,count"
)
response = await ds_client.get(path)
fetched = response.json()["rows"]
@ -660,7 +665,9 @@ def test_table_filter_extra_where_disabled_if_no_sql_allowed():
async def test_table_through(ds_client):
# Just the museums:
response = await ds_client.get(
'/fixtures/roadside_attractions.json?_shape=arrays&_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}'
"/fixtures/roadside_attractions.json?_shape=arrays"
'&_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}'
"&_extra=human_description_en"
)
assert response.json()["rows"] == [
[
@ -712,6 +719,7 @@ async def test_view(ds_client):
]
@pytest.mark.xfail
@pytest.mark.asyncio
async def test_unit_filters(ds_client):
response = await ds_client.get(
@ -731,7 +739,7 @@ def test_page_size_matching_max_returned_rows(
app_client_returned_rows_matches_page_size,
):
fetched = []
path = "/fixtures/no_primary_key.json"
path = "/fixtures/no_primary_key.json?_extra=next_url"
while path:
response = app_client_returned_rows_matches_page_size.get(path)
fetched.extend(response.json["rows"])
@ -911,12 +919,42 @@ async def test_facets(ds_client, path, expected_facet_results):
response = await ds_client.get(path)
facet_results = response.json()["facet_results"]
# We only compare the querystring portion of the taggle_url
for facet_name, facet_info in facet_results.items():
for facet_name, facet_info in facet_results["results"].items():
assert facet_name == facet_info["name"]
assert False is facet_info["truncated"]
for facet_value in facet_info["results"]:
facet_value["toggle_url"] = facet_value["toggle_url"].split("?")[1]
assert expected_facet_results == facet_results
assert expected_facet_results == facet_results["results"]
@pytest.mark.asyncio
@pytest.mark.skipif(not detect_json1(), reason="requires JSON1 extension")
async def test_facets_array(ds_client):
response = await ds_client.get("/fixtures/facetable.json?_facet_array=tags")
facet_results = response.json()["facet_results"]
assert facet_results["results"]["tags"]["results"] == [
{
"value": "tag1",
"label": "tag1",
"count": 2,
"toggle_url": "http://localhost/fixtures/facetable.json?_facet_array=tags&tags__arraycontains=tag1",
"selected": False,
},
{
"value": "tag2",
"label": "tag2",
"count": 1,
"toggle_url": "http://localhost/fixtures/facetable.json?_facet_array=tags&tags__arraycontains=tag2",
"selected": False,
},
{
"value": "tag3",
"label": "tag3",
"count": 1,
"toggle_url": "http://localhost/fixtures/facetable.json?_facet_array=tags&tags__arraycontains=tag3",
"selected": False,
},
]
@pytest.mark.asyncio
@ -926,58 +964,83 @@ async def test_suggested_facets(ds_client):
"name": suggestion["name"],
"querystring": suggestion["toggle_url"].split("?")[-1],
}
for suggestion in (await ds_client.get("/fixtures/facetable.json")).json()[
"suggested_facets"
]
for suggestion in (
await ds_client.get("/fixtures/facetable.json?_extra=suggested_facets")
).json()["suggested_facets"]
]
expected = [
{"name": "created", "querystring": "_facet=created"},
{"name": "planet_int", "querystring": "_facet=planet_int"},
{"name": "on_earth", "querystring": "_facet=on_earth"},
{"name": "state", "querystring": "_facet=state"},
{"name": "_city_id", "querystring": "_facet=_city_id"},
{"name": "_neighborhood", "querystring": "_facet=_neighborhood"},
{"name": "tags", "querystring": "_facet=tags"},
{"name": "complex_array", "querystring": "_facet=complex_array"},
{"name": "created", "querystring": "_facet_date=created"},
{"name": "created", "querystring": "_extra=suggested_facets&_facet=created"},
{
"name": "planet_int",
"querystring": "_extra=suggested_facets&_facet=planet_int",
},
{"name": "on_earth", "querystring": "_extra=suggested_facets&_facet=on_earth"},
{"name": "state", "querystring": "_extra=suggested_facets&_facet=state"},
{"name": "_city_id", "querystring": "_extra=suggested_facets&_facet=_city_id"},
{
"name": "_neighborhood",
"querystring": "_extra=suggested_facets&_facet=_neighborhood",
},
{"name": "tags", "querystring": "_extra=suggested_facets&_facet=tags"},
{
"name": "complex_array",
"querystring": "_extra=suggested_facets&_facet=complex_array",
},
{
"name": "created",
"querystring": "_extra=suggested_facets&_facet_date=created",
},
]
if detect_json1():
expected.append({"name": "tags", "querystring": "_facet_array=tags"})
expected.append(
{"name": "tags", "querystring": "_extra=suggested_facets&_facet_array=tags"}
)
assert expected == suggestions
def test_allow_facet_off():
with make_app_client(settings={"allow_facet": False}) as client:
assert 400 == client.get("/fixtures/facetable.json?_facet=planet_int").status
assert (
client.get(
"/fixtures/facetable.json?_facet=planet_int&_extra=suggested_facets"
).status
== 400
)
data = client.get("/fixtures/facetable.json?_extra=suggested_facets").json
# Should not suggest any facets either:
assert [] == client.get("/fixtures/facetable.json").json["suggested_facets"]
assert [] == data["suggested_facets"]
def test_suggest_facets_off():
with make_app_client(settings={"suggest_facets": False}) as client:
# Now suggested_facets should be []
assert [] == client.get("/fixtures/facetable.json").json["suggested_facets"]
assert (
[]
== client.get("/fixtures/facetable.json?_extra=suggested_facets").json[
"suggested_facets"
]
)
@pytest.mark.asyncio
@pytest.mark.parametrize("nofacet", (True, False))
async def test_nofacet(ds_client, nofacet):
path = "/fixtures/facetable.json?_facet=state"
path = "/fixtures/facetable.json?_facet=state&_extra=suggested_facets"
if nofacet:
path += "&_nofacet=1"
response = await ds_client.get(path)
if nofacet:
assert response.json()["suggested_facets"] == []
assert response.json()["facet_results"] == {}
assert response.json()["facet_results"]["results"] == {}
else:
assert response.json()["suggested_facets"] != []
assert response.json()["facet_results"] != {}
assert response.json()["facet_results"]["results"] != {}
@pytest.mark.asyncio
@pytest.mark.parametrize("nosuggest", (True, False))
async def test_nosuggest(ds_client, nosuggest):
path = "/fixtures/facetable.json?_facet=state"
path = "/fixtures/facetable.json?_facet=state&_extra=suggested_facets"
if nosuggest:
path += "&_nosuggest=1"
response = await ds_client.get(path)
@ -993,9 +1056,9 @@ async def test_nosuggest(ds_client, nosuggest):
@pytest.mark.asyncio
@pytest.mark.parametrize("nocount,expected_count", ((True, None), (False, 15)))
async def test_nocount(ds_client, nocount, expected_count):
path = "/fixtures/facetable.json"
path = "/fixtures/facetable.json?_extra=count"
if nocount:
path += "?_nocount=1"
path += "&_nocount=1"
response = await ds_client.get(path)
assert response.json()["count"] == expected_count
@ -1280,7 +1343,7 @@ def test_generated_columns_are_visible_in_datasette():
),
)
async def test_col_nocol(ds_client, path, expected_columns):
response = await ds_client.get(path)
response = await ds_client.get(path + "&_extra=columns")
assert response.status_code == 200
columns = response.json()["columns"]
assert columns == expected_columns

View file

@ -1160,6 +1160,13 @@ async def test_table_page_title(ds_client, path, expected):
assert title == expected
@pytest.mark.asyncio
async def test_table_post_method_not_allowed(ds_client):
response = await ds_client.post("/fixtures/facetable")
assert response.status_code == 405
assert "Method not allowed" in response.text
@pytest.mark.parametrize("allow_facet", (True, False))
def test_allow_facet_off(allow_facet):
with make_app_client(settings={"allow_facet": allow_facet}) as client: