github/datasette
1
0
Fork 0
mirror of https://github.com/simonw/datasette.git synced 2025-12-10 16:51:24 +01:00
Code Issues Projects Releases Packages Wiki Activity

Tilde encoding (#1659)

Browse source 
Closes #1657

Refs #1439
This commit is contained in:
Simon Willison 2022-03-15 11:01:57 -07:00 • committed by GitHub
commit a35393b29c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 125 additions and 141 deletions
Download patch file Download diff file Expand all files Collapse all files

17
tests/test_api.py
View file

@ -679,18 +679,9 @@ def test_row(app_client):
assert [{"id": "1", "content": "hello"}] == response.json["rows"]
def test_row_format_in_querystring(app_client):
# regression test for https://github.com/simonw/datasette/issues/563
response = app_client.get(
"/fixtures/simple_primary_key/1?_format=json&_shape=objects"
)
assert response.status == 200
assert [{"id": "1", "content": "hello"}] == response.json["rows"]
def test_row_strange_table_name(app_client):
response = app_client.get(
"/fixtures/table%2Fwith%2Fslashes.csv/3.json?_shape=objects"
"/fixtures/table~2Fwith~2Fslashes~2Ecsv/3.json?_shape=objects"
)
assert response.status == 200
assert [{"pk": "3", "content": "hey"}] == response.json["rows"]
@ -942,7 +933,7 @@ def test_cors(app_client_with_cors, path, status_code):
)
def test_database_with_space_in_name(app_client_two_attached_databases, path):
response = app_client_two_attached_databases.get(
"/extra-20database" + path, follow_redirects=True
"/extra~20database" + path, follow_redirects=True
)
assert response.status == 200
@ -953,7 +944,7 @@ def test_common_prefix_database_names(app_client_conflicting_database_names):
d["name"]
for d in app_client_conflicting_database_names.get("/-/databases.json").json
]
for db_name, path in (("foo", "/foo.json"), ("foo-bar", "/foo-2Dbar.json")):
for db_name, path in (("foo", "/foo.json"), ("foo-bar", "/foo-bar.json")):
data = app_client_conflicting_database_names.get(path).json
assert db_name == data["database"]
@ -996,7 +987,7 @@ async def test_hidden_sqlite_stat1_table():
@pytest.mark.asyncio
@pytest.mark.parametrize("db_name", ("foo", r"fo%o", "f~/c.d"))
async def test_dash_encoded_database_names(db_name):
async def test_tilde_encoded_database_names(db_name):
ds = Datasette()
ds.add_memory_database(db_name)
response = await ds.client.get("/.json")

6
tests/test_cli.py
View file

@ -9,7 +9,7 @@ from datasette.app import SETTINGS
from datasette.plugins import DEFAULT_PLUGINS
from datasette.cli import cli, serve
from datasette.version import __version__
from datasette.utils import dash_encode
from datasette.utils import tilde_encode
from datasette.utils.sqlite import sqlite3
from click.testing import CliRunner
import io
@ -295,12 +295,12 @@ def test_weird_database_names(ensure_eventloop, tmpdir, filename):
assert result1.exit_code == 0, result1.output
filename_no_stem = filename.rsplit(".", 1)[0]
expected_link = '<a href="/{}">{}</a>'.format(
dash_encode(filename_no_stem), filename_no_stem
tilde_encode(filename_no_stem), filename_no_stem
)
assert expected_link in result1.output
# Now try hitting that database page
result2 = runner.invoke(
cli, [db_path, "--get", "/{}".format(dash_encode(filename_no_stem))]
cli, [db_path, "--get", "/{}".format(tilde_encode(filename_no_stem))]
)
assert result2.exit_code == 0, result2.output

45
tests/test_html.py
View file

@ -29,7 +29,7 @@ def test_homepage(app_client_two_attached_databases):
)
# Should be two attached databases
assert [
{"href": r"/extra-20database", "text": "extra database"},
{"href": "/extra~20database", "text": "extra database"},
{"href": "/fixtures", "text": "fixtures"},
] == [{"href": a["href"], "text": a.text.strip()} for a in soup.select("h2 a")]
# Database should show count text and attached tables
@ -44,8 +44,8 @@ def test_homepage(app_client_two_attached_databases):
{"href": a["href"], "text": a.text.strip()} for a in links_p.findAll("a")
]
assert [
{"href": r"/extra-20database/searchable", "text": "searchable"},
{"href": r"/extra-20database/searchable_view", "text": "searchable_view"},
{"href": r"/extra~20database/searchable", "text": "searchable"},
{"href": r"/extra~20database/searchable_view", "text": "searchable_view"},
] == table_links
@ -139,15 +139,15 @@ def test_database_page(app_client):
queries_ul = soup.find("h2", text="Queries").find_next_sibling("ul")
assert queries_ul is not None
assert [
(
"/fixtures/-F0-9D-90-9C-F0-9D-90-A2-F0-9D-90-AD-F0-9D-90-A2-F0-9D-90-9E-F0-9D-90-AC",
"𝐜𝐢𝐭𝐢𝐞𝐬",
),
("/fixtures/from_async_hook", "from_async_hook"),
("/fixtures/from_hook", "from_hook"),
("/fixtures/magic_parameters", "magic_parameters"),
("/fixtures/neighborhood_search#fragment-goes-here", "Search neighborhoods"),
("/fixtures/pragma_cache_size", "pragma_cache_size"),
(
"/fixtures/~F0~9D~90~9C~F0~9D~90~A2~F0~9D~90~AD~F0~9D~90~A2~F0~9D~90~9E~F0~9D~90~AC",
"𝐜𝐢𝐭𝐢𝐞𝐬",
),
] == sorted(
[(a["href"], a.text) for a in queries_ul.find_all("a")], key=lambda p: p[0]
)
@ -193,11 +193,11 @@ def test_row_redirects_with_url_hash(app_client_with_hash):
def test_row_strange_table_name_with_url_hash(app_client_with_hash):
response = app_client_with_hash.get("/fixtures/table-2Fwith-2Fslashes-2Ecsv/3")
response = app_client_with_hash.get("/fixtures/table~2Fwith~2Fslashes~2Ecsv/3")
assert response.status == 302
assert response.headers["Location"].endswith("/table-2Fwith-2Fslashes-2Ecsv/3")
assert response.headers["Location"].endswith("/table~2Fwith~2Fslashes~2Ecsv/3")
response = app_client_with_hash.get(
"/fixtures/table-2Fwith-2Fslashes-2Ecsv/3", follow_redirects=True
"/fixtures/table~2Fwith~2Fslashes~2Ecsv/3", follow_redirects=True
)
assert response.status == 200
@ -229,7 +229,7 @@ def test_row_page_does_not_truncate():
["query", "db-fixtures", "query-neighborhood_search"],
),
(
"/fixtures/table%2Fwith%2Fslashes.csv",
"/fixtures/table~2Fwith~2Fslashes~2Ecsv",
["table", "db-fixtures", "table-tablewithslashescsv-fa7563"],
),
(
@ -255,7 +255,7 @@ def test_css_classes_on_body(app_client, path, expected_classes):
"table-fixtures-simple_primary_key.html, *table.html",
),
(
"/fixtures/table%2Fwith%2Fslashes.csv",
"/fixtures/table~2Fwith~2Fslashes~2Ecsv",
"table-fixtures-tablewithslashescsv-fa7563.html, *table.html",
),
(
@ -359,7 +359,7 @@ def test_row_links_from_other_tables(app_client, path, expected_text, expected_l
],
),
(
"/fixtures/compound_primary_key/a-2Fb,-2Ec-2Dd",
"/fixtures/compound_primary_key/a~2Fb,~2Ec~2Dd",
[
[
'<td class="col-pk1 type-str">a/b</td>',
@ -816,7 +816,8 @@ def test_base_url_affects_metadata_extra_css_urls(app_client_base_url_prefix):
),
("/fixtures/pragma_cache_size", None),
(
"/fixtures/𝐜𝐢𝐭𝐢𝐞𝐬",
# /fixtures/𝐜𝐢𝐭𝐢𝐞𝐬
"/fixtures/~F0~9D~90~9C~F0~9D~90~A2~F0~9D~90~AD~F0~9D~90~A2~F0~9D~90~9E~F0~9D~90~AC",
"/fixtures?sql=select+id%2C+name+from+facet_cities+order+by+id+limit+1%3B",
),
("/fixtures/magic_parameters", None),
@ -824,6 +825,7 @@ def test_base_url_affects_metadata_extra_css_urls(app_client_base_url_prefix):
)
def test_edit_sql_link_on_canned_queries(app_client, path, expected):
response = app_client.get(path)
assert response.status == 200
expected_link = f'<a href="{expected}" class="canned-query-edit-sql">Edit SQL</a>'
if expected:
assert expected_link in response.text
@ -898,8 +900,8 @@ def test_trace_correctly_escaped(app_client):
# Table page
("/fixtures/facetable", "http://localhost/fixtures/facetable.json"),
(
"/fixtures/table%2Fwith%2Fslashes.csv",
"http://localhost/fixtures/table%2Fwith%2Fslashes.csv?_format=json",
"/fixtures/table~2Fwith~2Fslashes~2Ecsv",
"http://localhost/fixtures/table~2Fwith~2Fslashes~2Ecsv.json",
),
# Row page
(
@ -930,6 +932,7 @@ def test_trace_correctly_escaped(app_client):
)
def test_alternate_url_json(app_client, path, expected):
response = app_client.get(path)
assert response.status == 200
link = response.headers["link"]
assert link == '{}; rel="alternate"; type="application/json+datasette"'.format(
expected
@ -959,13 +962,17 @@ def test_no_alternate_url_json(app_client, path):
(
(
"/fivethirtyeight/twitter-ratio%2Fsenators",
"/fivethirtyeight/twitter-2Dratio-2Fsenators",
"/fivethirtyeight/twitter-ratio~2Fsenators",
),
(
"/fixtures/table%2Fwith%2Fslashes.csv",
"/fixtures/table~2Fwith~2Fslashes~2Ecsv",
),
# query string should be preserved
("/foo/bar%2Fbaz?id=5", "/foo/bar-2Fbaz?id=5"),
("/foo/bar%2Fbaz?id=5", "/foo/bar~2Fbaz?id=5"),
),
)
def test_redirect_percent_encoding_to_dash_encoding(app_client, path, expected):
def test_redirect_percent_encoding_to_tilde_encoding(app_client, path, expected):
response = app_client.get(path)
assert response.status == 302
assert response.headers["location"] == expected

2
tests/test_internals_urls.py
View file

@ -121,7 +121,7 @@ def test_database(ds, base_url, format, expected):
("/", "name", None, "/_memory/name"),
("/prefix/", "name", None, "/prefix/_memory/name"),
("/", "name", "json", "/_memory/name.json"),
("/", "name.json", "json", "/_memory/name-2Ejson.json"),
("/", "name.json", "json", "/_memory/name~2Ejson.json"),
],
)
def test_table_and_query(ds, base_url, name, format, expected):

9
tests/test_table_api.py
View file

@ -138,13 +138,13 @@ def test_table_shape_object_compound_primary_key(app_client):
response = app_client.get("/fixtures/compound_primary_key.json?_shape=object")
assert response.json == {
"a,b": {"pk1": "a", "pk2": "b", "content": "c"},
"a-2Fb,-2Ec-2Dd": {"pk1": "a/b", "pk2": ".c-d", "content": "c"},
"a~2Fb,~2Ec-d": {"pk1": "a/b", "pk2": ".c-d", "content": "c"},
}
def test_table_with_slashes_in_name(app_client):
response = app_client.get(
"/fixtures/table%2Fwith%2Fslashes.csv?_shape=objects&_format=json"
"/fixtures/table~2Fwith~2Fslashes~2Ecsv.json?_shape=objects"
)
assert response.status == 200
data = response.json
@ -1032,7 +1032,10 @@ def test_infinity_returned_as_invalid_json_if_requested(app_client):
def test_custom_query_with_unicode_characters(app_client):
response = app_client.get("/fixtures/𝐜𝐢𝐭𝐢𝐞𝐬.json?_shape=array")
# /fixtures/𝐜𝐢𝐭𝐢𝐞𝐬.json
response = app_client.get(
"/fixtures/~F0~9D~90~9C~F0~9D~90~A2~F0~9D~90~AD~F0~9D~90~A2~F0~9D~90~9E~F0~9D~90~AC.json?_shape=array"
)
assert [{"id": 1, "name": "San Francisco"}] == response.json

2
tests/test_table_html.py
View file

@ -565,7 +565,7 @@ def test_table_html_compound_primary_key(app_client):
'<td class="col-content type-str">c</td>',
],
[
'<td class="col-Link type-pk"><a href="/fixtures/compound_primary_key/a-2Fb,-2Ec-2Dd">a/b,.c-d</a></td>',
'<td class="col-Link type-pk"><a href="/fixtures/compound_primary_key/a~2Fb,~2Ec-d">a/b,.c-d</a></td>',
'<td class="col-pk1 type-str">a/b</td>',
'<td class="col-pk2 type-str">.c-d</td>',
'<td class="col-content type-str">c</td>',

36
tests/test_utils.py
View file

@ -19,8 +19,8 @@ from unittest.mock import patch
("foo", ["foo"]),
("foo,bar", ["foo", "bar"]),
("123,433,112", ["123", "433", "112"]),
("123%2C433,112", ["123,433", "112"]),
("123%2F433%2F112", ["123/433/112"]),
("123~2C433,112", ["123,433", "112"]),
("123~2F433~2F112", ["123/433/112"]),
],
)
def test_urlsafe_components(path, expected):
@ -93,7 +93,7 @@ def test_path_with_replaced_args(path, args, expected):
"row,pks,expected_path",
[
({"A": "foo", "B": "bar"}, ["A", "B"], "foo,bar"),
({"A": "f,o", "B": "bar"}, ["A", "B"], "f-2Co,bar"),
({"A": "f,o", "B": "bar"}, ["A", "B"], "f~2Co,bar"),
({"A": 123}, ["A"], "123"),
(
utils.CustomRow(
@ -393,9 +393,7 @@ def test_table_columns():
("/foo?sql=select+1", "json", {}, "/foo.json?sql=select+1"),
("/foo/bar", "json", {}, "/foo/bar.json"),
("/foo/bar", "csv", {}, "/foo/bar.csv"),
("/foo/bar.csv", "json", {}, "/foo/bar.csv?_format=json"),
("/foo/bar", "csv", {"_dl": 1}, "/foo/bar.csv?_dl=1"),
("/foo/b.csv", "json", {"_dl": 1}, "/foo/b.csv?_dl=1&_format=json"),
(
"/sf-trees/Street_Tree_List?_search=cherry&_size=1000",
"csv",
@ -410,18 +408,6 @@ def test_path_with_format(path, format, extra_qs, expected):
assert expected == actual
def test_path_with_format_replace_format():
request = Request.fake("/foo/bar.csv")
assert (
utils.path_with_format(request=request, format="blob")
== "/foo/bar.csv?_format=blob"
)
assert (
utils.path_with_format(request=request, format="blob", replace_format="csv")
== "/foo/bar.blob"
)
@pytest.mark.parametrize(
"bytes,expected",
[
@ -652,15 +638,15 @@ async def test_derive_named_parameters(sql, expected):
"original,expected",
(
("abc", "abc"),
("/foo/bar", "-2Ffoo-2Fbar"),
("/-/bar", "-2F-2D-2Fbar"),
("-/db-/table.csv", "-2D-2Fdb-2D-2Ftable-2Ecsv"),
(r"%~-/", "-25-7E-2D-2F"),
("-25-7E-2D-2F", "-2D25-2D7E-2D2D-2D2F"),
("/foo/bar", "~2Ffoo~2Fbar"),
("/-/bar", "~2F-~2Fbar"),
("-/db-/table.csv", "-~2Fdb-~2Ftable~2Ecsv"),
(r"%~-/", "~25~7E-~2F"),
("~25~7E~2D~2F", "~7E25~7E7E~7E2D~7E2F"),
),
)
def test_dash_encoding(original, expected):
actual = utils.dash_encode(original)
def test_tilde_encoding(original, expected):
actual = utils.tilde_encode(original)
assert actual == expected
# And test round-trip
assert original == utils.dash_decode(actual)
assert original == utils.tilde_decode(actual)