github/datasette
1
0
Fork 0
mirror of https://github.com/simonw/datasette.git synced 2025-12-10 16:51:24 +01:00
Code Issues Projects Releases Packages Wiki Activity

Tilde encoding (#1659)

Browse source 
Closes #1657

Refs #1439
This commit is contained in:
Simon Willison 2022-03-15 11:01:57 -07:00 • committed by GitHub
commit a35393b29c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 125 additions and 141 deletions
Download patch file Download diff file Expand all files Collapse all files

11
datasette/app.py
View file

@ -1211,11 +1211,14 @@ class DatasetteRouter:
return await self.handle_404(request, send)
async def handle_404(self, request, send, exception=None):
# If path contains % encoding, redirect to dash encoding
# If path contains % encoding, redirect to tilde encoding
if "%" in request.path:
# Try the same path but with "%" replaced by "-"
# and "-" replaced with "-2D"
new_path = request.path.replace("-", "-2D").replace("%", "-")
# Try the same path but with "%" replaced by "~"
# and "~" replaced with "~7E"
# and "." replaced with "~2E"
new_path = (
request.path.replace("~", "~7E").replace("%", "~").replace(".", "~2E")
)
if request.query_string:
new_path += "?{}".format(request.query_string)
await asgi_send_redirect(send, new_path)

10
datasette/url_builder.py
View file

@ -1,4 +1,4 @@
from .utils import dash_encode, path_with_format, HASH_LENGTH, PrefixedUrlString
from .utils import tilde_encode, path_with_format, HASH_LENGTH, PrefixedUrlString
import urllib
@ -31,20 +31,20 @@ class Urls:
db = self.ds.databases[database]
if self.ds.setting("hash_urls") and db.hash:
path = self.path(
f"{dash_encode(database)}-{db.hash[:HASH_LENGTH]}", format=format
f"{tilde_encode(database)}-{db.hash[:HASH_LENGTH]}", format=format
)
else:
path = self.path(dash_encode(database), format=format)
path = self.path(tilde_encode(database), format=format)
return path
def table(self, database, table, format=None):
path = f"{self.database(database)}/{dash_encode(table)}"
path = f"{self.database(database)}/{tilde_encode(table)}"
if format is not None:
path = path_with_format(path=path, format=format)
return PrefixedUrlString(path)
def query(self, database, query, format=None):
path = f"{self.database(database)}/{dash_encode(query)}"
path = f"{self.database(database)}/{tilde_encode(query)}"
if format is not None:
path = path_with_format(path=path, format=format)
return PrefixedUrlString(path)

37
datasette/utils/__init__.py
View file

@ -15,6 +15,7 @@ import tempfile
import typing
import time
import types
import secrets
import shutil
import urllib
import yaml
@ -112,12 +113,12 @@ async def await_me_maybe(value: typing.Any) -> typing.Any:
def urlsafe_components(token):
"""Splits token on commas and dash-decodes each component"""
return [dash_decode(b) for b in token.split(",")]
"""Splits token on commas and tilde-decodes each component"""
return [tilde_decode(b) for b in token.split(",")]
def path_from_row_pks(row, pks, use_rowid, quote=True):
"""Generate an optionally dash-quoted unique identifier
"""Generate an optionally tilde-encoded unique identifier
for a row from its primary keys."""
if use_rowid:
bits = [row["rowid"]]
@ -126,7 +127,7 @@ def path_from_row_pks(row, pks, use_rowid, quote=True):
row[pk]["value"] if isinstance(row[pk], dict) else row[pk] for pk in pks
]
if quote:
bits = [dash_encode(str(bit)) for bit in bits]
bits = [tilde_encode(str(bit)) for bit in bits]
else:
bits = [str(bit) for bit in bits]
@ -1142,34 +1143,38 @@ def add_cors_headers(headers):
headers["Access-Control-Expose-Headers"] = "Link"
_DASH_ENCODING_SAFE = frozenset(
_TILDE_ENCODING_SAFE = frozenset(
b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
b"abcdefghijklmnopqrstuvwxyz"
b"0123456789_"
b"0123456789_-"
# This is the same as Python percent-encoding but I removed
# '.' and '-' and '~'
# '.' and '~'
)
class DashEncoder(dict):
class TildeEncoder(dict):
# Keeps a cache internally, via __missing__
def __missing__(self, b):
# Handle a cache miss, store encoded string in cache and return.
res = chr(b) if b in _DASH_ENCODING_SAFE else "-{:02X}".format(b)
res = chr(b) if b in _TILDE_ENCODING_SAFE else "~{:02X}".format(b)
self[b] = res
return res
_dash_encoder = DashEncoder().__getitem__
_tilde_encoder = TildeEncoder().__getitem__
@documented
def dash_encode(s: str) -> str:
"Returns dash-encoded string - for example ``/foo/bar`` -> ``-2Ffoo-2Fbar``"
return "".join(_dash_encoder(char) for char in s.encode("utf-8"))
def tilde_encode(s: str) -> str:
"Returns tilde-encoded string - for example ``/foo/bar`` -> ``~2Ffoo~2Fbar``"
return "".join(_tilde_encoder(char) for char in s.encode("utf-8"))
@documented
def dash_decode(s: str) -> str:
"Decodes a dash-encoded string, so ``-2Ffoo-2Fbar`` -> ``/foo/bar``"
return urllib.parse.unquote(s.replace("-", "%"))
def tilde_decode(s: str) -> str:
"Decodes a tilde-encoded string, so ``~2Ffoo~2Fbar`` -> ``/foo/bar``"
# Avoid accidentally decoding a %2f style sequence
temp = secrets.token_hex(16)
s = s.replace("%", temp)
decoded = urllib.parse.unquote(s.replace("~", "%"))
return decoded.replace(temp, "%")

25
datasette/views/base.py
View file

@ -10,6 +10,7 @@ import pint
from datasette import __version__
from datasette.database import QueryInterrupted
from datasette.utils.asgi import Request
from datasette.utils import (
add_cors_headers,
await_me_maybe,
@ -17,8 +18,8 @@ from datasette.utils import (
InvalidSql,
LimitedWriter,
call_with_supported_arguments,
dash_decode,
dash_encode,
tilde_decode,
tilde_encode,
path_from_row_pks,
path_with_added_args,
path_with_removed_args,
@ -205,14 +206,14 @@ class DataView(BaseView):
async def resolve_db_name(self, request, db_name, **kwargs):
hash = None
name = None
decoded_name = dash_decode(db_name)
decoded_name = tilde_decode(db_name)
if decoded_name not in self.ds.databases and "-" in db_name:
# No matching DB found, maybe it's a name-hash?
name_bit, hash_bit = db_name.rsplit("-", 1)
if dash_decode(name_bit) not in self.ds.databases:
if tilde_decode(name_bit) not in self.ds.databases:
raise NotFound(f"Database not found: {name}")
else:
name = dash_decode(name_bit)
name = tilde_decode(name_bit)
hash = hash_bit
else:
name = decoded_name
@ -235,7 +236,7 @@ class DataView(BaseView):
return await db.table_exists(t)
table, _format = await resolve_table_and_format(
table_and_format=dash_decode(kwargs["table_and_format"]),
table_and_format=tilde_decode(kwargs["table_and_format"]),
table_exists=async_table_exists,
allowed_formats=self.ds.renderers.keys(),
)
@ -243,11 +244,11 @@ class DataView(BaseView):
if _format:
kwargs["as_format"] = f".{_format}"
elif kwargs.get("table"):
kwargs["table"] = dash_decode(kwargs["table"])
kwargs["table"] = tilde_decode(kwargs["table"])
should_redirect = self.ds.urls.path(f"{name}-{expected}")
if kwargs.get("table"):
should_redirect += "/" + dash_encode(kwargs["table"])
should_redirect += "/" + tilde_encode(kwargs["table"])
if kwargs.get("pk_path"):
should_redirect += "/" + kwargs["pk_path"]
if kwargs.get("as_format"):
@ -291,6 +292,7 @@ class DataView(BaseView):
if not request.args.get(key)
]
if extra_parameters:
# Replace request object with a new one with modified scope
if not request.query_string:
new_query_string = "&".join(extra_parameters)
else:
@ -300,7 +302,8 @@ class DataView(BaseView):
new_scope = dict(
request.scope, query_string=new_query_string.encode("latin-1")
)
request.scope = new_scope
receive = request.receive
request = Request(new_scope, receive)
if stream:
# Some quick soundness checks
if not self.ds.setting("allow_csv_stream"):
@ -467,7 +470,7 @@ class DataView(BaseView):
return await db.table_exists(t)
table, _ext_format = await resolve_table_and_format(
table_and_format=dash_decode(args["table_and_format"]),
table_and_format=tilde_decode(args["table_and_format"]),
table_exists=async_table_exists,
allowed_formats=self.ds.renderers.keys(),
)
@ -475,7 +478,7 @@ class DataView(BaseView):
args["table"] = table
del args["table_and_format"]
elif "table" in args:
args["table"] = dash_decode(args["table"])
args["table"] = tilde_decode(args["table"])
return _format, args
async def view_get(self, request, database, hash, correct_hash_provided, **kwargs):

14
datasette/views/table.py
View file

@ -12,7 +12,8 @@ from datasette.utils import (
MultiParams,
append_querystring,
compound_keys_after_sql,
dash_encode,
tilde_decode,
tilde_encode,
escape_sqlite,
filters_should_redirect,
is_url,
@ -143,7 +144,7 @@ class RowTableShared(DataView):
'<a href="{base_url}{database}/{table}/{flat_pks_quoted}">{flat_pks}</a>'.format(
base_url=base_url,
database=database,
table=dash_encode(table),
table=tilde_encode(table),
flat_pks=str(markupsafe.escape(pk_path)),
flat_pks_quoted=path_from_row_pks(row, pks, not pks),
)
@ -200,8 +201,8 @@ class RowTableShared(DataView):
link_template.format(
database=database,
base_url=base_url,
table=dash_encode(other_table),
link_id=dash_encode(str(value)),
table=tilde_encode(other_table),
link_id=tilde_encode(str(value)),
id=str(markupsafe.escape(value)),
label=str(markupsafe.escape(label)) or "-",
)
@ -346,6 +347,8 @@ class TableView(RowTableShared):
write=bool(canned_query.get("write")),
)
table = tilde_decode(table)
db = self.ds.databases[database]
is_view = bool(await db.get_view_definition(table))
table_exists = bool(await db.table_exists(table))
@ -766,7 +769,7 @@ class TableView(RowTableShared):
if prefix is None:
prefix = "$null"
else:
prefix = dash_encode(str(prefix))
prefix = tilde_encode(str(prefix))
next_value = f"{prefix},{next_value}"
added_args = {"_next": next_value}
if sort:
@ -938,6 +941,7 @@ class RowView(RowTableShared):
name = "row"
async def data(self, request, database, hash, table, pk_path, default_labels=False):
table = tilde_decode(table)
await self.check_permissions(
request,
[