remote/datasette
1
0
Fork 0
mirror of https://github.com/simonw/datasette.git synced 2026-07-08 16:44:34 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'origin/main' into claude/json-api-docs-1-0-review-a3e83u

Browse source 
# Conflicts:
#	datasette/__init__.py
#	tests/test_api_write.py
This commit is contained in:
Claude 2026-07-06 22:25:27 +00:00
commit 194ee95ae2
No known key found for this signature in database
37 changed files with 4888 additions and 179 deletions
Download patch file Download diff file Expand all files Collapse all files

39
.github/actions/setup-sqlite-version/action.yml vendored Normal file
View file

@ -0,0 +1,39 @@
name: "Setup SQLite version"
description: "Build and activate a specific SQLite version from its amalgamation archive"
inputs:
version:
description: "The SQLite version to install"
required: true
cflags:
description: "CFLAGS to use when compiling SQLite"
required: false
default: ""
skip-activate:
description: "Set to true to skip modifying the library path"
required: false
default: "false"
fallback-urls:
description: "Whitespace-separated fallback download URLs to try after sqlite.org"
required: false
default: ""
outputs:
sqlite-location:
description: "Directory containing the compiled SQLite library"
value: ${{ steps.build.outputs.sqlite-location }}
runs:
using: "composite"
steps:
- shell: bash
run: mkdir -p "$RUNNER_TEMP/sqlite-versions/downloads"
- uses: actions/cache@v6
with:
path: ${{ runner.temp }}/sqlite-versions/downloads
key: setup-sqlite-version-${{ inputs.version }}-amalgamation-v1
- id: build
shell: bash
run: bash "$GITHUB_ACTION_PATH/setup-sqlite-version.sh"
env:
SQLITE_VERSION: ${{ inputs.version }}
SQLITE_CFLAGS: ${{ inputs.cflags }}
SQLITE_SKIP_ACTIVATE: ${{ inputs.skip-activate }}
SQLITE_EXTRA_FALLBACK_URLS: ${{ inputs.fallback-urls }}

144
.github/actions/setup-sqlite-version/setup-sqlite-version.sh vendored Normal file
View file

@ -0,0 +1,144 @@
#!/usr/bin/env bash
set -euo pipefail
version_spec="${SQLITE_VERSION:?SQLITE_VERSION is required}"
cflags="${SQLITE_CFLAGS:-}"
skip_activate="${SQLITE_SKIP_ACTIVATE:-false}"
extra_fallback_urls="${SQLITE_EXTRA_FALLBACK_URLS:-}"
case "$version_spec" in
3.46 | 3.46.0)
sqlite_version="3.46.0"
sqlite_year="2024"
amalgamation_id="3460000"
builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3460000.zip"
;;
3.25 | 3.25.0)
sqlite_version="3.25.0"
sqlite_year="2018"
amalgamation_id="3250000"
builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3250000.zip?v=1"
;;
*)
echo "::error::Unsupported SQLite version '$version_spec'. Add its release year and amalgamation id to $GITHUB_ACTION_PATH/setup-sqlite-version.sh."
exit 1
;;
esac
case "$(uname -s)" in
Linux)
library_name="libsqlite3.so.0"
library_path_var="LD_LIBRARY_PATH"
;;
Darwin)
library_name="libsqlite3.dylib"
library_path_var="DYLD_LIBRARY_PATH"
;;
*)
echo "::error::Unsupported platform $(uname -s)"
exit 1
;;
esac
runner_temp="${RUNNER_TEMP:-}"
if [ -z "$runner_temp" ]; then
runner_temp="$(mktemp -d)"
fi
filename="sqlite-amalgamation-${amalgamation_id}"
official_url="https://www.sqlite.org/${sqlite_year}/${filename}.zip"
download_dir="${runner_temp}/sqlite-versions/downloads"
source_root="${runner_temp}/sqlite-versions/source"
source_dir="${source_root}/${filename}"
build_dir="${runner_temp}/sqlite-versions/build/${sqlite_version}"
archive_path="${download_dir}/${filename}.zip"
mkdir -p "$download_dir" "$source_root" "$build_dir"
download_archive() {
local url
local candidate_path="${archive_path}.tmp"
local urls=("$official_url")
for url in $builtin_fallback_urls $extra_fallback_urls; do
urls+=("$url")
done
rm -f "$candidate_path"
for url in "${urls[@]}"; do
echo "Downloading SQLite ${sqlite_version} amalgamation from ${url}"
if curl \
--fail \
--location \
--show-error \
--retry 5 \
--retry-delay 2 \
--retry-max-time 180 \
--retry-all-errors \
--connect-timeout 20 \
--max-time 240 \
--output "$candidate_path" \
"$url"; then
mv "$candidate_path" "$archive_path"
return 0
fi
echo "::warning::Download failed from ${url}"
rm -f "$candidate_path"
done
echo "::error::Could not download SQLite ${sqlite_version} amalgamation"
return 1
}
if [ ! -f "${source_dir}/sqlite3.c" ]; then
if [ ! -f "$archive_path" ]; then
download_archive
fi
rm -rf "$source_dir"
unzip -q "$archive_path" -d "$source_root"
fi
if [ ! -f "${source_dir}/sqlite3.c" ]; then
echo "::error::Expected ${source_dir}/sqlite3.c after extracting ${archive_path}"
exit 1
fi
read -r -a cflag_args <<< "$cflags"
echo "Compiling SQLite ${sqlite_version} to ${build_dir}/${library_name}"
gcc \
-fPIC \
-shared \
"${cflag_args[@]}" \
"${source_dir}/sqlite3.c" \
"-I${source_dir}" \
-o "${build_dir}/${library_name}"
if [ "$library_name" = "libsqlite3.so.0" ]; then
ln -sf "$library_name" "${build_dir}/libsqlite3.so"
fi
if [ -n "${GITHUB_OUTPUT:-}" ]; then
echo "sqlite-location=${build_dir}" >> "$GITHUB_OUTPUT"
else
echo "sqlite-location=${build_dir}"
fi
case "$(printf '%s' "$skip_activate" | tr '[:upper:]' '[:lower:]')" in
true | 1 | yes)
echo "Skipping ${library_path_var} activation"
;;
*)
existing_value="${!library_path_var:-}"
if [ -n "${GITHUB_ENV:-}" ]; then
if [ -n "$existing_value" ]; then
echo "${library_path_var}=${build_dir}:${existing_value}" >> "$GITHUB_ENV"
else
echo "${library_path_var}=${build_dir}" >> "$GITHUB_ENV"
fi
fi
echo "Added ${build_dir} to ${library_path_var}"
;;
esac

2
.github/workflows/test-sqlite-support.yml vendored
View file

@ -34,7 +34,7 @@ jobs:
cache: pip
cache-dependency-path: pyproject.toml
- name: Set up SQLite ${{ matrix.sqlite-version }}
uses: asg017/sqlite-versions@71ea0de37ae739c33e447af91ba71dda8fcf22e6
uses: ./.github/actions/setup-sqlite-version
with:
version: ${{ matrix.sqlite-version }}
cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1"

22
.github/workflows/test.yml vendored
View file

@ -51,3 +51,25 @@ jobs:
run: |
pip install datasette-init datasette-json-html
tests/test-datasette-load-plugins.sh
test-sqlite-utils-4:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: "3.13"
cache: pip
cache-dependency-path: pyproject.toml
- name: Build extension for --load-extension test
run: |-
(cd tests && gcc ext.c -fPIC -shared -o ext.so)
- name: Install dependencies
run: |
pip install . --group dev
pip install --pre 'sqlite-utils>=4.0rc3'
pip freeze
- name: Run tests
run: |
pytest -n auto -m "not serial"
pytest -m "serial"

3
Justfile
View file

@ -33,10 +33,11 @@ export DATASETTE_SECRET := "not_a_secret"
uv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt
uv run codespell tests --ignore-words docs/codespell-ignore-words.txt
# Run linters: black, ruff, cog
# Run linters: black, ruff, prettier, cog
@lint: codespell
uv run black datasette tests --check
uv run ruff check datasette tests
npm run prettier -- --check
uv run cog --check README.md docs/*.rst
# Apply ruff fixes

8
datasette/__init__.py
View file

@ -2,7 +2,13 @@ from datasette.permissions import Permission # noqa
from datasette.version import __version_info__, __version__ # noqa
from datasette.events import Event # noqa
from datasette.tokens import TokenHandler, TokenInvalid, TokenRestrictions # noqa
from datasette.utils.asgi import Forbidden, NotFound, Request, Response # noqa
from datasette.utils.asgi import ( # noqa
Forbidden,
NotFound,
PayloadTooLarge,
Request,
Response,
)
from datasette.utils import actor_matches_allow # noqa
from datasette.views import Context # noqa
from .hookspecs import hookimpl # noqa

11
datasette/app.py
View file

@ -209,6 +209,11 @@ SETTINGS = (
100,
"Maximum rows that can be inserted at a time using the bulk insert API",
),
Setting(
"max_post_body_bytes",
2 * 1024 * 1024,
"Maximum size in bytes for a POST body read into memory, e.g. JSON API requests - set 0 to disable this limit",
),
Setting(
"num_sql_threads",
3,
@ -2874,7 +2879,11 @@ class DatasetteRouter:
if base_url != "/" and path.startswith(base_url):
path = "/" + path[len(base_url) :]
scope = dict(scope, route_path=path)
request = Request(scope, receive)
request = Request(
scope,
receive,
max_post_body_bytes=self.ds.setting("max_post_body_bytes"),
)
# Populate request_messages if ds_messages cookie is present
try:
request._messages = self.ds.unsign(

12
datasette/default_actions.py
View file

@ -61,6 +61,12 @@ def register_actions():
description="Create tables",
resource_class=DatabaseResource,
),
Action(
name="create-view",
abbr="cv",
description="Create views",
resource_class=DatabaseResource,
),
Action(
name="store-query",
abbr="sq",
@ -111,6 +117,12 @@ def register_actions():
description="Drop tables",
resource_class=TableResource,
),
Action(
name="drop-view",
abbr="dv",
description="Drop views",
resource_class=TableResource,
),
# Query-level actions (child-level)
Action(
name="view-query",

469
datasette/static/app.css
View file

@ -1641,6 +1641,11 @@ dialog.row-edit-dialog::backdrop {
overflow-y: auto;
}
.row-edit-fields[hidden],
.row-edit-bulk[hidden] {
display: none;
}
.row-edit-field {
display: grid;
grid-template-columns: minmax(120px, 180px) minmax(0, 1fr);
@ -1700,6 +1705,118 @@ textarea.row-edit-input {
background: var(--paper);
}
.row-edit-binary-control {
display: grid;
gap: 8px;
box-sizing: border-box;
width: 100%;
min-width: 0;
border: 1px solid var(--rule);
border-radius: 5px;
padding: 10px;
background: #fff;
}
.row-edit-binary-control:focus {
border-color: var(--accent);
outline: 3px solid rgba(26, 86, 219, 0.12);
}
.row-edit-binary-preview[hidden] {
display: none;
}
.row-edit-binary-preview img {
display: block;
max-width: min(240px, 100%);
max-height: 180px;
border: 1px solid var(--rule);
border-radius: 4px;
background: var(--paper);
}
.row-edit-binary-status {
display: flex;
flex-wrap: wrap;
align-items: baseline;
gap: 8px;
min-width: 0;
}
.row-edit-binary-size {
font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
font-size: 0.86rem;
}
.row-edit-binary-name {
color: var(--muted);
font-size: 0.82rem;
overflow-wrap: anywhere;
}
.row-edit-binary-name[hidden] {
display: none;
}
.row-edit-binary-actions {
display: flex;
flex-wrap: wrap;
gap: 8px;
}
.row-edit-binary-file-button,
.row-edit-binary-clear {
appearance: none;
border: 1px solid var(--rule);
border-radius: 4px;
background: #fff;
color: var(--accent);
cursor: pointer;
font: inherit;
font-size: 0.78rem;
line-height: 1.2;
padding: 6px 8px;
}
.row-edit-binary-file-button:hover,
.row-edit-binary-file-button:focus-within,
.row-edit-binary-clear:hover,
.row-edit-binary-clear:focus {
background: #f8fafc;
}
.row-edit-binary-file-button:focus-within,
.row-edit-binary-clear:focus {
outline: 3px solid rgba(26, 86, 219, 0.12);
outline-offset: 1px;
}
.row-edit-binary-file-button input[type="file"] {
position: absolute;
width: 1px;
height: 1px;
opacity: 0;
overflow: hidden;
}
.row-edit-binary-clear[hidden] {
display: none;
}
.row-edit-binary-drop-target {
border: 1px dashed var(--rule);
border-radius: 4px;
padding: 7px 8px;
color: var(--muted);
font-size: 0.78rem;
}
.row-edit-binary-dragover .row-edit-binary-drop-target {
border-color: var(--accent);
background: var(--paper);
color: var(--ink);
}
.row-edit-default {
display: grid;
grid-template-columns: minmax(0, 1fr) 7.25rem;
@ -1798,6 +1915,207 @@ textarea.row-edit-input {
margin: 0;
}
.row-edit-bulk {
display: grid;
gap: 8px;
padding: 16px 24px 24px;
overflow-y: auto;
}
.row-edit-bulk-editor {
display: grid;
gap: 8px;
}
.row-edit-bulk-editor[hidden] {
display: none;
}
.row-edit-bulk-actions {
display: flex;
align-items: center;
flex-wrap: wrap;
gap: 8px;
justify-content: flex-start;
}
.row-edit-bulk-actions .btn {
padding-left: 12px;
padding-right: 12px;
}
.row-edit-bulk-conflict {
display: grid;
grid-template-columns: minmax(120px, 180px) minmax(0, 1fr);
gap: 8px 12px;
align-items: start;
}
.row-edit-bulk-conflict[hidden] {
display: none;
}
.row-edit-bulk-conflict-label {
color: var(--ink);
font-size: 0.82rem;
padding-top: 8px;
}
.row-edit-bulk-conflict-control {
display: grid;
gap: 4px;
}
.row-edit-bulk-conflict-help {
color: var(--muted);
font-size: 0.78rem;
margin: 0;
}
.row-edit-copy-template-label-narrow {
display: none;
}
.row-edit-bulk-template-note {
color: var(--muted);
font-size: 0.82rem;
}
.row-edit-bulk-template-note-narrow {
display: none;
}
.row-edit-bulk-textarea {
min-height: 16rem;
resize: vertical;
font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
font-size: 0.82rem;
line-height: 1.45;
}
.row-edit-bulk-textarea.row-edit-bulk-drop-target {
border-color: var(--accent);
background: #f8fbff;
outline: 3px solid rgba(26, 86, 219, 0.12);
}
.row-edit-bulk-note {
color: var(--muted);
font-size: 0.82rem;
margin: 0;
}
.row-edit-bulk-note label,
.row-edit-bulk-note .button-as-link {
font: inherit;
}
@media (max-width: 640px) {
.row-edit-copy-template-label-wide {
display: none;
}
.row-edit-copy-template-label-narrow {
display: inline;
}
.row-edit-bulk-template-note-wide {
display: none;
}
.row-edit-bulk-template-note-narrow {
display: inline;
}
}
.row-edit-bulk-preview {
display: grid;
gap: 8px;
margin-top: 8px;
}
.row-edit-bulk-preview[hidden] {
display: none;
}
.row-edit-bulk-preview-summary {
color: var(--ink);
font-size: 0.9rem;
font-weight: 600;
margin: 0;
}
.row-edit-bulk-preview-table-wrap {
border: 1px solid var(--rule);
border-radius: 5px;
max-height: 18rem;
overflow: auto;
background: #fff;
}
.row-edit-bulk-preview-table {
border-collapse: collapse;
font-size: 0.78rem;
min-width: 100%;
width: max-content;
}
.row-edit-bulk-preview-table th,
.row-edit-bulk-preview-table td {
border-bottom: 1px solid var(--rule);
border-right: 1px solid var(--rule);
max-width: 18rem;
overflow-wrap: anywhere;
padding: 6px 8px;
text-align: left;
vertical-align: top;
white-space: normal;
}
.row-edit-bulk-preview-table th {
background: var(--paper);
color: var(--ink);
font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
font-weight: 600;
position: sticky;
top: 0;
z-index: 1;
}
.row-edit-bulk-preview-table tr:last-child td {
border-bottom: none;
}
.row-edit-bulk-preview-table th:last-child,
.row-edit-bulk-preview-table td:last-child {
border-right: none;
}
.row-edit-bulk-preview-null,
.row-edit-bulk-preview-auto {
color: var(--muted);
font-style: italic;
}
.row-edit-bulk-progress {
display: grid;
gap: 6px;
}
.row-edit-bulk-progress[hidden] {
display: none;
}
.row-edit-bulk-progress-bar {
width: 100%;
}
.row-edit-bulk-progress-status {
color: var(--ink);
font-size: 0.9rem;
margin: 0;
}
datasette-autocomplete {
display: block;
position: relative;
@ -1876,6 +2194,16 @@ datasette-autocomplete input[type="text"],
background: var(--paper);
}
.row-edit-mode-link {
color: var(--accent);
font-size: 0.9rem;
margin-right: auto;
}
.row-edit-mode-link[hidden] {
display: none;
}
.row-edit-dialog .btn {
border: none;
border-radius: 5px;
@ -2072,6 +2400,120 @@ select.table-create-input {
gap: 10px;
}
.table-create-columns[hidden],
.table-create-data[hidden],
.table-create-data-editor[hidden],
.table-create-data-preview[hidden] {
display: none;
}
.table-create-data,
.table-create-data-editor {
display: grid;
gap: 8px;
}
.table-create-data-label {
color: var(--ink);
font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
font-size: 0.82rem;
}
.table-create-data-textarea {
min-height: 16rem;
resize: vertical;
font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
font-size: 0.82rem;
line-height: 1.45;
}
.table-create-data-textarea.table-create-data-drop-target {
border-color: var(--accent);
background: #f8fbff;
outline: 3px solid rgba(26, 86, 219, 0.12);
}
.table-create-data-note {
color: var(--muted);
font-size: 0.82rem;
margin: 0;
}
.table-create-data-note label,
.table-create-data-note .button-as-link {
font: inherit;
}
.table-create-data-preview {
display: grid;
gap: 10px;
}
.table-create-data-preview-summary {
color: var(--ink);
font-size: 0.9rem;
font-weight: 600;
margin: 0;
}
.table-create-data-pk-field {
display: grid;
grid-template-columns: minmax(120px, 180px) minmax(0, 1fr);
gap: 12px;
align-items: center;
}
.table-create-data-preview-table-wrap {
border: 1px solid var(--rule);
border-radius: 5px;
max-height: 18rem;
overflow: auto;
background: #fff;
}
.table-create-data-preview-table {
border-collapse: collapse;
font-size: 0.78rem;
min-width: 100%;
width: max-content;
}
.table-create-data-preview-table th,
.table-create-data-preview-table td {
border-bottom: 1px solid var(--rule);
border-right: 1px solid var(--rule);
max-width: 18rem;
overflow-wrap: anywhere;
padding: 6px 8px;
text-align: left;
vertical-align: top;
white-space: normal;
}
.table-create-data-preview-table th {
background: var(--paper);
color: var(--ink);
font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
font-weight: 600;
position: sticky;
top: 0;
z-index: 1;
}
.table-create-data-preview-table tr:last-child td {
border-bottom: none;
}
.table-create-data-preview-table th:last-child,
.table-create-data-preview-table td:last-child {
border-right: none;
}
.table-create-data-preview-null {
color: var(--muted);
font-style: italic;
}
.table-create-column-list {
display: grid;
gap: 8px;
@ -2299,6 +2741,16 @@ select.table-create-input {
background: var(--paper);
}
.table-create-mode-link {
color: var(--accent);
font-size: 0.9rem;
margin-right: auto;
}
.table-create-mode-link[hidden] {
display: none;
}
.table-create-dialog .btn {
border: none;
border-radius: 5px;
@ -3012,7 +3464,8 @@ select.table-alter-input {
.row-edit-dialog .modal-header,
.row-edit-summary,
.row-edit-loading,
.row-edit-fields {
.row-edit-fields,
.row-edit-bulk {
padding-left: 18px;
padding-right: 18px;
}
@ -3031,6 +3484,15 @@ select.table-alter-input {
padding-top: 0;
}
.row-edit-bulk-conflict {
grid-template-columns: 1fr;
gap: 5px;
}
.row-edit-bulk-conflict-label {
padding-top: 0;
}
.row-edit-dialog .modal-footer {
padding-left: 18px;
padding-right: 18px;
@ -3062,6 +3524,11 @@ select.table-alter-input {
padding-top: 0;
}
.table-create-data-pk-field {
grid-template-columns: 1fr;
gap: 5px;
}
.table-create-column-headings {
display: none;
}

2403
datasette/static/edit-tools.js
View file

File diff suppressed because it is too large Load diff

48
datasette/utils/__init__.py
View file

@ -1,4 +1,5 @@
import asyncio
import binascii
from contextlib import contextmanager
import aiofiles
import click
@ -236,10 +237,8 @@ class CustomJSONEncoder(json.JSONEncoder):
- ``sqlite3.Row`` becomes a tuple
- ``sqlite3.Cursor`` becomes a list
If a binary blob can be decoded as UTF-8, the encoder returns it as text.
If it can't (for example, images), it is encoded as an object, with the actual
data base64-encoded, like so: ::
Binary blobs are encoded as an object, with the actual data base64-encoded,
like so: ::
{
"$base64": True,
@ -255,17 +254,42 @@ class CustomJSONEncoder(json.JSONEncoder):
if isinstance(obj, sqlite3.Cursor):
return list(obj)
if isinstance(obj, bytes):
# Does it encode to utf8?
try:
return obj.decode("utf8")
except UnicodeDecodeError:
return {
"$base64": True,
"encoded": base64.b64encode(obj).decode("latin1"),
}
return {
"$base64": True,
"encoded": base64.b64encode(obj).decode("latin1"),
}
return json.JSONEncoder.default(self, obj)
class WriteJsonValueError(ValueError):
pass
def decode_write_json_cell(value):
if not isinstance(value, dict):
return value
keys = set(value.keys())
if keys == {"$raw"}:
return value["$raw"]
if keys == {"$base64", "encoded"} and value.get("$base64") is True:
encoded = value["encoded"]
if not isinstance(encoded, str):
raise WriteJsonValueError("$base64 encoded value must be a string")
try:
return base64.b64decode(encoded, validate=True)
except binascii.Error as ex:
raise WriteJsonValueError("Invalid $base64 encoded value") from ex
return value
def decode_write_json_row(row):
return {key: decode_write_json_cell(value) for key, value in row.items()}
def decode_write_json_rows(rows):
return [decode_write_json_row(row) for row in rows]
@contextmanager
def sqlite_timelimit(conn, ms):
deadline = time.perf_counter() + (ms / 1000)

50
datasette/utils/asgi.py
View file

@ -67,13 +67,25 @@ class BadRequest(Base400):
status = 400
class PayloadTooLarge(Base400):
status = 413
SAMESITE_VALUES = ("strict", "lax", "none")
# Bodies read fully into memory (post_body/post_vars/json) are capped at this
# size unless the max_post_body_bytes setting says otherwise. Kept deliberately
# far below multipart's DEFAULT_MAX_REQUEST_SIZE: that parser streams to disk,
# while these bodies are held in RAM and json.loads() can multiply their
# footprint several times over.
DEFAULT_MAX_POST_BODY_BYTES = 2 * 1024 * 1024 # 2MB
class Request:
def __init__(self, scope, receive):
def __init__(self, scope, receive, max_post_body_bytes=DEFAULT_MAX_POST_BODY_BYTES):
self.scope = scope
self.receive = receive
self.max_post_body_bytes = max_post_body_bytes
def __repr__(self):
return '<asgi.Request method="{}" url="{}">'.format(self.method, self.url)
@ -141,15 +153,43 @@ class Request:
def actor(self):
return self.scope.get("actor", None)
async def post_body(self):
body = b""
async def post_body(self, max_bytes=None):
"""
Read the request body fully into memory.
The body is capped at max_bytes - or self.max_post_body_bytes
(default 2MB, set from the max_post_body_bytes setting for requests
created by Datasette) if max_bytes is not provided. Pass max_bytes=0
to disable the limit. Raises PayloadTooLarge (HTTP 413) if exceeded -
oversized bodies are rejected as soon as the limit is passed, without
buffering the rest.
"""
if max_bytes is None:
max_bytes = self.max_post_body_bytes
too_large = PayloadTooLarge(
"Request body exceeded maximum size of {} bytes".format(max_bytes)
)
if max_bytes:
# Reject early if the client declares an oversized body
try:
if int(self.headers.get("content-length", "")) > max_bytes:
raise too_large
except ValueError:
# Missing or malformed - the streaming check below still applies
pass
chunks = []
received = 0
more_body = True
while more_body:
message = await self.receive()
assert message["type"] == "http.request", message
body += message.get("body", b"")
chunk = message.get("body", b"")
received += len(chunk)
if max_bytes and received > max_bytes:
raise too_large
chunks.append(chunk)
more_body = message.get("more_body", False)
return body
return b"".join(chunks)
async def post_vars(self):
body = await self.post_body()

6
datasette/utils/sql_analysis.py
View file

@ -488,17 +488,17 @@ def analyze_sql_tables(
and key.operation in {"create", "alter", "drop"}
for key in operations
)
dropped_tables = {
dropped_tables_and_views = {
(key.database, key.table)
for key in operations
if key.operation == "drop" and key.target_type == "table"
if key.operation == "drop" and key.target_type in {"table", "view"}
}
def key_is_drop_table_delete(key: OperationKey) -> bool:
return (
key.operation == "delete"
and key.target_type == "table"
and (key.database, key.table) in dropped_tables
and (key.database, key.table) in dropped_tables_and_views
)
has_user_table_access_in_schema_operation = any(

2
datasette/views/database.py
View file

@ -326,7 +326,7 @@ class DatabaseContext(Context):
database_color: str = field(metadata={"help": "The color assigned to the database"})
database_page_data: dict = field(
metadata={
"help": 'JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions`` and optional ``customColumnTypes``.'
"help": 'JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions``, ``canInsertRows`` and optional ``customColumnTypes``.'
}
)
database_actions: callable = field(

13
datasette/views/row.py
View file

@ -8,7 +8,7 @@ from dataclasses import dataclass, field
import markupsafe
import sqlite_utils
from datasette.utils.asgi import NotFound, Forbidden, Response
from datasette.utils.asgi import NotFound, Forbidden, PayloadTooLarge, Response
from datasette.database import QueryInterrupted
from datasette.events import UpdateRowEvent, DeleteRowEvent
from datasette.resources import TableResource
@ -17,7 +17,9 @@ from datasette.utils import (
add_cors_headers,
await_me_maybe,
call_with_supported_arguments,
CustomJSONEncoder,
CustomRow,
decode_write_json_row,
InvalidSql,
make_slot_function,
path_from_row_pks,
@ -26,6 +28,7 @@ from datasette.utils import (
to_css_class,
escape_sqlite,
sqlite3,
WriteJsonValueError,
)
from datasette.plugins import pm
from datasette.extras import extra_names_from_request, ExtraScope
@ -785,6 +788,8 @@ class RowUpdateView(BaseView):
data = await request.json()
except json.JSONDecodeError as e:
return _error(["Invalid JSON: {}".format(e)])
except PayloadTooLarge as e:
return _error([str(e)], 413)
if not isinstance(data, dict):
return _error(["JSON must be a dictionary"])
@ -796,6 +801,10 @@ class RowUpdateView(BaseView):
return _error(["Invalid keys: {}".format(", ".join(invalid_keys))])
update = data["update"]
try:
update = decode_write_json_row(update)
except WriteJsonValueError as e:
return _error([str(e)], 400)
# Validate column types
from datasette.views.table import _validate_column_types
@ -857,4 +866,4 @@ class RowUpdateView(BaseView):
self.ds.INFO,
)
return Response.json(result, status=200)
return Response.json(result, status=200, default=CustomJSONEncoder().default)

88
datasette/views/table.py
View file

@ -22,9 +22,11 @@ from datasette.utils import (
add_cors_headers,
await_me_maybe,
call_with_supported_arguments,
CustomJSONEncoder,
CustomRow,
append_querystring,
compound_keys_after_sql,
decode_write_json_rows,
format_bytes,
make_slot_function,
tilde_encode,
@ -41,9 +43,17 @@ from datasette.utils import (
urlsafe_components,
value_as_boolean,
InvalidSql,
WriteJsonValueError,
sqlite3,
)
from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Request, Response
from datasette.utils.asgi import (
BadRequest,
Forbidden,
NotFound,
PayloadTooLarge,
Request,
Response,
)
from datasette.filters import Filters
import sqlite_utils
from dataclasses import dataclass, field
@ -206,7 +216,7 @@ class TableContext(Context):
)
table_insert_ui: dict = field(
metadata={
"help": "Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns`` and ``primaryKeys`` keys; each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``value_kind`` and ``column_type`` keys."
"help": "Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns``, ``bulkColumns``, ``primaryKeys`` and ``maxInsertRows`` keys, plus optional ``upsertPath`` if the current actor has permission to update rows. ``columns`` lists columns for the single-row insert form, while ``bulkColumns`` lists columns for the bulk insert form. Each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``is_auto_pk``, ``value_kind`` and ``column_type`` keys."
}
)
table_alter_ui: dict = field(
@ -481,8 +491,15 @@ async def _table_insert_ui(
):
return None
can_update = await datasette.allowed(
action="update-row",
resource=TableResource(database=database_name, table=table_name),
actor=request.actor,
)
column_types_map = await datasette.get_column_types(database_name, table_name)
columns = []
bulk_columns = []
column_details = await db.table_column_details(table_name)
for column in column_details:
if column.hidden:
@ -493,32 +510,40 @@ async def _table_insert_ui(
and len(pks) == 1
and SQLiteType.from_declared_type(column.type) == SQLiteType.INTEGER
)
column_type = column_types_map.get(column.name)
column_data = {
"name": column.name,
"sqlite_type": _column_sqlite_type_for_insert_form(column),
"notnull": column.notnull,
"default": column.default_value,
"has_default": column.default_value is not None,
"is_pk": is_pk,
"is_auto_pk": is_auto_pk,
"value_kind": _column_value_kind_for_insert_form(column),
"column_type": (
{"type": column_type.name, "config": column_type.config}
if column_type is not None
else None
),
}
bulk_columns.append(column_data)
if is_auto_pk:
continue
column_type = column_types_map.get(column.name)
columns.append(
{
"name": column.name,
"sqlite_type": _column_sqlite_type_for_insert_form(column),
"notnull": column.notnull,
"default": column.default_value,
"has_default": column.default_value is not None,
"is_pk": is_pk,
"value_kind": _column_value_kind_for_insert_form(column),
"column_type": (
{"type": column_type.name, "config": column_type.config}
if column_type is not None
else None
),
}
)
columns.append(column_data)
return {
data = {
"path": "{}/-/insert".format(datasette.urls.table(database_name, table_name)),
"tableName": table_name,
"columns": columns,
"bulkColumns": bulk_columns,
"primaryKeys": pks,
"maxInsertRows": datasette.setting("max_insert_rows"),
}
if can_update:
data["upsertPath"] = "{}/-/upsert".format(
datasette.urls.table(database_name, table_name)
)
return data
async def _table_alter_ui(
@ -1053,11 +1078,18 @@ class TableInsertView(BaseView):
pks = await db.primary_keys(table_name)
rows, errors, extras = await self._validate_data(
request, db, table_name, pks, upsert
)
try:
rows, errors, extras = await self._validate_data(
request, db, table_name, pks, upsert
)
except PayloadTooLarge as e:
return _error([str(e)], 413)
if errors:
return _error(errors, 400)
try:
rows = decode_write_json_rows(rows)
except WriteJsonValueError as e:
return _error([str(e)], 400)
# Validate column types
ct_errors = await _validate_column_types(
@ -1192,7 +1224,11 @@ class TableInsertView(BaseView):
)
)
return Response.json(result, status=200 if upsert else 201)
return Response.json(
result,
status=200 if upsert else 201,
default=CustomJSONEncoder().default,
)
class TableUpsertView(TableInsertView):
@ -1232,6 +1268,8 @@ class TableSetColumnTypeView(BaseView):
data = await request.json()
except json.JSONDecodeError as e:
return _error(["Invalid JSON: {}".format(e)], 400)
except PayloadTooLarge as e:
return _error([str(e)], 413)
if not isinstance(data, dict):
return _error(["JSON must be a dictionary"], 400)
@ -1350,6 +1388,8 @@ class TableDropView(BaseView):
confirm = data.get("confirm")
except json.JSONDecodeError:
pass
except PayloadTooLarge as e:
return _error([str(e)], 413)
if not confirm:
return Response.json(

17
datasette/views/table_create_alter.py
View file

@ -20,11 +20,13 @@ from datasette.column_types import SQLiteType
from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent
from datasette.resources import DatabaseResource, TableResource
from datasette.utils import (
decode_write_json_rows,
escape_sqlite,
get_outbound_foreign_keys,
table_column_details,
WriteJsonValueError,
)
from datasette.utils.asgi import NotFound, Response
from datasette.utils.asgi import NotFound, PayloadTooLarge, Response
from datasette.utils.sqlite import sqlite_hidden_table_names
from .base import BaseView, _error
@ -267,6 +269,11 @@ async def _create_table_ui_context(
"databaseName": database_name,
"columnTypes": CREATE_TABLE_COLUMN_TYPES,
"defaultExpressions": default_expression_options(),
"canInsertRows": await datasette.allowed(
action="insert-row",
resource=DatabaseResource(database=database_name),
actor=request.actor,
),
}
can_set_column_type = await datasette.allowed(
action="set-column-type",
@ -804,6 +811,8 @@ class TableCreateView(BaseView):
data = await request.json()
except json.JSONDecodeError as e:
return _error(["Invalid JSON: {}".format(e)])
except PayloadTooLarge as e:
return _error([str(e)], 413)
if not isinstance(data, dict):
return _error(["JSON must be an object"])
@ -838,6 +847,10 @@ class TableCreateView(BaseView):
actor=request.actor,
):
return _error(["Permission denied: need insert-row"], 403)
try:
rows = decode_write_json_rows(rows)
except WriteJsonValueError as e:
return _error([str(e)], 400)
alter = False
if rows:
@ -1161,6 +1174,8 @@ class TableAlterView(BaseView):
data = await request.json()
except json.JSONDecodeError as e:
return _error(["Invalid JSON: {}".format(e)], 400)
except PayloadTooLarge as e:
return _error([str(e)], 413)
if not isinstance(data, dict):
return _error(["JSON must be a dictionary"], 400)

51
datasette/views/table_extras.py
View file

@ -1,6 +1,7 @@
import itertools
from dataclasses import dataclass
from datasette.column_types import SQLiteType
from datasette.database import QueryInterrupted
from datasette.extras import Extra, ExtraExample, ExtraRegistry, ExtraScope, Provider
from datasette.plugins import pm
@ -375,6 +376,55 @@ class PrimaryKeysExtra(Extra):
return context.pks
def column_detail_as_json(column):
return {
"type": column.type,
"sqlite_type": SQLiteType.from_declared_type(column.type).value,
"notnull": bool(column.notnull),
"default": column.default_value,
"is_pk": bool(column.is_pk),
"pk_position": column.is_pk,
"hidden": column.hidden,
}
class ColumnDetailsExtra(Extra):
description = (
"SQLite schema details for columns in this table. The dictionary maps "
"column names to objects describing the schema for each column."
)
docs_note = (
"Each object has ``type`` as the declared type string returned by "
'SQLite, or ``""`` if no type was declared; ``sqlite_type`` as the '
"normalized SQLite affinity, one of ``TEXT``, ``INTEGER``, ``REAL``, "
"``BLOB`` or ``NUMERIC``; ``notnull`` as a boolean; ``default`` "
'as the raw SQL default expression string, such as ``"42"``, '
"``\"'hello'\"`` or ``\"datetime('now')\"``, or ``null`` if there is "
"no default; ``is_pk`` as a boolean; ``pk_position`` as the integer "
"primary key position reported by SQLite, or ``0`` for columns that "
"are not part of the primary key; and ``hidden`` as the integer value "
"reported by SQLite's ``PRAGMA table_xinfo``. ``hidden`` is ``0`` for "
"normal columns, ``1`` for hidden virtual table columns, ``2`` for "
"virtual generated columns and ``3`` for stored generated columns."
)
example = ExtraExample("/fixtures/binary_data.json?_size=0&_extra=column_details")
examples = {
ExtraScope.ROW: ExtraExample(
"/fixtures/binary_data/1.json?_extra=column_details"
)
}
scopes = {ExtraScope.TABLE, ExtraScope.ROW}
async def resolve(self, context):
column_details = await context.datasette._get_resource_column_details(
context.database_name, context.table_name
)
return {
column_name: column_detail_as_json(column)
for column_name, column in column_details.items()
}
class ActionsExtra(Extra):
description = 'Async callable returning table or view actions made available by core and plugin hooks. Each item is either a link with ``href``, ``label`` and optional ``description`` keys, or a button with ``type: "button"``, ``label``, optional ``description`` and optional ``attrs``. See :ref:`plugin_actions`, :ref:`plugin_hook_table_actions` and :ref:`plugin_hook_view_actions`.'
scopes = {ExtraScope.TABLE}
@ -1240,6 +1290,7 @@ TABLE_EXTRA_CLASSES = [
ColumnsExtra,
AllColumnsExtra,
PrimaryKeysExtra,
ColumnDetailsExtra,
DisplayColumnsAndRowsProvider,
DisplayColumnsExtra,
DisplayRowsExtra,

27
datasette/write_sql.py
View file

@ -138,6 +138,33 @@ def decision_for_write_sql_operation(
),
)
)
if operation.operation == "create" and operation.target_type == "view":
if operation.database is None:
return UnsupportedWriteSqlOperation(unsupported_message)
return RequireWriteSqlPermissions(
(
PermissionRequirement(
action="create-view",
resource=DatabaseResource(database=operation.database),
),
)
)
if (
operation.operation == "drop"
and operation.target_type == "view"
and operation.database is not None
and operation.table is not None
):
return RequireWriteSqlPermissions(
(
PermissionRequirement(
action="drop-view",
resource=TableResource(
database=operation.database, table=operation.table
),
),
)
)
if (
operation.operation == "alter"
and operation.target_type == "table"

24
docs/authentication.rst
View file

@ -50,7 +50,7 @@ The one exception is the "root" account, which you can sign into while using Dat
The ``--root`` flag is designed for local development and testing. When you start Datasette with ``--root``, the root user automatically receives every permission, including:
* All view permissions (``view-instance``, ``view-database``, ``view-table``, etc.)
* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``alter-table``, ``set-column-type``, ``drop-table``)
* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``create-view``, ``alter-table``, ``set-column-type``, ``drop-table``, ``drop-view``)
* Debug permissions (``permissions-debug``, ``debug-menu``)
* Any custom permissions defined by plugins
@ -1390,6 +1390,16 @@ create-table
Actor is allowed to create a database table.
``resource`` - ``datasette.resources.DatabaseResource(database)``
``database`` is the name of the database (string)
.. _actions_create_view:
create-view
-----------
Actor is allowed to create a database view.
``resource`` - ``datasette.resources.DatabaseResource(database)``
``database`` is the name of the database (string)
@ -1429,6 +1439,18 @@ Actor is allowed to drop a database table.
``table`` is the name of the table (string)
.. _actions_drop_view:
drop-view
---------
Actor is allowed to drop a database view.
``resource`` - ``datasette.resources.TableResource(database, table)``
``database`` is the name of the database (string)
``table`` is the name of the view (string)
.. _actions_execute_sql:
execute-sql

49
docs/binary_data.rst
View file

@ -12,7 +12,12 @@ Datasette includes special handling for these binary values. The Datasette inter
:width: 311px
:alt: Screenshot showing download links next to binary data in the table view
Binary data is represented in ``.json`` exports using Base64 encoding.
.. _binary_json_format:
Binary values in JSON
---------------------
Binary data is represented in ``.json`` exports using Base64 encoding. Datasette uses this representation for every ``BLOB`` value, including binary values that could also be decoded as UTF-8 text.
https://latest.datasette.io/fixtures/binary_data.json?_shape=array
@ -39,6 +44,48 @@ https://latest.datasette.io/fixtures/binary_data.json?_shape=array
}
]
The same format can be used with the :ref:`JSON write API <json_api_write>`.
If a column value in a ``row``, ``rows`` or ``update`` object is a JSON object with exactly ``"$base64"`` set to ``true`` and an ``"encoded"`` string, Datasette will decode that Base64 string and store the resulting bytes:
.. code-block:: json
{
"data": {
"$base64": true,
"encoded": "FRwCx60F/g=="
}
}
This works for inserts, upserts and updates. It also works when creating a table from example ``row`` or ``rows`` data: Datasette decodes the value before inferring the schema, allowing that column to be created as a ``BLOB`` column.
To store a JSON object with that exact shape literally, wrap it in a ``"$raw"`` object:
.. code-block:: json
{
"data": {
"$raw": {
"$base64": true,
"encoded": "FRwCx60F/g=="
}
}
}
``"$raw"`` unwraps exactly one layer. To store a literal ``"$raw"`` object containing a Base64 object, wrap it again:
.. code-block:: json
{
"data": {
"$raw": {
"$raw": {
"$base64": true,
"encoded": "FRwCx60F/g=="
}
}
}
}
.. _binary_linking:
Linking to binary downloads

13
docs/changelog.rst
View file

@ -4,6 +4,19 @@
Changelog
=========
.. _unreleased:
Unreleased
----------
- Table pages now offer an "Insert multiple rows" mode in the row insertion dialog. This accepts pasted TSV, CSV or JSON, previews the parsed rows before inserting them, validates unknown columns as data is pasted and displays omitted auto integer primary keys as ``auto`` in the preview. (:pr:`2813`)
- The bulk insert UI can skip rows with existing primary keys, or update existing rows and insert new rows using the existing ``/<database>/<table>/-/upsert`` API when the actor has both :ref:`insert-row <actions_insert_row>` and :ref:`update-row <actions_update_row>` permissions. (:pr:`2813`)
- The "Create table" dialog now includes a "Create table from data" mode. Paste TSV, CSV or JSON rows to preview inferred columns and types, choose the table name and primary key, then create the table and insert those rows in one step. (:pr:`2813`)
- Datasette's JSON APIs now consistently encode every ``BLOB`` value using the documented :ref:`binary value JSON format <binary_json_format>`, even when the bytes could be decoded as UTF-8 text.
- The insert and edit row dialogs now provide a dedicated control for ``BLOB`` values. Existing binary values are shown by byte size, image values under 10MB are previewed as thumbnails, and replacements can be attached, dropped or pasted into the control.
- The table and row JSON APIs now support ``?_extra=column_details`` for returning SQLite schema details for columns, including declared type, SQLite affinity, primary key, ``NOT NULL``, default and hidden-column metadata.
- POST bodies that Datasette reads fully into memory - such as JSON submitted to the write API - are now capped by the new :ref:`setting_max_post_body_bytes` setting, defaulting to 2MB. Oversized requests are rejected with an HTTP 413 error as soon as the limit is exceeded, protecting smaller servers from memory exhaustion. File uploads are unaffected - ``request.form()`` streams those to disk and has its own separate limits.
.. _v1_0_a35:
1.0a35 (2026-06-23)

3
docs/cli-reference.rst
View file

@ -244,6 +244,9 @@ These can be passed to ``datasette serve`` using ``datasette serve --setting nam
custom query (default=1000)
max_insert_rows Maximum rows that can be inserted at a time using
the bulk insert API (default=100)
max_post_body_bytes Maximum size in bytes for a POST body read into
memory, e.g. JSON API requests - set 0 to disable
this limit (default=2097152)
num_sql_threads Number of threads in the thread pool for
executing SQLite queries (default=3)
sql_time_limit_ms Time limit for a SQL query in milliseconds

7
docs/internals.rst
View file

@ -52,6 +52,9 @@ The request object is passed to various plugin hooks. It represents an incoming
``.actor`` - dictionary (str -> Any) or None
The currently authenticated actor (see :ref:`actors <authentication_actor>`), or ``None`` if the request is unauthenticated.
``.max_post_body_bytes`` - integer
The maximum number of bytes ``await request.post_body()`` will read into memory, or ``0`` for no limit. Set from the :ref:`setting_max_post_body_bytes` setting (default 2MB) for requests created by Datasette. Can be passed to the ``Request`` constructor as a keyword argument.
The object also has the following awaitable methods:
``await request.form(files=False, ...)`` - FormData
@ -109,9 +112,11 @@ The object also has the following awaitable methods:
``await request.json()`` - Any
Returns the parsed JSON body of a request submitted by ``POST``.
``await request.post_body()`` - bytes
``await request.post_body(max_bytes=None)`` - bytes
Returns the un-parsed body of a request submitted by ``POST`` - useful for things like incoming JSON data.
The body is read fully into memory, capped at ``request.max_post_body_bytes`` - which Datasette sets from the :ref:`setting_max_post_body_bytes` setting (default 2MB). Bodies that exceed the limit raise a ``datasette.PayloadTooLarge`` exception, which Datasette turns into an HTTP 413 error response. Pass ``max_bytes=`` to override the limit for a specific call, or ``max_bytes=0`` to disable it. ``request.post_vars()`` and ``request.json()`` read the body through this method, so the same limit applies to them.
And a class method that can be used to create fake request objects for use in tests:
``fake(path_with_query_string, method="GET", scheme="http", url_vars=None)``

48
docs/json_api.rst
View file

@ -497,6 +497,25 @@ The available table extras are listed below.
"pk"
]
``column_details``
SQLite schema details for columns in this table. The dictionary maps column names to objects describing the schema for each column. (Each object has ``type`` as the declared type string returned by SQLite, or ``""`` if no type was declared; ``sqlite_type`` as the normalized SQLite affinity, one of ``TEXT``, ``INTEGER``, ``REAL``, ``BLOB`` or ``NUMERIC``; ``notnull`` as a boolean; ``default`` as the raw SQL default expression string, such as ``"42"``, ``"'hello'"`` or ``"datetime('now')"``, or ``null`` if there is no default; ``is_pk`` as a boolean; ``pk_position`` as the integer primary key position reported by SQLite, or ``0`` for columns that are not part of the primary key; and ``hidden`` as the integer value reported by SQLite's ``PRAGMA table_xinfo``. ``hidden`` is ``0`` for normal columns, ``1`` for hidden virtual table columns, ``2`` for virtual generated columns and ``3`` for stored generated columns.)
``GET /fixtures/binary_data.json?_size=0&_extra=column_details``
.. code-block:: json
{
"data": {
"type": "BLOB",
"sqlite_type": "BLOB",
"notnull": false,
"default": null,
"is_pk": false,
"pk_position": 0,
"hidden": 0
}
}
``display_columns``
Column metadata used by the HTML table display. Each item includes ``name``, ``sortable``, ``is_pk``, ``type``, ``notnull``, ``description``, ``column_type`` and ``column_type_config`` keys.
@ -902,6 +921,25 @@ The following extras are available for row JSON responses.
"id"
]
``column_details``
SQLite schema details for columns in this table. The dictionary maps column names to objects describing the schema for each column. (Each object has ``type`` as the declared type string returned by SQLite, or ``""`` if no type was declared; ``sqlite_type`` as the normalized SQLite affinity, one of ``TEXT``, ``INTEGER``, ``REAL``, ``BLOB`` or ``NUMERIC``; ``notnull`` as a boolean; ``default`` as the raw SQL default expression string, such as ``"42"``, ``"'hello'"`` or ``"datetime('now')"``, or ``null`` if there is no default; ``is_pk`` as a boolean; ``pk_position`` as the integer primary key position reported by SQLite, or ``0`` for columns that are not part of the primary key; and ``hidden`` as the integer value reported by SQLite's ``PRAGMA table_xinfo``. ``hidden`` is ``0`` for normal columns, ``1`` for hidden virtual table columns, ``2`` for virtual generated columns and ``3`` for stored generated columns.)
``GET /fixtures/binary_data/1.json?_extra=column_details``
.. code-block:: json
{
"data": {
"type": "BLOB",
"sqlite_type": "BLOB",
"notnull": false,
"default": null,
"is_pk": false,
"pk_position": 0,
"hidden": 0
}
}
``render_cell``
Rendered HTML for each cell using the render_cell plugin hook (See the :ref:`render_cell() plugin hook <plugin_hook_render_cell>` documentation.)
@ -1626,6 +1664,8 @@ The JSON write API
Datasette provides a write API for JSON data. This is a POST-only API that requires an authenticated API token, see :ref:`CreateTokenView`. The token will need to have the specified :ref:`authentication_permissions`.
The row-based write APIs can write :ref:`binary values in JSON <binary_json_format>` using Datasette's Base64 representation for BLOB data.
.. _ExecuteWriteView:
Executing write SQL
@ -1756,6 +1796,8 @@ A single row can be inserted using the ``"row"`` key:
}
}
Column values can use the :ref:`binary value JSON format <binary_json_format>` to write BLOB data.
If successful, this will return a ``201`` status code and the newly inserted row, for example:
.. code-block:: json
@ -1863,6 +1905,8 @@ An upsert is an insert or update operation. If a row with a matching primary key
The upsert API is mostly the same shape as the :ref:`insert API <TableInsertView>`. It requires both the :ref:`actions_insert_row` and :ref:`actions_update_row` permissions.
It also accepts the same :ref:`binary value JSON format <binary_json_format>`.
::
POST /<database>/<table>/-/upsert
@ -1995,6 +2039,8 @@ To update a row, make a ``POST`` to ``/<database>/<table>/<row-pks>/-/update``.
You only need to pass the columns you want to update. Any other columns will be left unchanged.
Updated values can use the :ref:`binary value JSON format <binary_json_format>`.
If successful, this will return a ``200`` status code and a ``{"ok": true}`` response body.
Add ``"return": true`` to the request body to return the updated row:
@ -2200,6 +2246,8 @@ Datasette will create a table with a schema that matches those rows and insert t
"pk": "id"
}
Example rows can use the :ref:`binary value JSON format <binary_json_format>`, allowing Datasette to infer ``BLOB`` columns.
Doing this requires both the :ref:`actions_create_table` and :ref:`actions_insert_row` permissions.
The ``201`` response here will be similar to the ``columns`` form, but will also include the number of rows that were inserted as ``row_count``:

17
docs/settings.rst
View file

@ -125,6 +125,23 @@ You can increase or decrease this limit like so::
datasette mydatabase.db --setting max_insert_rows 1000
.. _setting_max_post_body_bytes:
max_post_body_bytes
~~~~~~~~~~~~~~~~~~~
Maximum size in bytes for a POST body that Datasette reads fully into memory, such as JSON submitted to the :ref:`write API <json_api_write>`. Requests with larger bodies are rejected with an HTTP 413 error. Defaults to 2,097,152 (2MB).
This limit exists to protect against memory exhaustion: unlike file uploads handled by ``request.form()``, which stream to disk, these bodies are held entirely in memory and parsing them as JSON can multiply their memory footprint several times over.
If you increase :ref:`setting_max_insert_rows` to support larger bulk inserts you may need to increase this limit as well::
datasette mydatabase.db --setting max_post_body_bytes 10485760
Set it to 0 to disable the limit entirely::
datasette mydatabase.db --setting max_post_body_bytes 0
.. _setting_num_sql_threads:
num_sql_threads

4
docs/template_context.rst
View file

@ -98,7 +98,7 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re
The color assigned to the database
``database_page_data`` - ``dict``
JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions`` and optional ``customColumnTypes``.
JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions``, ``canInsertRows`` and optional ``customColumnTypes``.
``editable`` - ``bool``
Boolean indicating if the database is editable
@ -389,7 +389,7 @@ Many of these keys are shared with the :ref:`JSON API <json_api>` for this page.
SQL definition for this table
``table_insert_ui`` - ``dict``
Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns`` and ``primaryKeys`` keys; each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``value_kind`` and ``column_type`` keys.
Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns``, ``bulkColumns``, ``primaryKeys`` and ``maxInsertRows`` keys, plus optional ``upsertPath`` if the current actor has permission to update rows. ``columns`` lists columns for the single-row insert form, while ``bulkColumns`` lists columns for the bulk insert form. Each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``is_auto_pk``, ``value_kind`` and ``column_type`` keys.
``table_page_data`` - ``dict``
JSON data used by JavaScript on the table page. Includes ``database``, ``table`` and ``tableUrl``, plus optional ``foreignKeys`` mapping column names to autocomplete URLs, optional ``insertRow`` data and optional ``alterTable`` data.

2
pyproject.toml
View file

@ -35,7 +35,7 @@ dependencies = [
"PyYAML>=5.3",
"mergedeep>=1.1.1",
"itsdangerous>=1.1",
"sqlite-utils>=3.30,<4.0",
"sqlite-utils>=3.30",
"asyncinject>=0.7",
"setuptools",
"pip",

42
tests/test_api.py
View file

@ -435,7 +435,7 @@ async def test_row_foreign_key_tables(ds_client):
@pytest.mark.asyncio
async def test_row_extras(ds_client):
response = await ds_client.get(
"/fixtures/simple_primary_key/1.json?_extra=database,table,primary_keys,query,request,debug,foreign_key_tables"
"/fixtures/simple_primary_key/1.json?_extra=database,table,primary_keys,query,request,debug,foreign_key_tables,column_details"
)
assert response.status_code == 200
data = response.json()
@ -452,6 +452,45 @@ async def test_row_extras(ds_client):
"format": "json",
}
assert len(data["foreign_key_tables"]) == 5
id_detail = data["column_details"]["id"]
assert id_detail["type"].lower() == "integer"
assert id_detail == {
"type": id_detail["type"],
"sqlite_type": "INTEGER",
"notnull": False,
"default": None,
"is_pk": True,
"pk_position": 1,
"hidden": 0,
}
content_detail = data["column_details"]["content"]
assert content_detail["type"].lower() == "text"
assert content_detail == {
"type": content_detail["type"],
"sqlite_type": "TEXT",
"notnull": False,
"default": None,
"is_pk": False,
"pk_position": 0,
"hidden": 0,
}
@pytest.mark.asyncio
async def test_column_details_extra_row_for_null_blob(ds_client):
response = await ds_client.get("/fixtures/binary_data/3.json?_extra=column_details")
assert response.status_code == 200
data_detail = response.json()["column_details"]["data"]
assert data_detail["type"].lower() == "blob"
assert data_detail == {
"type": data_detail["type"],
"sqlite_type": "BLOB",
"notnull": False,
"default": None,
"is_pk": False,
"pk_position": 0,
"hidden": 0,
}
@pytest.mark.asyncio
@ -627,6 +666,7 @@ async def test_settings_json(ds_client):
"facet_time_limit_ms": 200,
"max_returned_rows": 100,
"max_insert_rows": 100,
"max_post_body_bytes": 2 * 1024 * 1024,
"sql_time_limit_ms": 200,
"allow_download": True,
"allow_signed_tokens": True,

197
tests/test_api_write.py
View file

@ -3,9 +3,31 @@ from datasette.events import RenameTableEvent
from datasette.utils import error_body, escape_sqlite, sqlite3
from .utils import last_event
import pytest
import re
import time
def schema_variants(schema):
# sqlite-utils < 4 quotes identifiers [like_this] and uses FLOAT;
# sqlite-utils >= 4 quotes them "like_this" and uses REAL. Given a
# schema fragment in the old format, return both variants so tests
# can pass against either version.
converted = re.sub(r"\[([^\]]+)\]", r'"\1"', schema).replace("FLOAT", "REAL")
return (schema, converted)
def assert_schema_contains(fragment, schema):
assert any(
variant in schema for variant in schema_variants(fragment)
), "Expected schema to contain {!r}, got {!r}".format(fragment, schema)
def assert_schema_not_contains(fragment, schema):
assert not any(
variant in schema for variant in schema_variants(fragment)
), "Expected schema not to contain {!r}, got {!r}".format(fragment, schema)
@pytest.fixture
def ds_write(tmp_path_factory):
db_directory = tmp_path_factory.mktemp("dbs")
@ -50,6 +72,133 @@ def _insert_and_fetch_created(conn, table, insert_sql):
).fetchone()
BASE64_WRITE_API_VALUE = {"$base64": True, "encoded": "AAEC/f7/"}
BASE64_WRITE_API_LITERAL = '{"$base64": true, "encoded": "AAEC/f7/"}'
@pytest.mark.asyncio
async def test_base64_write_api_create_table_infers_blob_and_raw_escapes(ds_write):
token = write_token(ds_write)
response = await ds_write.client.post(
"/data/-/create",
json={
"table": "binary_create",
"row": {
"id": 1,
"data": BASE64_WRITE_API_VALUE,
"literal": {"$raw": BASE64_WRITE_API_VALUE},
"double_raw": {"$raw": {"$raw": BASE64_WRITE_API_VALUE}},
},
"pk": "id",
},
headers=_headers(token),
)
assert response.status_code == 201
assert_schema_contains("[data] BLOB", response.json()["schema"])
assert_schema_contains("[literal] TEXT", response.json()["schema"])
rows = (await ds_write.get_database("data").execute("""
select
typeof(data) as data_type,
hex(data) as data_hex,
typeof(literal) as literal_type,
literal,
typeof(double_raw) as double_raw_type,
double_raw
from binary_create
""")).dicts()
assert rows == [
{
"data_type": "blob",
"data_hex": "000102FDFEFF",
"literal_type": "text",
"literal": BASE64_WRITE_API_LITERAL,
"double_raw_type": "text",
"double_raw": '{"$raw": {"$base64": true, "encoded": "AAEC/f7/"}}',
}
]
@pytest.mark.asyncio
async def test_base64_write_api_insert_upsert_update_decode_blobs(ds_write):
token = write_token(ds_write)
db = ds_write.get_database("data")
await db.execute_write(
"create table binary_api (id integer primary key, data blob, literal text)"
)
insert_response = await ds_write.client.post(
"/data/binary_api/-/insert",
json={
"row": {
"id": 1,
"data": BASE64_WRITE_API_VALUE,
"literal": {"$raw": BASE64_WRITE_API_VALUE},
}
},
headers=_headers(token),
)
assert insert_response.status_code == 201
assert insert_response.json()["rows"][0]["data"] == BASE64_WRITE_API_VALUE
upsert_response = await ds_write.client.post(
"/data/binary_api/-/upsert",
json={
"rows": [
{
"id": 2,
"data": BASE64_WRITE_API_VALUE,
"literal": {"$raw": BASE64_WRITE_API_VALUE},
}
]
},
headers=_headers(token),
)
assert upsert_response.status_code == 200
assert upsert_response.json() == {"ok": True}
update_response = await ds_write.client.post(
"/data/binary_api/1/-/update",
json={
"update": {
"data": {"$base64": True, "encoded": "/wAB"},
"literal": {"$raw": {"$raw": BASE64_WRITE_API_VALUE}},
},
"return": True,
},
headers=_headers(token),
)
assert update_response.status_code == 200
assert update_response.json()["row"]["data"] == {"$base64": True, "encoded": "/wAB"}
rows = (await db.execute("""
select
id,
typeof(data) as data_type,
hex(data) as data_hex,
typeof(literal) as literal_type,
literal
from binary_api
order by id
""")).dicts()
assert rows == [
{
"id": 1,
"data_type": "blob",
"data_hex": "FF0001",
"literal_type": "text",
"literal": '{"$raw": {"$base64": true, "encoded": "AAEC/f7/"}}',
},
{
"id": 2,
"data_type": "blob",
"data_hex": "000102FDFEFF",
"literal_type": "text",
"literal": BASE64_WRITE_API_LITERAL,
},
]
@pytest.mark.asyncio
async def test_api_explorer_upsert_example_json(ds_write):
response = await ds_write.client.get("/-/api", actor={"id": "root"})
@ -180,6 +329,35 @@ async def test_insert_rows(ds_write, return_rows):
assert response.json()["rows"] == actual_rows
@pytest.mark.asyncio
async def test_insert_rows_post_body_too_large(tmp_path_factory):
db_path = str(tmp_path_factory.mktemp("dbs") / "data.db")
conn = sqlite3.connect(db_path)
conn.execute("create table docs (id integer primary key, title text)")
conn.close()
ds = Datasette([db_path], settings={"max_post_body_bytes": 100})
ds.root_enabled = True
token = write_token(ds)
response = await ds.client.post(
"/data/docs/-/insert",
json={"rows": [{"title": "x" * 200}]},
headers=_headers(token),
)
assert response.status_code == 413
assert response.json() == {
"ok": False,
"errors": ["Request body exceeded maximum size of 100 bytes"],
}
# A small body should still work
response2 = await ds.client.post(
"/data/docs/-/insert",
json={"row": {"title": "hi"}},
headers=_headers(token),
)
assert response2.status_code == 201
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"path,input,special_case,expected_status,expected_errors",
@ -1047,7 +1225,9 @@ async def test_alter_table_foreign_key_operations(ds_write):
assert response.status_code == 200, response.text
data = response.json()
assert data["operations_applied"] == 2
assert "[owner_id] INTEGER REFERENCES [owners]([id])" in data["schema"]
assert_schema_contains(
"[owner_id] INTEGER REFERENCES [owners]([id])", data["schema"]
)
response = await ds_write.client.post(
"/data/docs/-/alter",
@ -1058,7 +1238,7 @@ async def test_alter_table_foreign_key_operations(ds_write):
)
assert response.status_code == 200, response.text
data = response.json()
assert "[owner_id] INTEGER REFERENCES" not in data["schema"]
assert_schema_not_contains("[owner_id] INTEGER REFERENCES", data["schema"])
response = await ds_write.client.post(
"/data/docs/-/alter",
@ -1082,7 +1262,9 @@ async def test_alter_table_foreign_key_operations(ds_write):
)
assert response.status_code == 200, response.text
data = response.json()
assert "[owner_id] INTEGER REFERENCES [categories]([id])" in data["schema"]
assert_schema_contains(
"[owner_id] INTEGER REFERENCES [categories]([id])", data["schema"]
)
response = await ds_write.client.post(
"/data/docs/-/alter",
@ -1091,7 +1273,7 @@ async def test_alter_table_foreign_key_operations(ds_write):
)
assert response.status_code == 200, response.text
data = response.json()
assert "[owner_id] INTEGER REFERENCES" not in data["schema"]
assert_schema_not_contains("[owner_id] INTEGER REFERENCES", data["schema"])
@pytest.mark.asyncio
@ -2019,6 +2201,9 @@ async def test_create_table(
if expected_response.get("ok") is False:
# Error expectations list their messages; derive the canonical envelope
expected_response = error_body(expected_response["errors"], expected_status)
if isinstance(expected_response, dict) and "schema" in expected_response:
assert data.get("schema") in schema_variants(expected_response["schema"])
expected_response = dict(expected_response, schema=data.get("schema"))
assert data == expected_response
# Should have tracked the expected events
events = ds_write._tracked_events
@ -2061,7 +2246,9 @@ async def test_create_table_with_foreign_key(ds_write):
)
assert response.status_code == 201
data = response.json()
assert "[owner_id] INTEGER REFERENCES [owners]([id])" in data["schema"]
assert_schema_contains(
"[owner_id] INTEGER REFERENCES [owners]([id])", data["schema"]
)
@pytest.mark.asyncio

75
tests/test_internal_db.py
View file

@ -1,5 +1,6 @@
import pytest
import sqlite_utils
from datasette.utils import escape_sqlite
# ensure refresh_schemas() gets called before interacting with internal_db
@ -76,19 +77,71 @@ async def test_internal_foreign_key_references(ds_client):
internal_db = await ensure_internal(ds_client)
def inner(conn):
db = sqlite_utils.Database(conn)
table_names = db.table_names()
for table in db.tables:
for fk in table.foreign_keys:
other_table = fk.other_table
other_column = fk.other_column
message = 'Column "{}.{}" references other column "{}.{}" which does not exist'.format(
table.name, fk.column, other_table, other_column
table_names = [
row[0]
for row in conn.execute(
"select name from sqlite_master where type = 'table'"
).fetchall()
]
def columns_for_table(table_name):
return {
row[1]
for row in conn.execute(
"PRAGMA table_info({})".format(escape_sqlite(table_name))
).fetchall()
}
def primary_keys_for_table(table_name):
return [
name
for _, name in sorted(
(row[5], row[1])
for row in conn.execute(
"PRAGMA table_info({})".format(escape_sqlite(table_name))
).fetchall()
if row[5]
)
]
columns_by_table = {
table_name: columns_for_table(table_name) for table_name in table_names
}
for table_name in table_names:
foreign_key_rows = conn.execute(
"PRAGMA foreign_key_list({})".format(escape_sqlite(table_name))
).fetchall()
foreign_keys_by_id = {}
for foreign_key in foreign_key_rows:
foreign_keys_by_id.setdefault(foreign_key[0], []).append(foreign_key)
for foreign_key_rows in foreign_keys_by_id.values():
foreign_key_rows.sort(key=lambda row: row[1])
other_table = foreign_key_rows[0][2]
other_columns = [row[4] for row in foreign_key_rows]
message = 'Column "{}.{}" references other table "{}" which does not exist'.format(
table_name, foreign_key_rows[0][3], other_table
)
assert other_table in table_names, message + " (bad table)"
assert other_column in db[other_table].columns_dict, (
message + " (bad column)"
if all(other_column is None for other_column in other_columns):
other_columns = primary_keys_for_table(other_table)
length_message = 'Foreign key from "{}" to "{}" has {} columns but references {} columns'.format(
table_name,
other_table,
len(foreign_key_rows),
len(other_columns),
)
assert len(other_columns) == len(foreign_key_rows), length_message
for foreign_key, other_column in zip(foreign_key_rows, other_columns):
column = foreign_key[3]
message = 'Column "{}.{}" references other column "{}.{}" which does not exist'.format(
table_name, column, other_table, other_column
)
assert other_column in columns_by_table[other_table], (
message + " (bad column)"
)
await internal_db.execute_fn(inner)

96
tests/test_internals_request.py
View file

@ -1,8 +1,38 @@
from datasette.utils.asgi import Request
from datasette.utils.asgi import PayloadTooLarge, Request
import json
import pytest
def _post_scope(headers=None):
return {
"http_version": "1.1",
"method": "POST",
"path": "/",
"raw_path": b"/",
"query_string": b"",
"scheme": "http",
"type": "http",
"headers": headers or [[b"content-type", b"application/json"]],
}
def _receive_chunks(chunks):
messages = [
{
"type": "http.request",
"body": chunk,
"more_body": i < len(chunks) - 1,
}
for i, chunk in enumerate(chunks)
]
messages.reverse()
async def receive():
return messages.pop()
return receive
@pytest.mark.asyncio
async def test_request_post_vars():
scope = {
@ -106,6 +136,70 @@ async def test_request_json_invalid():
await request.json()
@pytest.mark.asyncio
async def test_request_post_body_multiple_chunks():
request = Request(_post_scope(), _receive_chunks([b"hello ", b"world"]))
assert await request.post_body() == b"hello world"
@pytest.mark.asyncio
async def test_request_post_body_content_length_too_large():
# Should reject based on content-length without reading the body
async def receive():
raise AssertionError("receive() should not be called")
scope = _post_scope(
headers=[
[b"content-type", b"application/json"],
[b"content-length", b"101"],
]
)
request = Request(scope, receive)
with pytest.raises(PayloadTooLarge):
await request.post_body(max_bytes=100)
@pytest.mark.asyncio
async def test_request_post_body_streaming_too_large():
# No content-length header - limit enforced as chunks arrive
chunks = [b"a" * 60, b"b" * 60, b"c" * 60]
request = Request(_post_scope(), _receive_chunks(chunks))
with pytest.raises(PayloadTooLarge):
await request.post_body(max_bytes=100)
@pytest.mark.asyncio
async def test_request_post_body_limit_from_constructor():
request = Request(
_post_scope(), _receive_chunks([b"too much data"]), max_post_body_bytes=5
)
with pytest.raises(PayloadTooLarge):
await request.post_body()
@pytest.mark.asyncio
async def test_request_post_body_limit_disabled():
body = b"a" * (3 * 1024 * 1024)
request = Request(_post_scope(), _receive_chunks([body]), max_post_body_bytes=0)
assert await request.post_body() == body
@pytest.mark.asyncio
async def test_request_post_body_default_limit():
# Bodies over 2MB are rejected by default
request = Request(_post_scope(), _receive_chunks([b"a" * (2 * 1024 * 1024 + 1)]))
with pytest.raises(PayloadTooLarge):
await request.post_body()
@pytest.mark.asyncio
async def test_request_json_too_large():
body = json.dumps({"rows": ["x" * 100]}).encode("utf-8")
request = Request(_post_scope(), _receive_chunks([body]), max_post_body_bytes=50)
with pytest.raises(PayloadTooLarge):
await request.json()
def test_request_args():
request = Request.fake("/foo?multi=1&multi=2&single=3")
assert "1" == request.args.get("multi")

718
tests/test_playwright.py
View file

@ -1,3 +1,4 @@
import base64
import json
import socket
import subprocess
@ -10,6 +11,10 @@ import pytest
from datasette.fixtures import write_fixture_database
from datasette.utils.sqlite import sqlite3
PNG_1X1_BYTES = base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII="
)
def find_free_port():
with socket.socket() as sock:
@ -17,7 +22,7 @@ def find_free_port():
return sock.getsockname()[1]
def wait_for_server(process, url, timeout=10):
def wait_for_server(process, url, timeout=30):
deadline = time.monotonic() + timeout
last_error = None
while time.monotonic() < deadline:
@ -36,7 +41,20 @@ def wait_for_server(process, url, timeout=10):
except httpx.HTTPError as ex:
last_error = repr(ex)
time.sleep(0.1)
raise AssertionError(f"Timed out waiting for {url}: {last_error}")
if process.poll() is None:
process.terminate()
try:
stdout, stderr = process.communicate(timeout=5)
except subprocess.TimeoutExpired:
process.kill()
stdout, stderr = process.communicate()
else:
stdout, stderr = process.communicate()
raise AssertionError(
f"Timed out waiting for {url}: {last_error}\n"
f"stdout:\n{stdout}\n"
f"stderr:\n{stderr}"
)
@pytest.fixture
@ -102,6 +120,22 @@ def write_playwright_database(db_path):
id integer primary key,
created_ms integer default (CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER))
);
create table binary_files (
id integer primary key,
name text not null,
data blob
);
create table bulk_defaults (
id integer primary key,
title text not null,
status text not null default 'todo',
score integer default 5
);
create table upsert_items (
id text primary key,
title text,
metadata text
);
insert into projects (title, metadata, logo, notes, score) values
(
'Build Datasette',
@ -110,7 +144,22 @@ def write_playwright_database(db_path):
'Initial notes',
5
);
insert into upsert_items (id, title, metadata) values
('existing', 'Existing title', '{"old": true}');
""")
conn.execute(
"insert into binary_files (name, data) values (?, ?)",
("Raw bytes", b"\x00\x01\x02\x03"),
)
conn.execute(
"insert into binary_files (name, data) values (?, ?)",
("PNG image", PNG_1X1_BYTES),
)
conn.execute(
"insert into binary_files (name, data) values (?, ?)",
("Null bytes", None),
)
conn.commit()
finally:
conn.close()
@ -123,6 +172,7 @@ def write_playwright_config(config_path):
"data": {
"permissions": {
"create-table": True,
"insert-row": True,
"set-column-type": True,
},
"tables": {
@ -145,6 +195,25 @@ def write_playwright_config(config_path):
"alter-table": True,
},
},
"binary_files": {
"label_column": "name",
"permissions": {
"insert-row": True,
"update-row": True,
"delete-row": True,
},
},
"bulk_defaults": {
"permissions": {
"insert-row": True,
},
},
"upsert_items": {
"permissions": {
"insert-row": True,
"update-row": True,
},
},
},
},
},
@ -278,6 +347,58 @@ def project_row(datasette_server, pk):
return rows[0]
def binary_file_blob(datasette_server, pk):
response = httpx.get(
f"{datasette_server}data/binary_files/{pk}.blob",
params={"_blob_column": "data"},
)
response.raise_for_status()
return response.content
def wait_for_binary_control_file(binary_control, size, name=None):
binary_control.locator(
".row-edit-binary-size", has_text=f"Binary: {size} bytes"
).wait_for()
if name:
binary_control.locator(".row-edit-binary-name", has_text=name).wait_for()
def bulk_default_rows(datasette_server, **filters):
params = {
"_shape": "objects",
**{key: str(value) for key, value in filters.items()},
}
response = httpx.get(f"{datasette_server}data/bulk_defaults.json", params=params)
response.raise_for_status()
return response.json()["rows"]
def upsert_item_rows(datasette_server, **filters):
params = {
"_shape": "objects",
**{key: str(value) for key, value in filters.items()},
}
response = httpx.get(f"{datasette_server}data/upsert_items.json", params=params)
response.raise_for_status()
return response.json()["rows"]
def upsert_item_row(datasette_server, pk):
rows = upsert_item_rows(datasette_server, id=pk)
assert len(rows) == 1
return rows[0]
def open_bulk_insert_dialog(page, url):
page.goto(url)
page.locator('button[data-table-action="insert-row"]').click()
dialog = page.locator("#row-edit-dialog")
dialog.wait_for()
dialog.locator(".row-edit-bulk-insert").click()
return dialog
def open_jump_menu(page):
page.keyboard.press("/")
page.locator("navigation-search .search-input").wait_for()
@ -381,6 +502,165 @@ def test_create_table_flow(page, datasette_server):
assert "NOT NULL DEFAULT 'Untitled'" in schema
@pytest.mark.playwright
def test_create_table_from_data_flow(page, datasette_server):
page.goto(f"{datasette_server}data")
page.locator("details.actions-menu-links summary").click()
page.locator('button[data-database-action="create-table"]').click()
dialog = page.locator("#table-create-dialog")
dialog.wait_for()
from_data = dialog.locator(".table-create-from-data")
assert from_data.inner_text() == "Create table from data"
from_data.click()
assert dialog.locator(".table-create-columns").is_hidden()
assert dialog.locator(".table-create-data").is_visible()
assert dialog.locator(".table-create-save").inner_text() == "Preview rows"
assert (
dialog.locator(".table-create-data-note").inner_text()
== "Paste TSV, CSV, or JSON. You can also open a file or drop it onto this textarea"
)
assert dialog.locator(".table-create-data-open-file").inner_text() == "open a file"
assert (
dialog.locator(".table-create-data-editor").evaluate(
"""node => Array.from(node.children)
.filter((child) => !child.hidden)
.map((child) => child.className)
.join(" ")"""
)
== ("table-create-data-note table-create-input table-create-data-textarea")
)
assert dialog.locator(".table-create-manual").inner_text() == (
"Create table manually"
)
assert (
dialog.get_by_label("Paste TSV, CSV, or JSON").get_attribute("id")
== "table-create-data-textarea"
)
textarea = dialog.locator(".table-create-data-textarea")
dropped_value = textarea.evaluate("""node => new Promise((resolve) => {
node.addEventListener("input", () => resolve(node.value), { once: true });
const file = new File(["short_id,name\\nx,Ada"], "Repo Export 2026!!.CSV", { type: "text/csv" });
const dataTransfer = new DataTransfer();
dataTransfer.items.add(file);
node.dispatchEvent(new DragEvent("dragenter", { bubbles: true, cancelable: true, dataTransfer }));
node.dispatchEvent(new DragEvent("dragover", { bubbles: true, cancelable: true, dataTransfer }));
node.dispatchEvent(new DragEvent("drop", { bubbles: true, cancelable: true, dataTransfer }));
})""")
assert dropped_value == "short_id,name\nx,Ada"
assert dialog.locator(".table-create-table-name").input_value() == (
"repo_export_2026"
)
dialog.locator(".table-create-table-name").fill("playwright_from_data")
textarea.fill(
json.dumps(
{
"metadata": [{"short_id": "a", "name": "Ignored", "score": 9}],
"people": [
{"short_id": "b", "name": "Ada", "score": 1},
{"short_id": "c", "name": "Bob", "score": 2.5},
],
}
)
)
dialog.locator(".table-create-save").click()
assert dialog.locator(".table-create-data-textarea").is_hidden()
assert dialog.locator(".table-create-save").inner_text() == "Create table"
assert dialog.locator(".table-create-cancel").inner_text() == "Back"
assert dialog.locator(".table-create-data-preview-summary").inner_text() == (
"Previewing 2 rows."
)
assert dialog.locator(".table-create-data-primary-key").input_value() == "short_id"
preview_text = dialog.locator(".table-create-data-preview-table").inner_text()
assert "short_id" in preview_text
assert "Ada" in preview_text
assert "2.5" in preview_text
preview_cell_style = dialog.locator(
".table-create-data-preview-table td"
).first.evaluate(
"""node => ({
overflowWrap: getComputedStyle(node).overflowWrap,
whiteSpace: getComputedStyle(node).whiteSpace
})"""
)
assert preview_cell_style == {
"overflowWrap": "anywhere",
"whiteSpace": "normal",
}
dialog.locator(".table-create-cancel").click()
assert dialog.evaluate("node => node.open")
assert dialog.locator(".table-create-data-textarea").is_visible()
assert '"people"' in dialog.locator(".table-create-data-textarea").input_value()
assert dialog.locator(".table-create-save").inner_text() == "Preview rows"
dialog.locator(".table-create-save").click()
assert dialog.locator(".table-create-save").inner_text() == "Create table"
dialog.locator(".table-create-save").click()
page.wait_for_url("**/data/playwright_from_data")
response = httpx.get(
f"{datasette_server}data/playwright_from_data.json?_shape=objects"
)
response.raise_for_status()
data = response.json()
assert data["rows"] == [
{"short_id": "b", "name": "Ada", "score": 1},
{"short_id": "c", "name": "Bob", "score": 2.5},
]
@pytest.mark.playwright
def test_create_table_from_csv_keeps_numeric_type_when_values_are_blank(
page, datasette_server
):
page.goto(f"{datasette_server}data")
page.locator("details.actions-menu-links summary").click()
page.locator('button[data-database-action="create-table"]').click()
dialog = page.locator("#table-create-dialog")
dialog.wait_for()
dialog.locator(".table-create-from-data").click()
dialog.locator(".table-create-table-name").fill("playwright_numeric_blanks")
dialog.locator(".table-create-data-textarea").fill("name,score\nA,1\nB,")
dialog.locator(".table-create-save").click()
assert dialog.locator(".table-create-save").inner_text() == "Create table"
preview_text = dialog.locator(".table-create-data-preview-table").inner_text()
assert "A" in preview_text
assert "1" in preview_text
assert "B" in preview_text
assert "null" in preview_text
dialog.locator(".table-create-save").click()
page.wait_for_url("**/data/playwright_numeric_blanks")
response = httpx.get(
f"{datasette_server}data/playwright_numeric_blanks.json?_shape=objects"
)
response.raise_for_status()
assert response.json()["rows"] == [
{"name": "A", "score": 1},
{"name": "B", "score": None},
]
schema_response = httpx.get(
f"{datasette_server}data/-/query.json",
params={
"sql": (
"select type from pragma_table_info('playwright_numeric_blanks') "
"where name = 'score'"
)
},
)
schema_response.raise_for_status()
assert schema_response.json()["rows"] == [{"type": "INTEGER"}]
@pytest.mark.playwright
def test_create_table_foreign_key_selection_updates_column_type(page, datasette_server):
page.goto(f"{datasette_server}data")
@ -801,11 +1081,128 @@ def test_navigation_search_renders_jump_sections_from_javascript_plugins(
@pytest.mark.playwright
def test_insert_row_flow_uses_custom_column_field(page, datasette_server):
page.add_init_script("""
(() => {
let clipboardText = "";
Object.defineProperty(navigator, "clipboard", {
configurable: true,
get: () => ({
writeText: async (text) => {
clipboardText = String(text);
},
readText: async () => clipboardText,
}),
});
})();
""")
page.goto(f"{datasette_server}data/projects")
page.locator('button[data-table-action="insert-row"]').click()
dialog = page.locator("#row-edit-dialog")
dialog.wait_for()
bulk_insert = dialog.locator(".row-edit-bulk-insert")
bulk_insert.wait_for()
assert bulk_insert.inner_text() == "Insert multiple rows"
current_url = page.url
bulk_insert.click()
assert page.url == current_url
assert dialog.evaluate("node => node.open")
assert dialog.locator(".row-edit-fields").is_hidden()
assert dialog.locator(".row-edit-bulk").is_visible()
assert dialog.locator(".row-edit-save").inner_text() == "Preview rows"
assert (
dialog.locator(".row-edit-bulk-note").inner_text()
== "Paste TSV, CSV, or JSON. You can also open a file or drop it onto this textarea"
)
assert dialog.locator(".row-edit-bulk-open-file").inner_text() == "open a file"
assert (
dialog.locator(".row-edit-bulk-editor").evaluate(
"""node => Array.from(node.children)
.filter((child) => !child.hidden)
.map((child) => child.className)
.join(" ")"""
)
== (
"row-edit-bulk-note row-edit-input row-edit-bulk-textarea "
"row-edit-bulk-actions"
)
)
copy_template = dialog.locator(".row-edit-copy-template")
assert copy_template.inner_text() == "Copy spreadsheet template"
assert (
dialog.get_by_label("Paste TSV, CSV, or JSON").get_attribute("id")
== "row-edit-bulk-textarea"
)
assert dialog.locator(".row-edit-copy-template-label-narrow").text_content() == (
"Copy template"
)
assert dialog.locator(".row-edit-bulk-template-note").inner_text() == (
"You can paste the template into Google Sheets or Excel."
)
assert dialog.locator(".row-edit-bulk-template-note-narrow").text_content() == (
"Paste into Google Sheets or Excel"
)
copy_template.click()
page.wait_for_function(
"""() => document.querySelector(".row-edit-copy-template").textContent === "Copied" """
)
assert page.evaluate("navigator.clipboard.readText()") == (
"title\tmetadata\tlogo\tnotes\tscore"
)
textarea = dialog.locator(".row-edit-bulk-textarea")
textarea.fill("title\tmetadata\nFrom TSV\t{}")
assert textarea.input_value() == "title\tmetadata\nFrom TSV\t{}"
dropped_value = textarea.evaluate("""node => new Promise((resolve) => {
node.addEventListener("input", () => resolve(node.value), { once: true });
const file = new File(["\\n\\ntitle,metadata\\nFrom CSV,{}\\n,\\n , \\n"], "rows.csv", { type: "text/csv" });
const dataTransfer = new DataTransfer();
dataTransfer.items.add(file);
node.dispatchEvent(new DragEvent("dragenter", { bubbles: true, cancelable: true, dataTransfer }));
node.dispatchEvent(new DragEvent("dragover", { bubbles: true, cancelable: true, dataTransfer }));
node.dispatchEvent(new DragEvent("drop", { bubbles: true, cancelable: true, dataTransfer }));
})""")
assert dropped_value == "\n\ntitle,metadata\nFrom CSV,{}\n,\n , \n"
dialog.locator(".row-edit-save").click()
assert dialog.evaluate("node => node.open")
assert dialog.locator(".row-edit-bulk-textarea").is_hidden()
assert dialog.locator(".row-edit-save").inner_text() == "Insert these rows"
assert dialog.locator(".row-edit-bulk-preview-summary").inner_text() == (
"Previewing 1 row."
)
preview_text = dialog.locator(".row-edit-bulk-preview-table").inner_text()
assert "id" in preview_text
assert "title" in preview_text
assert "metadata" in preview_text
assert "From CSV" in preview_text
assert dialog.locator(".row-edit-bulk-preview-auto").first.text_content() == "auto"
assert "null" not in preview_text
assert "undefined" not in preview_text
preview_cell_style = dialog.locator(
".row-edit-bulk-preview-table td"
).first.evaluate(
"""node => ({
overflowWrap: getComputedStyle(node).overflowWrap,
whiteSpace: getComputedStyle(node).whiteSpace
})"""
)
assert preview_cell_style == {
"overflowWrap": "anywhere",
"whiteSpace": "normal",
}
assert dialog.locator(".row-edit-cancel").inner_text() == "Back"
dialog.locator(".row-edit-cancel").click()
assert dialog.evaluate("node => node.open")
assert dialog.locator(".row-edit-bulk-textarea").is_visible()
assert dialog.locator(".row-edit-bulk-textarea").input_value() == (
"\n\ntitle,metadata\nFrom CSV,{}\n,\n , \n"
)
assert dialog.locator(".row-edit-save").inner_text() == "Preview rows"
single_insert = dialog.locator(".row-edit-single-insert")
assert single_insert.inner_text() == "Insert single row"
single_insert.click()
assert dialog.locator(".row-edit-bulk").is_hidden()
assert dialog.locator(".row-edit-fields").is_visible()
assert dialog.locator(".row-edit-save").inner_text() == "Insert row"
dialog.locator('input[name="title"]').fill("Launch Datasette Cloud")
dialog.locator('textarea[name="metadata"]').fill(
'{"ok": false, "source": "playwright"}'
@ -835,6 +1232,206 @@ def test_insert_row_flow_uses_custom_column_field(page, datasette_server):
assert data["score"] == 5
@pytest.mark.playwright
def test_bulk_insert_preview_inserts_rows(page, datasette_server):
page.goto(f"{datasette_server}data/projects")
page.locator('button[data-table-action="insert-row"]').click()
dialog = page.locator("#row-edit-dialog")
dialog.wait_for()
dialog.locator(".row-edit-bulk-insert").click()
dialog.locator(".row-edit-bulk-textarea").fill(
json.dumps(
{
"metadata": [{"title": "Ignored", "metadata": "{}"}],
"projects": [
{"title": "Bulk one", "metadata": "{}"},
{"title": "Bulk two", "metadata": "{}"},
],
}
)
)
dialog.locator(".row-edit-save").click()
assert dialog.locator(".row-edit-save").inner_text() == "Insert these rows"
dialog.locator(".row-edit-save").click()
dialog.locator(
".row-edit-bulk-progress-status", has_text="2 rows inserted."
).wait_for()
assert dialog.locator(".row-edit-cancel").inner_text() == "Close and view table"
dialog.locator(".row-edit-cancel").click()
page.wait_for_load_state("domcontentloaded")
assert page.url == f"{datasette_server}data/projects"
assert project_rows(datasette_server, title="Bulk one")
assert project_rows(datasette_server, title="Bulk two")
@pytest.mark.playwright
def test_bulk_insert_upsert_option_updates_existing_and_inserts_new(
page, datasette_server
):
dialog = open_bulk_insert_dialog(page, f"{datasette_server}data/upsert_items")
textarea = dialog.locator(".row-edit-bulk-textarea")
conflict_field = dialog.locator(".row-edit-bulk-conflict")
conflict_select = dialog.locator(".row-edit-bulk-conflict-mode")
assert conflict_field.is_hidden()
textarea.fill("title\nNo primary key")
assert conflict_field.is_hidden()
textarea.fill(
"id,title,metadata\nexisting,Updated by upsert,{}\nnew,Inserted by upsert,{}"
)
assert conflict_field.is_visible()
assert (
dialog.locator(".row-edit-bulk-conflict-label").inner_text()
== "If the row exists already"
)
assert conflict_select.input_value() == "ignore"
assert (
conflict_select.evaluate("""node => Array.from(node.options)
.filter((option) => !option.hidden)
.map((option) => option.textContent.trim())""")
== [
"Stop with an error",
"Skip existing rows",
"Update existing and insert new",
]
)
conflict_select.select_option("upsert")
assert conflict_select.input_value() == "upsert"
dialog.locator(".row-edit-save").click()
assert dialog.locator(".row-edit-save").inner_text() == "Update or insert rows"
preview_text = dialog.locator(".row-edit-bulk-preview-table").inner_text()
assert "Updated by upsert" in preview_text
assert "Inserted by upsert" in preview_text
dialog.locator(".row-edit-save").click()
dialog.locator(
".row-edit-bulk-progress-status", has_text="2 rows upserted."
).wait_for()
assert upsert_item_row(datasette_server, "existing")["title"] == (
"Updated by upsert"
)
assert upsert_item_row(datasette_server, "new")["title"] == "Inserted by upsert"
@pytest.mark.playwright
def test_bulk_insert_conflicts_hide_upsert_without_update_permission(
page, datasette_server
):
dialog = open_bulk_insert_dialog(page, f"{datasette_server}data/bulk_defaults")
textarea = dialog.locator(".row-edit-bulk-textarea")
conflict_field = dialog.locator(".row-edit-bulk-conflict")
conflict_select = dialog.locator(".row-edit-bulk-conflict-mode")
textarea.fill("title\nOnly title")
assert conflict_field.is_hidden()
textarea.fill("id,title\n1,Only title")
assert conflict_field.is_visible()
assert conflict_select.input_value() == "ignore"
assert (
conflict_select.evaluate("""node => Array.from(node.options)
.filter((option) => !option.hidden)
.map((option) => option.textContent.trim())""")
== [
"Stop with an error",
"Skip existing rows",
]
)
assert conflict_select.locator('option[value="upsert"]').evaluate(
"node => node.hidden && node.disabled"
)
@pytest.mark.playwright
def test_bulk_insert_live_validation_reports_unknown_columns(page, datasette_server):
dialog = open_bulk_insert_dialog(page, f"{datasette_server}data/projects")
textarea = dialog.locator(".row-edit-bulk-textarea")
error = dialog.locator(".row-edit-error")
save = dialog.locator(".row-edit-save")
textarea.fill(json.dumps([{"id2": 1, "title": "Unknown column"}]))
error.wait_for()
assert error.inner_text() == "JSON row 1 has unknown column id2."
assert save.is_disabled()
assert textarea.evaluate("node => document.activeElement === node")
textarea.fill("id2,title\n1,Unknown column")
assert error.inner_text() == "Unknown column id2 in header row."
assert save.is_disabled()
textarea.fill("[")
assert error.is_hidden()
assert not save.is_disabled()
textarea.fill(json.dumps([{"id": 1, "title": "Known column"}]))
assert error.is_hidden()
assert not save.is_disabled()
assert dialog.locator(".row-edit-bulk-conflict").is_visible()
assert dialog.locator(".row-edit-bulk-conflict-mode").input_value() == "ignore"
@pytest.mark.playwright
def test_bulk_insert_omits_columns_absent_from_pasted_input(page, datasette_server):
page.goto(f"{datasette_server}data/bulk_defaults")
page.locator('button[data-table-action="insert-row"]').click()
dialog = page.locator("#row-edit-dialog")
dialog.wait_for()
dialog.locator(".row-edit-bulk-insert").click()
dialog.locator(".row-edit-bulk-textarea").fill("title\nOnly title")
dialog.locator(".row-edit-save").click()
assert dialog.locator(".row-edit-save").inner_text() == "Insert these rows"
preview_text = dialog.locator(".row-edit-bulk-preview-table").inner_text()
assert "Only title" in preview_text
assert "undefined" not in preview_text
assert dialog.locator(".row-edit-bulk-preview-auto").inner_text() == "auto"
dialog.locator(".row-edit-save").click()
dialog.locator(
".row-edit-bulk-progress-status", has_text="1 row inserted."
).wait_for()
rows = bulk_default_rows(datasette_server, title="Only title")
assert rows == [
{
"id": 1,
"title": "Only title",
"status": "todo",
"score": 5,
}
]
@pytest.mark.playwright
def test_bulk_insert_preview_accepts_single_column_input(page, datasette_server):
page.goto(f"{datasette_server}data/projects")
page.locator('button[data-table-action="insert-row"]').click()
dialog = page.locator("#row-edit-dialog")
dialog.wait_for()
dialog.locator(".row-edit-bulk-insert").click()
dialog.locator(".row-edit-bulk-textarea").fill("title\none\ntwo\nthree")
dialog.locator(".row-edit-save").click()
assert dialog.locator(".row-edit-save").inner_text() == "Insert these rows"
assert dialog.locator(".row-edit-bulk-preview-summary").inner_text() == (
"Previewing 3 rows."
)
preview_text = dialog.locator(".row-edit-bulk-preview-table").inner_text()
assert "one" in preview_text
assert "two" in preview_text
assert "three" in preview_text
assert "null" not in preview_text
assert "undefined" not in preview_text
@pytest.mark.playwright
def test_edit_row_flow_validates_json_and_saves_changes(page, datasette_server):
page.goto(f"{datasette_server}data/projects")
@ -842,6 +1439,9 @@ def test_edit_row_flow_validates_json_and_saves_changes(page, datasette_server):
dialog = page.locator("#row-edit-dialog")
dialog.wait_for()
assert dialog.locator(".row-edit-bulk-insert").is_hidden()
assert dialog.locator(".row-edit-single-insert").is_hidden()
assert dialog.locator(".row-edit-bulk").is_hidden()
title = dialog.locator('input[name="title"]')
title.wait_for()
title.fill("Build Datasette, edited")
@ -877,6 +1477,120 @@ def test_edit_row_flow_validates_json_and_saves_changes(page, datasette_server):
assert data["notes"] == "Edited from Playwright"
@pytest.mark.playwright
def test_edit_row_binary_control_shows_size_and_image_preview(page, datasette_server):
page.goto(f"{datasette_server}data/binary_files")
page.locator('tr[data-row="1"] button[data-row-action="edit"]').click()
dialog = page.locator("#row-edit-dialog")
dialog.wait_for()
raw_control = dialog.locator('.row-edit-binary-control[data-column="data"]')
raw_control.wait_for()
assert (
raw_control.locator(".row-edit-binary-size").inner_text() == "Binary: 4 bytes"
)
assert raw_control.locator(".row-edit-binary-preview img").count() == 0
assert dialog.locator('textarea[name="data"]').count() == 0
assert dialog.locator('input[type="hidden"][name="data"]').count() == 1
dialog.locator(".row-edit-cancel").click()
page.locator('tr[data-row="2"] button[data-row-action="edit"]').click()
image_control = dialog.locator('.row-edit-binary-control[data-column="data"]')
image_control.wait_for()
assert (
image_control.locator(".row-edit-binary-size").inner_text()
== f"Binary: {len(PNG_1X1_BYTES)} bytes"
)
image = image_control.locator(".row-edit-binary-preview img")
image.wait_for()
assert image.get_attribute("src").startswith("blob:")
@pytest.mark.playwright
def test_edit_row_binary_control_replaces_blob_from_file(page, datasette_server):
replacement = b"Replacement \x00 bytes"
page.goto(f"{datasette_server}data/binary_files")
page.locator('tr[data-row="1"] button[data-row-action="edit"]').click()
dialog = page.locator("#row-edit-dialog")
binary_control = dialog.locator('.row-edit-binary-control[data-column="data"]')
binary_control.wait_for()
binary_control.locator('input[type="file"]').set_input_files(
{
"name": "replacement.bin",
"mimeType": "application/octet-stream",
"buffer": replacement,
}
)
wait_for_binary_control_file(binary_control, len(replacement), "replacement.bin")
dialog.locator(".row-edit-save").click()
page.locator(".row-mutation-status", has_text="Updated row 1").wait_for()
assert binary_file_blob(datasette_server, 1) == replacement
@pytest.mark.playwright
def test_edit_row_binary_control_handles_null_blob(page, datasette_server):
replacement = b"From NULL"
page.goto(f"{datasette_server}data/binary_files")
page.locator('tr[data-row="3"] button[data-row-action="edit"]').click()
dialog = page.locator("#row-edit-dialog")
binary_control = dialog.locator('.row-edit-binary-control[data-column="data"]')
binary_control.wait_for()
assert (
binary_control.locator(".row-edit-binary-size").inner_text() == "No binary data"
)
binary_control.locator('input[type="file"]').set_input_files(
{
"name": "from-null.bin",
"mimeType": "application/octet-stream",
"buffer": replacement,
}
)
wait_for_binary_control_file(binary_control, len(replacement), "from-null.bin")
dialog.locator(".row-edit-save").click()
page.locator(".row-mutation-status", has_text="Updated row 3").wait_for()
assert binary_file_blob(datasette_server, 3) == replacement
@pytest.mark.playwright
def test_insert_row_binary_control_accepts_pasted_file(page, datasette_server):
pasted = b"Pasted \x00 bytes"
page.goto(f"{datasette_server}data/binary_files")
page.locator('button[data-table-action="insert-row"]').click()
dialog = page.locator("#row-edit-dialog")
dialog.wait_for()
dialog.locator('input[name="name"]').fill("Pasted bytes")
binary_control = dialog.locator('.row-edit-binary-control[data-column="data"]')
binary_control.wait_for()
binary_control.evaluate(
"""(node, bytes) => {
const transfer = new DataTransfer();
transfer.items.add(
new File([new Uint8Array(bytes)], "pasted.bin", {
type: "application/octet-stream"
})
);
const event = new Event("paste", { bubbles: true, cancelable: true });
Object.defineProperty(event, "clipboardData", { value: transfer });
node.dispatchEvent(event);
}""",
list(pasted),
)
wait_for_binary_control_file(binary_control, len(pasted), "pasted.bin")
dialog.locator(".row-edit-save").click()
page.locator(".row-mutation-status", has_text="Inserted row 4").wait_for()
assert binary_file_blob(datasette_server, 4) == pasted
@pytest.mark.playwright
def test_delete_row_flow_removes_row(page, datasette_server):
page.goto(f"{datasette_server}data/projects")

92
tests/test_queries.py
View file

@ -3221,6 +3221,74 @@ async def test_execute_write_create_table_uses_create_table_permission():
assert not await db.table_exists("should_not_exist")
@pytest.mark.asyncio
async def test_execute_write_create_view_uses_create_view_permission():
ds = Datasette(
memory=True,
default_deny=True,
config={
"permissions": {
"insert-row": {"id": "row-writer"},
"update-row": {"id": "row-writer"},
},
"databases": {
"data": {
"permissions": {
"view-database": {"id": ["creator", "row-writer"]},
"execute-write-sql": {"id": ["creator", "row-writer"]},
"create-view": {"id": "creator"},
}
}
},
},
)
db = ds.add_memory_database("execute_write_create_view", name="data")
await db.execute_write("create table dogs (id integer primary key, name text)")
await ds.invoke_startup()
analysis_response = await ds.client.get(
"/data/-/execute-write/analyze",
actor={"id": "creator"},
params={"sql": "create view dog_names as select id, name from dogs"},
)
allowed_response = await ds.client.post(
"/data/-/execute-write",
actor={"id": "creator"},
json={"sql": "create view dog_names as select id, name from dogs"},
)
row_permission_response = await ds.client.post(
"/data/-/execute-write",
actor={"id": "row-writer"},
json={"sql": "create view should_not_exist as select id from dogs"},
)
assert analysis_response.status_code == 200
analysis_data = analysis_response.json()
assert analysis_data["ok"] is True
assert analysis_data["execute_disabled"] is False
assert analysis_data["analysis_rows"] == [
{
"operation": "create",
"database": "data",
"table": "dog_names",
"required_permission": "create-view",
"source": None,
"allowed": True,
}
]
assert allowed_response.status_code == 200
assert allowed_response.json()["ok"] is True
assert allowed_response.json()["message"] == "Query executed"
assert await db.view_exists("dog_names")
assert row_permission_response.status_code == 403
assert row_permission_response.json()["errors"] == [
"Permission denied: need create-view on data"
]
assert not await db.view_exists("should_not_exist")
@pytest.mark.parametrize(
(
"database_name",
@ -3271,8 +3339,20 @@ async def test_execute_write_create_table_uses_create_table_permission():
(),
"drop-table",
),
(
"execute_write_drop_view",
"dropper",
"drop view dogs_view",
"drop view cats_view",
"Permission denied: need drop-view on data/cats_view",
(
"create view dogs_view as select * from dogs",
"create view cats_view as select * from cats",
),
"drop-view",
),
),
ids=("alter-table", "create-index", "drop-index", "drop-table"),
ids=("alter-table", "create-index", "drop-index", "drop-table", "drop-view"),
)
@pytest.mark.asyncio
async def test_execute_write_schema_operations_use_schema_permissions(
@ -3307,7 +3387,12 @@ async def test_execute_write_schema_operations_use_schema_permissions(
"drop-table": {"id": "dropper"},
"view-table": {"id": "alterer"},
}
}
},
"dogs_view": {
"permissions": {
"drop-view": {"id": "dropper"},
}
},
},
}
},
@ -3360,6 +3445,9 @@ async def test_execute_write_schema_operations_use_schema_permissions(
elif expected_state == "drop-table":
assert not await db.table_exists("dogs")
assert await db.table_exists("cats")
elif expected_state == "drop-view":
assert not await db.view_exists("dogs_view")
assert await db.view_exists("cats_view")
@pytest.mark.asyncio

103
tests/test_table_api.py
View file

@ -1338,6 +1338,109 @@ async def test_binary_data_in_json(ds_client, path, expected_json, expected_text
assert response.text == expected_text
@pytest.mark.asyncio
async def test_column_details_extra_table(ds_client):
response = await ds_client.get(
"/fixtures/binary_data.json?_size=0&_extra=column_details"
)
assert response.status_code == 200
data_detail = response.json()["column_details"]["data"]
assert data_detail["type"].lower() == "blob"
assert data_detail == {
"type": data_detail["type"],
"sqlite_type": "BLOB",
"notnull": False,
"default": None,
"is_pk": False,
"pk_position": 0,
"hidden": 0,
}
response = await ds_client.get(
"/fixtures/simple_primary_key.json?_size=0&_extra=column_details"
)
assert response.status_code == 200
column_details = response.json()["column_details"]
id_detail = column_details["id"]
assert id_detail["type"].lower() == "integer"
assert id_detail == {
"type": id_detail["type"],
"sqlite_type": "INTEGER",
"notnull": False,
"default": None,
"is_pk": True,
"pk_position": 1,
"hidden": 0,
}
content_detail = column_details["content"]
assert content_detail["type"].lower() == "text"
assert content_detail == {
"type": content_detail["type"],
"sqlite_type": "TEXT",
"notnull": False,
"default": None,
"is_pk": False,
"pk_position": 0,
"hidden": 0,
}
response = await ds_client.get(
"/fixtures/compound_three_primary_keys.json?_size=0&_extra=column_details"
)
assert response.status_code == 200
column_details = response.json()["column_details"]
assert column_details["pk1"]["is_pk"] is True
assert column_details["pk1"]["pk_position"] == 1
assert column_details["pk2"]["is_pk"] is True
assert column_details["pk2"]["pk_position"] == 2
assert column_details["pk3"]["is_pk"] is True
assert column_details["pk3"]["pk_position"] == 3
assert column_details["content"]["is_pk"] is False
assert column_details["content"]["pk_position"] == 0
def test_column_details_extra_defaults_and_notnull():
with make_app_client(extra_databases={"defaults.db": """
CREATE TABLE defaults (
i INTEGER NOT NULL DEFAULT 42,
s TEXT DEFAULT 'hello',
dt TEXT DEFAULT (datetime('now'))
);
"""}) as client:
response = client.get("/defaults/defaults.json?_size=0&_extra=column_details")
assert response.status == 200
column_details = response.json["column_details"]
assert column_details["i"]["notnull"] is True
assert column_details["i"]["default"] == "42"
assert column_details["s"]["notnull"] is False
assert column_details["s"]["default"] == "'hello'"
assert column_details["dt"]["default"] == "datetime('now')"
@pytest.mark.skipif(
sqlite_version() < (3, 31, 0),
reason="generated columns were added in SQLite 3.31.0",
)
def test_column_details_extra_generated_columns():
with make_app_client(extra_databases={"generated.db": """
CREATE TABLE generated_columns (
body TEXT,
body_length_virtual INTEGER
GENERATED ALWAYS AS (length(body)) VIRTUAL,
body_length_stored INTEGER
GENERATED ALWAYS AS (length(body)) STORED
);
"""}) as client:
response = client.get(
"/generated/generated_columns.json?_size=0&_extra=column_details"
)
assert response.status == 200
column_details = response.json["column_details"]
assert column_details["body"]["hidden"] == 0
assert column_details["body_length_virtual"]["hidden"] == 2
assert column_details["body_length_stored"]["hidden"] == 3
@pytest.mark.asyncio
@pytest.mark.parametrize(
"qs",

36
tests/test_table_html.py
View file

@ -1027,6 +1027,7 @@ async def test_database_create_table_action_button_and_data():
"databaseName": "data",
"columnTypes": ["text", "integer", "float", "blob"],
"defaultExpressions": DEFAULT_EXPRESSION_OPTIONS,
"canInsertRows": False,
},
}
assert "customColumnTypes" not in database_data_from_soup(soup)["createTable"]
@ -1050,6 +1051,40 @@ async def test_database_create_table_action_button_and_data():
ds.close()
@pytest.mark.asyncio
async def test_database_create_table_data_includes_insert_row_permission():
ds = Datasette(
[],
config={
"databases": {
"data": {
"permissions": {
"create-table": {"id": "root"},
"insert-row": {"id": "root"},
},
},
},
},
)
try:
db = ds.add_database(
Database(ds, memory_name="test_database_create_table_insert_permission"),
name="data",
)
await db.execute_write_script("""
create table items (id integer primary key, name text);
""")
response = await ds.client.get("/data", actor={"id": "root"})
assert response.status_code == 200
create_table_data = database_data_from_soup(Soup(response.text, "html.parser"))[
"createTable"
]
assert create_table_data["canInsertRows"] is True
finally:
ds.close()
@pytest.mark.asyncio
async def test_database_create_table_data_includes_custom_column_types():
ds = Datasette(
@ -1316,6 +1351,7 @@ async def test_table_insert_action_button_and_data():
assert insert_data["path"] == "/data/items/-/insert"
assert insert_data["tableName"] == "items"
assert insert_data["primaryKeys"] == ["id"]
assert insert_data["maxInsertRows"] == 100
assert [column["name"] for column in insert_data["columns"]] == [
"name",
"score",

6
tests/test_utils.py
View file

@ -132,7 +132,11 @@ def test_path_from_row_pks(row, pks, expected_path):
"""
{"CategoryID": 1, "Description": "Soft drinks", "Picture": {"$base64": true, "encoded": "FRwCx60F/g=="}}
""".strip(),
)
),
(
{"message": b"hello"},
'{"message": {"$base64": true, "encoded": "aGVsbG8="}}',
),
],
)
def test_custom_json_encoder(obj, expected):