+
+ You can paste the template into Google Sheets or Excel.Paste into Google Sheets or Excel
+
+
+
+
+
+
+
+
@@ -5056,6 +7198,27 @@ function ensureRowEditDialog(manager) {
loading: dialog.querySelector(".row-edit-loading"),
error: dialog.querySelector(".row-edit-error"),
fields: dialog.querySelector(".row-edit-fields"),
+ bulkInsertPanel: dialog.querySelector(".row-edit-bulk"),
+ bulkInsertEditor: dialog.querySelector(".row-edit-bulk-editor"),
+ bulkInsertTextarea: dialog.querySelector(".row-edit-bulk-textarea"),
+ bulkInsertPreview: dialog.querySelector(".row-edit-bulk-preview"),
+ bulkInsertProgress: dialog.querySelector(".row-edit-bulk-progress"),
+ bulkInsertProgressBar: dialog.querySelector(".row-edit-bulk-progress-bar"),
+ bulkInsertProgressStatus: dialog.querySelector(
+ ".row-edit-bulk-progress-status",
+ ),
+ bulkInsertConflictField: dialog.querySelector(".row-edit-bulk-conflict"),
+ bulkInsertConflictSelect: dialog.querySelector(
+ ".row-edit-bulk-conflict-mode",
+ ),
+ bulkInsertConflictHelp: dialog.querySelector(
+ ".row-edit-bulk-conflict-help",
+ ),
+ copyTemplateButton: dialog.querySelector(".row-edit-copy-template"),
+ bulkInsertOpenFileButton: dialog.querySelector(".row-edit-bulk-open-file"),
+ bulkInsertFileInput: dialog.querySelector(".row-edit-bulk-file-input"),
+ bulkInsertLink: dialog.querySelector(".row-edit-bulk-insert"),
+ singleInsertLink: dialog.querySelector(".row-edit-single-insert"),
cancelButton: dialog.querySelector(".row-edit-cancel"),
saveButton: dialog.querySelector(".row-edit-save"),
currentButton: null,
@@ -5063,9 +7226,25 @@ function ensureRowEditDialog(manager) {
currentRowId: null,
currentPkPath: null,
currentInsertUrl: null,
+ currentUpsertUrl: null,
currentUpdateUrl: null,
currentFragmentUrl: null,
mode: "edit",
+ insertMode: "single",
+ bulkInsertConflictMode: "ignore",
+ bulkInsertHasPrimaryKeyColumns: false,
+ bulkInsertLiveValidationError: null,
+ bulkInsertColumns: [],
+ bulkInsertTemplateColumns: [],
+ bulkInsertColumnDetails: [],
+ bulkInsertPreviewRows: null,
+ bulkInsertPreviewReady: false,
+ bulkInsertInserted: false,
+ bulkInsertInsertedCount: 0,
+ bulkInsertMaxRows: 100,
+ shouldReloadOnClose: false,
+ redirectOnCloseUrl: null,
+ copyTemplateResetTimer: null,
loadId: 0,
manager: manager,
isLoading: false,
@@ -5081,12 +7260,139 @@ function ensureRowEditDialog(manager) {
});
rowEditDialogState.cancelButton.addEventListener("click", function () {
+ if (
+ rowEditIsMultipleInsert(rowEditDialogState) &&
+ rowEditDialogState.bulkInsertPreviewReady &&
+ !rowEditDialogState.bulkInsertInserted &&
+ !rowEditDialogState.isSaving
+ ) {
+ resetBulkInsertPreview(rowEditDialogState);
+ updateRowEditDialogButtons(rowEditDialogState);
+ rowEditDialogState.bulkInsertTextarea.focus();
+ return;
+ }
if (!rowEditDialogState.isSaving) {
rowEditDialogState.shouldRestoreFocus = true;
dialog.close();
}
});
+ rowEditDialogState.bulkInsertLink.addEventListener("click", function (ev) {
+ ev.preventDefault();
+ showMultipleRowInsert(rowEditDialogState);
+ });
+
+ rowEditDialogState.singleInsertLink.addEventListener("click", function (ev) {
+ ev.preventDefault();
+ showSingleRowInsert(rowEditDialogState);
+ });
+
+ rowEditDialogState.copyTemplateButton.addEventListener(
+ "click",
+ async function () {
+ try {
+ await copyTextToClipboard(bulkInsertTemplateText(rowEditDialogState));
+ clearRowEditDialogError(rowEditDialogState);
+ setBulkInsertCopyButtonCopied(rowEditDialogState);
+ } catch (_error) {
+ showRowEditDialogError(
+ rowEditDialogState,
+ "Could not copy the spreadsheet template.",
+ );
+ }
+ },
+ );
+
+ rowEditDialogState.bulkInsertOpenFileButton.addEventListener(
+ "click",
+ function () {
+ rowEditDialogState.bulkInsertFileInput.click();
+ },
+ );
+
+ rowEditDialogState.bulkInsertFileInput.addEventListener(
+ "change",
+ async function (ev) {
+ var files = ev.target.files;
+ await loadBulkInsertTextFile(
+ rowEditDialogState,
+ files && files.length ? files[0] : null,
+ );
+ ev.target.value = "";
+ },
+ );
+
+ rowEditDialogState.bulkInsertTextarea.addEventListener(
+ "dragenter",
+ function (ev) {
+ ev.preventDefault();
+ rowEditDialogState.bulkInsertTextarea.classList.add(
+ "row-edit-bulk-drop-target",
+ );
+ },
+ );
+
+ rowEditDialogState.bulkInsertTextarea.addEventListener(
+ "dragover",
+ function (ev) {
+ ev.preventDefault();
+ rowEditDialogState.bulkInsertTextarea.classList.add(
+ "row-edit-bulk-drop-target",
+ );
+ },
+ );
+
+ rowEditDialogState.bulkInsertTextarea.addEventListener(
+ "dragleave",
+ function () {
+ rowEditDialogState.bulkInsertTextarea.classList.remove(
+ "row-edit-bulk-drop-target",
+ );
+ },
+ );
+
+ rowEditDialogState.bulkInsertTextarea.addEventListener(
+ "drop",
+ async function (ev) {
+ ev.preventDefault();
+ rowEditDialogState.bulkInsertTextarea.classList.remove(
+ "row-edit-bulk-drop-target",
+ );
+ var files = ev.dataTransfer && ev.dataTransfer.files;
+ if (!files || !files.length) {
+ return;
+ }
+ await loadBulkInsertTextFile(rowEditDialogState, files[0]);
+ },
+ );
+
+ rowEditDialogState.bulkInsertTextarea.addEventListener(
+ "dragend",
+ function () {
+ rowEditDialogState.bulkInsertTextarea.classList.remove(
+ "row-edit-bulk-drop-target",
+ );
+ },
+ );
+
+ rowEditDialogState.bulkInsertTextarea.addEventListener("input", function () {
+ resetBulkInsertPreview(rowEditDialogState);
+ syncBulkInsertTextareaValidation(rowEditDialogState);
+ updateRowEditDialogButtons(rowEditDialogState);
+ });
+
+ rowEditDialogState.bulkInsertConflictSelect.addEventListener(
+ "change",
+ function () {
+ rowEditDialogState.bulkInsertConflictMode =
+ rowEditDialogState.bulkInsertConflictSelect.value;
+ syncBulkInsertConflictUi(rowEditDialogState);
+ resetBulkInsertPreview(rowEditDialogState);
+ syncBulkInsertTextareaValidation(rowEditDialogState);
+ updateRowEditDialogButtons(rowEditDialogState);
+ },
+ );
+
dialog.addEventListener("click", function (ev) {
if (ev.target === dialog) {
closeRowEditDialogIfConfirmed(rowEditDialogState);
@@ -5108,8 +7414,17 @@ function ensureRowEditDialog(manager) {
dialog.addEventListener("close", function () {
var state = rowEditDialogState;
+ var shouldReloadOnClose = state.shouldReloadOnClose;
+ var redirectOnCloseUrl = state.redirectOnCloseUrl;
state.loadId += 1;
state.isClosePending = false;
+ state.bulkInsertLiveValidationError = null;
+ state.shouldReloadOnClose = false;
+ state.redirectOnCloseUrl = null;
+ clearTimeout(state.copyTemplateResetTimer);
+ state.copyTemplateResetTimer = null;
+ setBulkInsertCopyButtonReady(state);
+ resetBulkInsertPreview(state);
clearRowEditDialogError(state);
state.hasLoaded = false;
destroyRowEditFields(state);
@@ -5122,6 +7437,13 @@ function ensureRowEditDialog(manager) {
) {
state.currentButton.focus();
}
+ if (shouldReloadOnClose) {
+ if (redirectOnCloseUrl) {
+ location.href = redirectOnCloseUrl;
+ } else {
+ location.reload();
+ }
+ }
});
return rowEditDialogState;
@@ -5144,8 +7466,10 @@ async function openRowEditDialog(button, manager) {
state.currentRowId = row.getAttribute("data-row") || "";
state.currentPkPath = rowDisplayLabel(row);
state.currentInsertUrl = null;
+ state.currentUpsertUrl = null;
state.currentUpdateUrl = rowUpdateUrl(row);
state.currentFragmentUrl = rowFragmentUrl(row);
+ state.insertMode = "single";
if (state.currentUpdateUrl) {
state.form.action = new URL(
state.currentUpdateUrl,
@@ -5171,6 +7495,7 @@ async function openRowEditDialog(button, manager) {
);
state.summary.hidden = true;
state.summary.textContent = "";
+ syncRowEditInsertModeUi(state);
if (!state.dialog.open) {
state.dialog.showModal();
@@ -5219,8 +7544,16 @@ function openRowInsertDialog(button, manager) {
state.currentRowId = null;
state.currentPkPath = null;
state.currentInsertUrl = tableInsertUrl();
+ state.currentUpsertUrl = tableUpsertUrl();
state.currentUpdateUrl = null;
state.currentFragmentUrl = null;
+ state.insertMode = "single";
+ state.bulkInsertConflictMode = "ignore";
+ state.bulkInsertLiveValidationError = null;
+ state.bulkInsertTextarea.value = "";
+ state.shouldReloadOnClose = false;
+ state.redirectOnCloseUrl = null;
+ resetBulkInsertPreview(state);
state.shouldRestoreFocus = true;
state.hasLoaded = false;
state.loadId += 1;
@@ -5238,14 +7571,10 @@ function openRowInsertDialog(button, manager) {
setRowEditDialogLoading(state, false);
destroyRowEditFields(state);
state.dialog.removeAttribute("aria-describedby");
- setRowDialogTitle(
- state.title,
- insertData.tableName
- ? "Insert row into " + insertData.tableName
- : "Insert row",
- );
+ setRowInsertDialogTitle(state);
state.summary.hidden = true;
state.summary.textContent = "";
+ syncRowEditInsertModeUi(state);
if (!state.dialog.open) {
state.dialog.showModal();
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 19de31ff..64c63dde 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -1,4 +1,5 @@
import asyncio
+import binascii
from contextlib import contextmanager
import aiofiles
import click
@@ -236,10 +237,8 @@ class CustomJSONEncoder(json.JSONEncoder):
- ``sqlite3.Row`` becomes a tuple
- ``sqlite3.Cursor`` becomes a list
- If a binary blob can be decoded as UTF-8, the encoder returns it as text.
-
- If it can't (for example, images), it is encoded as an object, with the actual
- data base64-encoded, like so: ::
+ Binary blobs are encoded as an object, with the actual data base64-encoded,
+ like so: ::
{
"$base64": True,
@@ -255,17 +254,42 @@ class CustomJSONEncoder(json.JSONEncoder):
if isinstance(obj, sqlite3.Cursor):
return list(obj)
if isinstance(obj, bytes):
- # Does it encode to utf8?
- try:
- return obj.decode("utf8")
- except UnicodeDecodeError:
- return {
- "$base64": True,
- "encoded": base64.b64encode(obj).decode("latin1"),
- }
+ return {
+ "$base64": True,
+ "encoded": base64.b64encode(obj).decode("latin1"),
+ }
return json.JSONEncoder.default(self, obj)
+class WriteJsonValueError(ValueError):
+ pass
+
+
+def decode_write_json_cell(value):
+ if not isinstance(value, dict):
+ return value
+ keys = set(value.keys())
+ if keys == {"$raw"}:
+ return value["$raw"]
+ if keys == {"$base64", "encoded"} and value.get("$base64") is True:
+ encoded = value["encoded"]
+ if not isinstance(encoded, str):
+ raise WriteJsonValueError("$base64 encoded value must be a string")
+ try:
+ return base64.b64decode(encoded, validate=True)
+ except binascii.Error as ex:
+ raise WriteJsonValueError("Invalid $base64 encoded value") from ex
+ return value
+
+
+def decode_write_json_row(row):
+ return {key: decode_write_json_cell(value) for key, value in row.items()}
+
+
+def decode_write_json_rows(rows):
+ return [decode_write_json_row(row) for row in rows]
+
+
@contextmanager
def sqlite_timelimit(conn, ms):
deadline = time.perf_counter() + (ms / 1000)
diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py
index e1631b10..7777308f 100644
--- a/datasette/utils/asgi.py
+++ b/datasette/utils/asgi.py
@@ -67,13 +67,25 @@ class BadRequest(Base400):
status = 400
+class PayloadTooLarge(Base400):
+ status = 413
+
+
SAMESITE_VALUES = ("strict", "lax", "none")
+# Bodies read fully into memory (post_body/post_vars/json) are capped at this
+# size unless the max_post_body_bytes setting says otherwise. Kept deliberately
+# far below multipart's DEFAULT_MAX_REQUEST_SIZE: that parser streams to disk,
+# while these bodies are held in RAM and json.loads() can multiply their
+# footprint several times over.
+DEFAULT_MAX_POST_BODY_BYTES = 2 * 1024 * 1024 # 2MB
+
class Request:
- def __init__(self, scope, receive):
+ def __init__(self, scope, receive, max_post_body_bytes=DEFAULT_MAX_POST_BODY_BYTES):
self.scope = scope
self.receive = receive
+ self.max_post_body_bytes = max_post_body_bytes
def __repr__(self):
return ''.format(self.method, self.url)
@@ -141,15 +153,43 @@ class Request:
def actor(self):
return self.scope.get("actor", None)
- async def post_body(self):
- body = b""
+ async def post_body(self, max_bytes=None):
+ """
+ Read the request body fully into memory.
+
+ The body is capped at max_bytes - or self.max_post_body_bytes
+ (default 2MB, set from the max_post_body_bytes setting for requests
+ created by Datasette) if max_bytes is not provided. Pass max_bytes=0
+ to disable the limit. Raises PayloadTooLarge (HTTP 413) if exceeded -
+ oversized bodies are rejected as soon as the limit is passed, without
+ buffering the rest.
+ """
+ if max_bytes is None:
+ max_bytes = self.max_post_body_bytes
+ too_large = PayloadTooLarge(
+ "Request body exceeded maximum size of {} bytes".format(max_bytes)
+ )
+ if max_bytes:
+ # Reject early if the client declares an oversized body
+ try:
+ if int(self.headers.get("content-length", "")) > max_bytes:
+ raise too_large
+ except ValueError:
+ # Missing or malformed - the streaming check below still applies
+ pass
+ chunks = []
+ received = 0
more_body = True
while more_body:
message = await self.receive()
assert message["type"] == "http.request", message
- body += message.get("body", b"")
+ chunk = message.get("body", b"")
+ received += len(chunk)
+ if max_bytes and received > max_bytes:
+ raise too_large
+ chunks.append(chunk)
more_body = message.get("more_body", False)
- return body
+ return b"".join(chunks)
async def post_vars(self):
body = await self.post_body()
diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py
index 3a509bd2..1be28982 100644
--- a/datasette/utils/sql_analysis.py
+++ b/datasette/utils/sql_analysis.py
@@ -488,17 +488,17 @@ def analyze_sql_tables(
and key.operation in {"create", "alter", "drop"}
for key in operations
)
- dropped_tables = {
+ dropped_tables_and_views = {
(key.database, key.table)
for key in operations
- if key.operation == "drop" and key.target_type == "table"
+ if key.operation == "drop" and key.target_type in {"table", "view"}
}
def key_is_drop_table_delete(key: OperationKey) -> bool:
return (
key.operation == "delete"
and key.target_type == "table"
- and (key.database, key.table) in dropped_tables
+ and (key.database, key.table) in dropped_tables_and_views
)
has_user_table_access_in_schema_operation = any(
diff --git a/datasette/views/database.py b/datasette/views/database.py
index b7131a05..c9dcfa89 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -326,7 +326,7 @@ class DatabaseContext(Context):
database_color: str = field(metadata={"help": "The color assigned to the database"})
database_page_data: dict = field(
metadata={
- "help": 'JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions`` and optional ``customColumnTypes``.'
+ "help": 'JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions``, ``canInsertRows`` and optional ``customColumnTypes``.'
}
)
database_actions: callable = field(
diff --git a/datasette/views/row.py b/datasette/views/row.py
index 296a0ac1..2a103180 100644
--- a/datasette/views/row.py
+++ b/datasette/views/row.py
@@ -8,7 +8,7 @@ from dataclasses import dataclass, field
import markupsafe
import sqlite_utils
-from datasette.utils.asgi import NotFound, Forbidden, Response
+from datasette.utils.asgi import NotFound, Forbidden, PayloadTooLarge, Response
from datasette.database import QueryInterrupted
from datasette.events import UpdateRowEvent, DeleteRowEvent
from datasette.resources import TableResource
@@ -17,7 +17,9 @@ from datasette.utils import (
add_cors_headers,
await_me_maybe,
call_with_supported_arguments,
+ CustomJSONEncoder,
CustomRow,
+ decode_write_json_row,
InvalidSql,
make_slot_function,
path_from_row_pks,
@@ -26,6 +28,7 @@ from datasette.utils import (
to_css_class,
escape_sqlite,
sqlite3,
+ WriteJsonValueError,
)
from datasette.plugins import pm
from datasette.extras import extra_names_from_request, ExtraScope
@@ -785,6 +788,8 @@ class RowUpdateView(BaseView):
data = await request.json()
except json.JSONDecodeError as e:
return _error(["Invalid JSON: {}".format(e)])
+ except PayloadTooLarge as e:
+ return _error([str(e)], 413)
if not isinstance(data, dict):
return _error(["JSON must be a dictionary"])
@@ -796,6 +801,10 @@ class RowUpdateView(BaseView):
return _error(["Invalid keys: {}".format(", ".join(invalid_keys))])
update = data["update"]
+ try:
+ update = decode_write_json_row(update)
+ except WriteJsonValueError as e:
+ return _error([str(e)], 400)
# Validate column types
from datasette.views.table import _validate_column_types
@@ -857,4 +866,4 @@ class RowUpdateView(BaseView):
self.ds.INFO,
)
- return Response.json(result, status=200)
+ return Response.json(result, status=200, default=CustomJSONEncoder().default)
diff --git a/datasette/views/table.py b/datasette/views/table.py
index 69d5a075..0a9c88d4 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -22,9 +22,11 @@ from datasette.utils import (
add_cors_headers,
await_me_maybe,
call_with_supported_arguments,
+ CustomJSONEncoder,
CustomRow,
append_querystring,
compound_keys_after_sql,
+ decode_write_json_rows,
format_bytes,
make_slot_function,
tilde_encode,
@@ -41,9 +43,17 @@ from datasette.utils import (
urlsafe_components,
value_as_boolean,
InvalidSql,
+ WriteJsonValueError,
sqlite3,
)
-from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Request, Response
+from datasette.utils.asgi import (
+ BadRequest,
+ Forbidden,
+ NotFound,
+ PayloadTooLarge,
+ Request,
+ Response,
+)
from datasette.filters import Filters
import sqlite_utils
from dataclasses import dataclass, field
@@ -206,7 +216,7 @@ class TableContext(Context):
)
table_insert_ui: dict = field(
metadata={
- "help": "Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns`` and ``primaryKeys`` keys; each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``value_kind`` and ``column_type`` keys."
+ "help": "Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns``, ``bulkColumns``, ``primaryKeys`` and ``maxInsertRows`` keys, plus optional ``upsertPath`` if the current actor has permission to update rows. ``columns`` lists columns for the single-row insert form, while ``bulkColumns`` lists columns for the bulk insert form. Each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``is_auto_pk``, ``value_kind`` and ``column_type`` keys."
}
)
table_alter_ui: dict = field(
@@ -481,8 +491,15 @@ async def _table_insert_ui(
):
return None
+ can_update = await datasette.allowed(
+ action="update-row",
+ resource=TableResource(database=database_name, table=table_name),
+ actor=request.actor,
+ )
+
column_types_map = await datasette.get_column_types(database_name, table_name)
columns = []
+ bulk_columns = []
column_details = await db.table_column_details(table_name)
for column in column_details:
if column.hidden:
@@ -493,32 +510,40 @@ async def _table_insert_ui(
and len(pks) == 1
and SQLiteType.from_declared_type(column.type) == SQLiteType.INTEGER
)
+ column_type = column_types_map.get(column.name)
+ column_data = {
+ "name": column.name,
+ "sqlite_type": _column_sqlite_type_for_insert_form(column),
+ "notnull": column.notnull,
+ "default": column.default_value,
+ "has_default": column.default_value is not None,
+ "is_pk": is_pk,
+ "is_auto_pk": is_auto_pk,
+ "value_kind": _column_value_kind_for_insert_form(column),
+ "column_type": (
+ {"type": column_type.name, "config": column_type.config}
+ if column_type is not None
+ else None
+ ),
+ }
+ bulk_columns.append(column_data)
if is_auto_pk:
continue
- column_type = column_types_map.get(column.name)
- columns.append(
- {
- "name": column.name,
- "sqlite_type": _column_sqlite_type_for_insert_form(column),
- "notnull": column.notnull,
- "default": column.default_value,
- "has_default": column.default_value is not None,
- "is_pk": is_pk,
- "value_kind": _column_value_kind_for_insert_form(column),
- "column_type": (
- {"type": column_type.name, "config": column_type.config}
- if column_type is not None
- else None
- ),
- }
- )
+ columns.append(column_data)
- return {
+ data = {
"path": "{}/-/insert".format(datasette.urls.table(database_name, table_name)),
"tableName": table_name,
"columns": columns,
+ "bulkColumns": bulk_columns,
"primaryKeys": pks,
+ "maxInsertRows": datasette.setting("max_insert_rows"),
}
+ if can_update:
+ data["upsertPath"] = "{}/-/upsert".format(
+ datasette.urls.table(database_name, table_name)
+ )
+ return data
async def _table_alter_ui(
@@ -1053,11 +1078,18 @@ class TableInsertView(BaseView):
pks = await db.primary_keys(table_name)
- rows, errors, extras = await self._validate_data(
- request, db, table_name, pks, upsert
- )
+ try:
+ rows, errors, extras = await self._validate_data(
+ request, db, table_name, pks, upsert
+ )
+ except PayloadTooLarge as e:
+ return _error([str(e)], 413)
if errors:
return _error(errors, 400)
+ try:
+ rows = decode_write_json_rows(rows)
+ except WriteJsonValueError as e:
+ return _error([str(e)], 400)
# Validate column types
ct_errors = await _validate_column_types(
@@ -1192,7 +1224,11 @@ class TableInsertView(BaseView):
)
)
- return Response.json(result, status=200 if upsert else 201)
+ return Response.json(
+ result,
+ status=200 if upsert else 201,
+ default=CustomJSONEncoder().default,
+ )
class TableUpsertView(TableInsertView):
@@ -1232,6 +1268,8 @@ class TableSetColumnTypeView(BaseView):
data = await request.json()
except json.JSONDecodeError as e:
return _error(["Invalid JSON: {}".format(e)], 400)
+ except PayloadTooLarge as e:
+ return _error([str(e)], 413)
if not isinstance(data, dict):
return _error(["JSON must be a dictionary"], 400)
@@ -1350,6 +1388,8 @@ class TableDropView(BaseView):
confirm = data.get("confirm")
except json.JSONDecodeError:
pass
+ except PayloadTooLarge as e:
+ return _error([str(e)], 413)
if not confirm:
return Response.json(
diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py
index ffeb7f14..d0384184 100644
--- a/datasette/views/table_create_alter.py
+++ b/datasette/views/table_create_alter.py
@@ -20,11 +20,13 @@ from datasette.column_types import SQLiteType
from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent
from datasette.resources import DatabaseResource, TableResource
from datasette.utils import (
+ decode_write_json_rows,
escape_sqlite,
get_outbound_foreign_keys,
table_column_details,
+ WriteJsonValueError,
)
-from datasette.utils.asgi import NotFound, Response
+from datasette.utils.asgi import NotFound, PayloadTooLarge, Response
from datasette.utils.sqlite import sqlite_hidden_table_names
from .base import BaseView, _error
@@ -267,6 +269,11 @@ async def _create_table_ui_context(
"databaseName": database_name,
"columnTypes": CREATE_TABLE_COLUMN_TYPES,
"defaultExpressions": default_expression_options(),
+ "canInsertRows": await datasette.allowed(
+ action="insert-row",
+ resource=DatabaseResource(database=database_name),
+ actor=request.actor,
+ ),
}
can_set_column_type = await datasette.allowed(
action="set-column-type",
@@ -804,6 +811,8 @@ class TableCreateView(BaseView):
data = await request.json()
except json.JSONDecodeError as e:
return _error(["Invalid JSON: {}".format(e)])
+ except PayloadTooLarge as e:
+ return _error([str(e)], 413)
if not isinstance(data, dict):
return _error(["JSON must be an object"])
@@ -838,6 +847,10 @@ class TableCreateView(BaseView):
actor=request.actor,
):
return _error(["Permission denied: need insert-row"], 403)
+ try:
+ rows = decode_write_json_rows(rows)
+ except WriteJsonValueError as e:
+ return _error([str(e)], 400)
alter = False
if rows:
@@ -1161,6 +1174,8 @@ class TableAlterView(BaseView):
data = await request.json()
except json.JSONDecodeError as e:
return _error(["Invalid JSON: {}".format(e)], 400)
+ except PayloadTooLarge as e:
+ return _error([str(e)], 413)
if not isinstance(data, dict):
return _error(["JSON must be a dictionary"], 400)
diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py
index f8ece55b..e36e8161 100644
--- a/datasette/views/table_extras.py
+++ b/datasette/views/table_extras.py
@@ -1,6 +1,7 @@
import itertools
from dataclasses import dataclass
+from datasette.column_types import SQLiteType
from datasette.database import QueryInterrupted
from datasette.extras import Extra, ExtraExample, ExtraRegistry, ExtraScope, Provider
from datasette.plugins import pm
@@ -375,6 +376,55 @@ class PrimaryKeysExtra(Extra):
return context.pks
+def column_detail_as_json(column):
+ return {
+ "type": column.type,
+ "sqlite_type": SQLiteType.from_declared_type(column.type).value,
+ "notnull": bool(column.notnull),
+ "default": column.default_value,
+ "is_pk": bool(column.is_pk),
+ "pk_position": column.is_pk,
+ "hidden": column.hidden,
+ }
+
+
+class ColumnDetailsExtra(Extra):
+ description = (
+ "SQLite schema details for columns in this table. The dictionary maps "
+ "column names to objects describing the schema for each column."
+ )
+ docs_note = (
+ "Each object has ``type`` as the declared type string returned by "
+ 'SQLite, or ``""`` if no type was declared; ``sqlite_type`` as the '
+ "normalized SQLite affinity, one of ``TEXT``, ``INTEGER``, ``REAL``, "
+ "``BLOB`` or ``NUMERIC``; ``notnull`` as a boolean; ``default`` "
+ 'as the raw SQL default expression string, such as ``"42"``, '
+ "``\"'hello'\"`` or ``\"datetime('now')\"``, or ``null`` if there is "
+ "no default; ``is_pk`` as a boolean; ``pk_position`` as the integer "
+ "primary key position reported by SQLite, or ``0`` for columns that "
+ "are not part of the primary key; and ``hidden`` as the integer value "
+ "reported by SQLite's ``PRAGMA table_xinfo``. ``hidden`` is ``0`` for "
+ "normal columns, ``1`` for hidden virtual table columns, ``2`` for "
+ "virtual generated columns and ``3`` for stored generated columns."
+ )
+ example = ExtraExample("/fixtures/binary_data.json?_size=0&_extra=column_details")
+ examples = {
+ ExtraScope.ROW: ExtraExample(
+ "/fixtures/binary_data/1.json?_extra=column_details"
+ )
+ }
+ scopes = {ExtraScope.TABLE, ExtraScope.ROW}
+
+ async def resolve(self, context):
+ column_details = await context.datasette._get_resource_column_details(
+ context.database_name, context.table_name
+ )
+ return {
+ column_name: column_detail_as_json(column)
+ for column_name, column in column_details.items()
+ }
+
+
class ActionsExtra(Extra):
description = 'Async callable returning table or view actions made available by core and plugin hooks. Each item is either a link with ``href``, ``label`` and optional ``description`` keys, or a button with ``type: "button"``, ``label``, optional ``description`` and optional ``attrs``. See :ref:`plugin_actions`, :ref:`plugin_hook_table_actions` and :ref:`plugin_hook_view_actions`.'
scopes = {ExtraScope.TABLE}
@@ -1240,6 +1290,7 @@ TABLE_EXTRA_CLASSES = [
ColumnsExtra,
AllColumnsExtra,
PrimaryKeysExtra,
+ ColumnDetailsExtra,
DisplayColumnsAndRowsProvider,
DisplayColumnsExtra,
DisplayRowsExtra,
diff --git a/datasette/write_sql.py b/datasette/write_sql.py
index cdc0c6d3..ca144a72 100644
--- a/datasette/write_sql.py
+++ b/datasette/write_sql.py
@@ -138,6 +138,33 @@ def decision_for_write_sql_operation(
),
)
)
+ if operation.operation == "create" and operation.target_type == "view":
+ if operation.database is None:
+ return UnsupportedWriteSqlOperation(unsupported_message)
+ return RequireWriteSqlPermissions(
+ (
+ PermissionRequirement(
+ action="create-view",
+ resource=DatabaseResource(database=operation.database),
+ ),
+ )
+ )
+ if (
+ operation.operation == "drop"
+ and operation.target_type == "view"
+ and operation.database is not None
+ and operation.table is not None
+ ):
+ return RequireWriteSqlPermissions(
+ (
+ PermissionRequirement(
+ action="drop-view",
+ resource=TableResource(
+ database=operation.database, table=operation.table
+ ),
+ ),
+ )
+ )
if (
operation.operation == "alter"
and operation.target_type == "table"
diff --git a/docs/authentication.rst b/docs/authentication.rst
index 0a476c1c..51fa07d5 100644
--- a/docs/authentication.rst
+++ b/docs/authentication.rst
@@ -50,7 +50,7 @@ The one exception is the "root" account, which you can sign into while using Dat
The ``--root`` flag is designed for local development and testing. When you start Datasette with ``--root``, the root user automatically receives every permission, including:
* All view permissions (``view-instance``, ``view-database``, ``view-table``, etc.)
-* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``alter-table``, ``set-column-type``, ``drop-table``)
+* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``create-view``, ``alter-table``, ``set-column-type``, ``drop-table``, ``drop-view``)
* Debug permissions (``permissions-debug``, ``debug-menu``)
* Any custom permissions defined by plugins
@@ -1390,6 +1390,16 @@ create-table
Actor is allowed to create a database table.
+``resource`` - ``datasette.resources.DatabaseResource(database)``
+ ``database`` is the name of the database (string)
+
+.. _actions_create_view:
+
+create-view
+-----------
+
+Actor is allowed to create a database view.
+
``resource`` - ``datasette.resources.DatabaseResource(database)``
``database`` is the name of the database (string)
@@ -1429,6 +1439,18 @@ Actor is allowed to drop a database table.
``table`` is the name of the table (string)
+.. _actions_drop_view:
+
+drop-view
+---------
+
+Actor is allowed to drop a database view.
+
+``resource`` - ``datasette.resources.TableResource(database, table)``
+ ``database`` is the name of the database (string)
+
+ ``table`` is the name of the view (string)
+
.. _actions_execute_sql:
execute-sql
diff --git a/docs/binary_data.rst b/docs/binary_data.rst
index 0c890fe5..d41e36be 100644
--- a/docs/binary_data.rst
+++ b/docs/binary_data.rst
@@ -12,7 +12,12 @@ Datasette includes special handling for these binary values. The Datasette inter
:width: 311px
:alt: Screenshot showing download links next to binary data in the table view
-Binary data is represented in ``.json`` exports using Base64 encoding.
+.. _binary_json_format:
+
+Binary values in JSON
+---------------------
+
+Binary data is represented in ``.json`` exports using Base64 encoding. Datasette uses this representation for every ``BLOB`` value, including binary values that could also be decoded as UTF-8 text.
https://latest.datasette.io/fixtures/binary_data.json?_shape=array
@@ -39,6 +44,48 @@ https://latest.datasette.io/fixtures/binary_data.json?_shape=array
}
]
+The same format can be used with the :ref:`JSON write API `.
+If a column value in a ``row``, ``rows`` or ``update`` object is a JSON object with exactly ``"$base64"`` set to ``true`` and an ``"encoded"`` string, Datasette will decode that Base64 string and store the resulting bytes:
+
+.. code-block:: json
+
+ {
+ "data": {
+ "$base64": true,
+ "encoded": "FRwCx60F/g=="
+ }
+ }
+
+This works for inserts, upserts and updates. It also works when creating a table from example ``row`` or ``rows`` data: Datasette decodes the value before inferring the schema, allowing that column to be created as a ``BLOB`` column.
+
+To store a JSON object with that exact shape literally, wrap it in a ``"$raw"`` object:
+
+.. code-block:: json
+
+ {
+ "data": {
+ "$raw": {
+ "$base64": true,
+ "encoded": "FRwCx60F/g=="
+ }
+ }
+ }
+
+``"$raw"`` unwraps exactly one layer. To store a literal ``"$raw"`` object containing a Base64 object, wrap it again:
+
+.. code-block:: json
+
+ {
+ "data": {
+ "$raw": {
+ "$raw": {
+ "$base64": true,
+ "encoded": "FRwCx60F/g=="
+ }
+ }
+ }
+ }
+
.. _binary_linking:
Linking to binary downloads
diff --git a/docs/changelog.rst b/docs/changelog.rst
index ec32b57e..4541981f 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,19 @@
Changelog
=========
+.. _unreleased:
+
+Unreleased
+----------
+
+- Table pages now offer an "Insert multiple rows" mode in the row insertion dialog. This accepts pasted TSV, CSV or JSON, previews the parsed rows before inserting them, validates unknown columns as data is pasted and displays omitted auto integer primary keys as ``auto`` in the preview. (:pr:`2813`)
+- The bulk insert UI can skip rows with existing primary keys, or update existing rows and insert new rows using the existing ``//
/-/upsert`` API when the actor has both :ref:`insert-row ` and :ref:`update-row ` permissions. (:pr:`2813`)
+- The "Create table" dialog now includes a "Create table from data" mode. Paste TSV, CSV or JSON rows to preview inferred columns and types, choose the table name and primary key, then create the table and insert those rows in one step. (:pr:`2813`)
+- Datasette's JSON APIs now consistently encode every ``BLOB`` value using the documented :ref:`binary value JSON format `, even when the bytes could be decoded as UTF-8 text.
+- The insert and edit row dialogs now provide a dedicated control for ``BLOB`` values. Existing binary values are shown by byte size, image values under 10MB are previewed as thumbnails, and replacements can be attached, dropped or pasted into the control.
+- The table and row JSON APIs now support ``?_extra=column_details`` for returning SQLite schema details for columns, including declared type, SQLite affinity, primary key, ``NOT NULL``, default and hidden-column metadata.
+- POST bodies that Datasette reads fully into memory - such as JSON submitted to the write API - are now capped by the new :ref:`setting_max_post_body_bytes` setting, defaulting to 2MB. Oversized requests are rejected with an HTTP 413 error as soon as the limit is exceeded, protecting smaller servers from memory exhaustion. File uploads are unaffected - ``request.form()`` streams those to disk and has its own separate limits.
+
.. _v1_0_a35:
1.0a35 (2026-06-23)
diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst
index 7ca88c4e..2302f742 100644
--- a/docs/cli-reference.rst
+++ b/docs/cli-reference.rst
@@ -244,6 +244,9 @@ These can be passed to ``datasette serve`` using ``datasette serve --setting nam
custom query (default=1000)
max_insert_rows Maximum rows that can be inserted at a time using
the bulk insert API (default=100)
+ max_post_body_bytes Maximum size in bytes for a POST body read into
+ memory, e.g. JSON API requests - set 0 to disable
+ this limit (default=2097152)
num_sql_threads Number of threads in the thread pool for
executing SQLite queries (default=3)
sql_time_limit_ms Time limit for a SQL query in milliseconds
diff --git a/docs/internals.rst b/docs/internals.rst
index 9f75640f..e5e7aca5 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -52,6 +52,9 @@ The request object is passed to various plugin hooks. It represents an incoming
``.actor`` - dictionary (str -> Any) or None
The currently authenticated actor (see :ref:`actors `), or ``None`` if the request is unauthenticated.
+``.max_post_body_bytes`` - integer
+ The maximum number of bytes ``await request.post_body()`` will read into memory, or ``0`` for no limit. Set from the :ref:`setting_max_post_body_bytes` setting (default 2MB) for requests created by Datasette. Can be passed to the ``Request`` constructor as a keyword argument.
+
The object also has the following awaitable methods:
``await request.form(files=False, ...)`` - FormData
@@ -109,9 +112,11 @@ The object also has the following awaitable methods:
``await request.json()`` - Any
Returns the parsed JSON body of a request submitted by ``POST``.
-``await request.post_body()`` - bytes
+``await request.post_body(max_bytes=None)`` - bytes
Returns the un-parsed body of a request submitted by ``POST`` - useful for things like incoming JSON data.
+ The body is read fully into memory, capped at ``request.max_post_body_bytes`` - which Datasette sets from the :ref:`setting_max_post_body_bytes` setting (default 2MB). Bodies that exceed the limit raise a ``datasette.PayloadTooLarge`` exception, which Datasette turns into an HTTP 413 error response. Pass ``max_bytes=`` to override the limit for a specific call, or ``max_bytes=0`` to disable it. ``request.post_vars()`` and ``request.json()`` read the body through this method, so the same limit applies to them.
+
And a class method that can be used to create fake request objects for use in tests:
``fake(path_with_query_string, method="GET", scheme="http", url_vars=None)``
diff --git a/docs/json_api.rst b/docs/json_api.rst
index 4b1f26d2..32d28733 100644
--- a/docs/json_api.rst
+++ b/docs/json_api.rst
@@ -497,6 +497,25 @@ The available table extras are listed below.
"pk"
]
+``column_details``
+ SQLite schema details for columns in this table. The dictionary maps column names to objects describing the schema for each column. (Each object has ``type`` as the declared type string returned by SQLite, or ``""`` if no type was declared; ``sqlite_type`` as the normalized SQLite affinity, one of ``TEXT``, ``INTEGER``, ``REAL``, ``BLOB`` or ``NUMERIC``; ``notnull`` as a boolean; ``default`` as the raw SQL default expression string, such as ``"42"``, ``"'hello'"`` or ``"datetime('now')"``, or ``null`` if there is no default; ``is_pk`` as a boolean; ``pk_position`` as the integer primary key position reported by SQLite, or ``0`` for columns that are not part of the primary key; and ``hidden`` as the integer value reported by SQLite's ``PRAGMA table_xinfo``. ``hidden`` is ``0`` for normal columns, ``1`` for hidden virtual table columns, ``2`` for virtual generated columns and ``3`` for stored generated columns.)
+
+ ``GET /fixtures/binary_data.json?_size=0&_extra=column_details``
+
+ .. code-block:: json
+
+ {
+ "data": {
+ "type": "BLOB",
+ "sqlite_type": "BLOB",
+ "notnull": false,
+ "default": null,
+ "is_pk": false,
+ "pk_position": 0,
+ "hidden": 0
+ }
+ }
+
``display_columns``
Column metadata used by the HTML table display. Each item includes ``name``, ``sortable``, ``is_pk``, ``type``, ``notnull``, ``description``, ``column_type`` and ``column_type_config`` keys.
@@ -902,6 +921,25 @@ The following extras are available for row JSON responses.
"id"
]
+``column_details``
+ SQLite schema details for columns in this table. The dictionary maps column names to objects describing the schema for each column. (Each object has ``type`` as the declared type string returned by SQLite, or ``""`` if no type was declared; ``sqlite_type`` as the normalized SQLite affinity, one of ``TEXT``, ``INTEGER``, ``REAL``, ``BLOB`` or ``NUMERIC``; ``notnull`` as a boolean; ``default`` as the raw SQL default expression string, such as ``"42"``, ``"'hello'"`` or ``"datetime('now')"``, or ``null`` if there is no default; ``is_pk`` as a boolean; ``pk_position`` as the integer primary key position reported by SQLite, or ``0`` for columns that are not part of the primary key; and ``hidden`` as the integer value reported by SQLite's ``PRAGMA table_xinfo``. ``hidden`` is ``0`` for normal columns, ``1`` for hidden virtual table columns, ``2`` for virtual generated columns and ``3`` for stored generated columns.)
+
+ ``GET /fixtures/binary_data/1.json?_extra=column_details``
+
+ .. code-block:: json
+
+ {
+ "data": {
+ "type": "BLOB",
+ "sqlite_type": "BLOB",
+ "notnull": false,
+ "default": null,
+ "is_pk": false,
+ "pk_position": 0,
+ "hidden": 0
+ }
+ }
+
``render_cell``
Rendered HTML for each cell using the render_cell plugin hook (See the :ref:`render_cell() plugin hook ` documentation.)
@@ -1626,6 +1664,8 @@ The JSON write API
Datasette provides a write API for JSON data. This is a POST-only API that requires an authenticated API token, see :ref:`CreateTokenView`. The token will need to have the specified :ref:`authentication_permissions`.
+The row-based write APIs can write :ref:`binary values in JSON ` using Datasette's Base64 representation for BLOB data.
+
.. _ExecuteWriteView:
Executing write SQL
@@ -1756,6 +1796,8 @@ A single row can be inserted using the ``"row"`` key:
}
}
+Column values can use the :ref:`binary value JSON format ` to write BLOB data.
+
If successful, this will return a ``201`` status code and the newly inserted row, for example:
.. code-block:: json
@@ -1863,6 +1905,8 @@ An upsert is an insert or update operation. If a row with a matching primary key
The upsert API is mostly the same shape as the :ref:`insert API `. It requires both the :ref:`actions_insert_row` and :ref:`actions_update_row` permissions.
+It also accepts the same :ref:`binary value JSON format `.
+
::
POST //
/-/upsert
@@ -1995,6 +2039,8 @@ To update a row, make a ``POST`` to ``//