diff --git a/Justfile b/Justfile index 5fcd9afd..6ffff870 100644 --- a/Justfile +++ b/Justfile @@ -33,10 +33,11 @@ export DATASETTE_SECRET := "not_a_secret" uv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt uv run codespell tests --ignore-words docs/codespell-ignore-words.txt -# Run linters: black, ruff, cog +# Run linters: black, ruff, prettier, cog @lint: codespell uv run black datasette tests --check uv run ruff check datasette tests + npm run prettier -- --check uv run cog --check README.md docs/*.rst # Apply ruff fixes diff --git a/datasette/default_actions.py b/datasette/default_actions.py index 2f78570b..602e0df4 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -61,6 +61,12 @@ def register_actions(): description="Create tables", resource_class=DatabaseResource, ), + Action( + name="create-view", + abbr="cv", + description="Create views", + resource_class=DatabaseResource, + ), Action( name="store-query", abbr="sq", @@ -111,6 +117,12 @@ def register_actions(): description="Drop tables", resource_class=TableResource, ), + Action( + name="drop-view", + abbr="dv", + description="Drop views", + resource_class=TableResource, + ), # Query-level actions (child-level) Action( name="view-query", diff --git a/datasette/static/app.css b/datasette/static/app.css index 5bfd0aca..d101e4b7 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -1641,6 +1641,11 @@ dialog.row-edit-dialog::backdrop { overflow-y: auto; } +.row-edit-fields[hidden], +.row-edit-bulk[hidden] { + display: none; +} + .row-edit-field { display: grid; grid-template-columns: minmax(120px, 180px) minmax(0, 1fr); @@ -1910,6 +1915,207 @@ textarea.row-edit-input { margin: 0; } +.row-edit-bulk { + display: grid; + gap: 8px; + padding: 16px 24px 24px; + overflow-y: auto; +} + +.row-edit-bulk-editor { + display: grid; + gap: 8px; +} + +.row-edit-bulk-editor[hidden] { + display: none; +} + +.row-edit-bulk-actions { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 8px; + justify-content: flex-start; +} + +.row-edit-bulk-actions .btn { + padding-left: 12px; + padding-right: 12px; +} + +.row-edit-bulk-conflict { + display: grid; + grid-template-columns: minmax(120px, 180px) minmax(0, 1fr); + gap: 8px 12px; + align-items: start; +} + +.row-edit-bulk-conflict[hidden] { + display: none; +} + +.row-edit-bulk-conflict-label { + color: var(--ink); + font-size: 0.82rem; + padding-top: 8px; +} + +.row-edit-bulk-conflict-control { + display: grid; + gap: 4px; +} + +.row-edit-bulk-conflict-help { + color: var(--muted); + font-size: 0.78rem; + margin: 0; +} + +.row-edit-copy-template-label-narrow { + display: none; +} + +.row-edit-bulk-template-note { + color: var(--muted); + font-size: 0.82rem; +} + +.row-edit-bulk-template-note-narrow { + display: none; +} + +.row-edit-bulk-textarea { + min-height: 16rem; + resize: vertical; + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.82rem; + line-height: 1.45; +} + +.row-edit-bulk-textarea.row-edit-bulk-drop-target { + border-color: var(--accent); + background: #f8fbff; + outline: 3px solid rgba(26, 86, 219, 0.12); +} + +.row-edit-bulk-note { + color: var(--muted); + font-size: 0.82rem; + margin: 0; +} + +.row-edit-bulk-note label, +.row-edit-bulk-note .button-as-link { + font: inherit; +} + +@media (max-width: 640px) { + .row-edit-copy-template-label-wide { + display: none; + } + + .row-edit-copy-template-label-narrow { + display: inline; + } + + .row-edit-bulk-template-note-wide { + display: none; + } + + .row-edit-bulk-template-note-narrow { + display: inline; + } +} + +.row-edit-bulk-preview { + display: grid; + gap: 8px; + margin-top: 8px; +} + +.row-edit-bulk-preview[hidden] { + display: none; +} + +.row-edit-bulk-preview-summary { + color: var(--ink); + font-size: 0.9rem; + font-weight: 600; + margin: 0; +} + +.row-edit-bulk-preview-table-wrap { + border: 1px solid var(--rule); + border-radius: 5px; + max-height: 18rem; + overflow: auto; + background: #fff; +} + +.row-edit-bulk-preview-table { + border-collapse: collapse; + font-size: 0.78rem; + min-width: 100%; + width: max-content; +} + +.row-edit-bulk-preview-table th, +.row-edit-bulk-preview-table td { + border-bottom: 1px solid var(--rule); + border-right: 1px solid var(--rule); + max-width: 18rem; + overflow-wrap: anywhere; + padding: 6px 8px; + text-align: left; + vertical-align: top; + white-space: normal; +} + +.row-edit-bulk-preview-table th { + background: var(--paper); + color: var(--ink); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-weight: 600; + position: sticky; + top: 0; + z-index: 1; +} + +.row-edit-bulk-preview-table tr:last-child td { + border-bottom: none; +} + +.row-edit-bulk-preview-table th:last-child, +.row-edit-bulk-preview-table td:last-child { + border-right: none; +} + +.row-edit-bulk-preview-null, +.row-edit-bulk-preview-auto { + color: var(--muted); + font-style: italic; +} + +.row-edit-bulk-progress { + display: grid; + gap: 6px; +} + +.row-edit-bulk-progress[hidden] { + display: none; +} + +.row-edit-bulk-progress-bar { + width: 100%; +} + +.row-edit-bulk-progress-status { + color: var(--ink); + font-size: 0.9rem; + margin: 0; +} + datasette-autocomplete { display: block; position: relative; @@ -1988,6 +2194,16 @@ datasette-autocomplete input[type="text"], background: var(--paper); } +.row-edit-mode-link { + color: var(--accent); + font-size: 0.9rem; + margin-right: auto; +} + +.row-edit-mode-link[hidden] { + display: none; +} + .row-edit-dialog .btn { border: none; border-radius: 5px; @@ -2184,6 +2400,120 @@ select.table-create-input { gap: 10px; } +.table-create-columns[hidden], +.table-create-data[hidden], +.table-create-data-editor[hidden], +.table-create-data-preview[hidden] { + display: none; +} + +.table-create-data, +.table-create-data-editor { + display: grid; + gap: 8px; +} + +.table-create-data-label { + color: var(--ink); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.82rem; +} + +.table-create-data-textarea { + min-height: 16rem; + resize: vertical; + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.82rem; + line-height: 1.45; +} + +.table-create-data-textarea.table-create-data-drop-target { + border-color: var(--accent); + background: #f8fbff; + outline: 3px solid rgba(26, 86, 219, 0.12); +} + +.table-create-data-note { + color: var(--muted); + font-size: 0.82rem; + margin: 0; +} + +.table-create-data-note label, +.table-create-data-note .button-as-link { + font: inherit; +} + +.table-create-data-preview { + display: grid; + gap: 10px; +} + +.table-create-data-preview-summary { + color: var(--ink); + font-size: 0.9rem; + font-weight: 600; + margin: 0; +} + +.table-create-data-pk-field { + display: grid; + grid-template-columns: minmax(120px, 180px) minmax(0, 1fr); + gap: 12px; + align-items: center; +} + +.table-create-data-preview-table-wrap { + border: 1px solid var(--rule); + border-radius: 5px; + max-height: 18rem; + overflow: auto; + background: #fff; +} + +.table-create-data-preview-table { + border-collapse: collapse; + font-size: 0.78rem; + min-width: 100%; + width: max-content; +} + +.table-create-data-preview-table th, +.table-create-data-preview-table td { + border-bottom: 1px solid var(--rule); + border-right: 1px solid var(--rule); + max-width: 18rem; + overflow-wrap: anywhere; + padding: 6px 8px; + text-align: left; + vertical-align: top; + white-space: normal; +} + +.table-create-data-preview-table th { + background: var(--paper); + color: var(--ink); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-weight: 600; + position: sticky; + top: 0; + z-index: 1; +} + +.table-create-data-preview-table tr:last-child td { + border-bottom: none; +} + +.table-create-data-preview-table th:last-child, +.table-create-data-preview-table td:last-child { + border-right: none; +} + +.table-create-data-preview-null { + color: var(--muted); + font-style: italic; +} + .table-create-column-list { display: grid; gap: 8px; @@ -2411,6 +2741,16 @@ select.table-create-input { background: var(--paper); } +.table-create-mode-link { + color: var(--accent); + font-size: 0.9rem; + margin-right: auto; +} + +.table-create-mode-link[hidden] { + display: none; +} + .table-create-dialog .btn { border: none; border-radius: 5px; @@ -3124,7 +3464,8 @@ select.table-alter-input { .row-edit-dialog .modal-header, .row-edit-summary, .row-edit-loading, - .row-edit-fields { + .row-edit-fields, + .row-edit-bulk { padding-left: 18px; padding-right: 18px; } @@ -3143,6 +3484,15 @@ select.table-alter-input { padding-top: 0; } + .row-edit-bulk-conflict { + grid-template-columns: 1fr; + gap: 5px; + } + + .row-edit-bulk-conflict-label { + padding-top: 0; + } + .row-edit-dialog .modal-footer { padding-left: 18px; padding-right: 18px; @@ -3174,6 +3524,11 @@ select.table-alter-input { padding-top: 0; } + .table-create-data-pk-field { + grid-template-columns: 1fr; + gap: 5px; + } + .table-create-column-headings { display: none; } diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 2aa9adcf..9e8b93f6 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -5,6 +5,7 @@ var rowEditDialogState = null; var ROW_EDIT_BINARY_IMAGE_PREVIEW_MAX_BYTES = 10 * 1024 * 1024; var TABLE_CREATE_DIALOG_ID = "table-create-dialog"; var tableCreateDialogState = null; +var TABLE_CREATE_AUTOMATIC_PK = "__datasette_automatic_pk__"; var TABLE_ALTER_DIALOG_ID = "table-alter-dialog"; var tableAlterDialogState = null; @@ -811,12 +812,60 @@ async function loadTableCreateForeignKeyTargets(state) { ); } +function tableCreateIsDataMode(state) { + return state && state.mode === "data"; +} + +function tableCreateSaveButtonText(state) { + if (tableCreateIsDataMode(state)) { + return state.dataPreviewReady ? "Create table" : "Preview rows"; + } + return "Create table"; +} + +function tableCreateCanInsertRows() { + var data = databaseCreateTableData() || {}; + return !!data.canInsertRows; +} + +function syncTableCreateModeUi(state) { + if (!state) { + return; + } + var isDataMode = tableCreateIsDataMode(state); + state.columnsPanel.hidden = isDataMode; + state.dataPanel.hidden = !isDataMode; + state.dataEditor.hidden = !isDataMode || state.dataPreviewReady; + state.dataPreview.hidden = !isDataMode || !state.dataPreviewReady; + state.createFromDataLink.hidden = isDataMode || !tableCreateCanInsertRows(); + state.manualCreateLink.hidden = !isDataMode; +} + +function updateTableCreateDialogButtons(state) { + if (!state) { + return; + } + syncTableCreateModeUi(state); + state.cancelButton.disabled = state.isSaving; + state.saveButton.disabled = state.isSaving; + state.addColumnButton.disabled = state.isSaving; + state.cancelButton.textContent = + tableCreateIsDataMode(state) && state.dataPreviewReady ? "Back" : "Cancel"; + state.saveButton.textContent = state.isSaving + ? "Creating..." + : tableCreateSaveButtonText(state); +} + function tableCreateDialogSignature(state) { if (!state || !state.form) { return ""; } - return JSON.stringify({ + var signature = { table: state.tableName.value, + data: state.dataTextarea ? state.dataTextarea.value : "", + dataPrimaryKey: state.dataPkSelect + ? state.dataPkSelect.value + : TABLE_CREATE_AUTOMATIC_PK, columns: tableCreateDialogRows(state).map(function (row) { return { name: row.querySelector(".table-create-column-name").value, @@ -839,7 +888,8 @@ function tableCreateDialogSignature(state) { ).value || "", }; }), - }); + }; + return JSON.stringify(signature); } function tableCreateDialogHasChanges(state) { @@ -865,18 +915,23 @@ function showTableCreateDialogError(state, message) { function setTableCreateDialogSaving(state, isSaving) { state.isSaving = isSaving; - state.cancelButton.disabled = isSaving; - state.saveButton.disabled = isSaving; - state.addColumnButton.disabled = isSaving; - state.saveButton.textContent = isSaving ? "Creating..." : "Create table"; state.columnList .querySelectorAll("input, select, button") .forEach(function (control) { control.disabled = isSaving; }); + state.fields + .querySelectorAll( + ".table-create-data input, .table-create-data select, .table-create-data textarea, .table-create-data button", + ) + .forEach(function (control) { + control.disabled = isSaving; + }); + state.tableName.disabled = isSaving; if (!isSaving) { updateTableCreateColumnRules(state); } + updateTableCreateDialogButtons(state); updateTableCreateMoveButtons(state); } @@ -1232,8 +1287,11 @@ function addTableCreateColumn(state, column) { } function resetTableCreateDialog(state) { + state.mode = "manual"; state.nextColumnIndex = 0; state.tableName.value = ""; + state.dataTextarea.value = ""; + resetTableCreateDataPreview(state); state.columnList.textContent = ""; addTableCreateColumn(state, { name: "id", @@ -1246,9 +1304,39 @@ function resetTableCreateDialog(state) { primaryKey: false, }); updateTableCreateColumnRules(state); + updateTableCreateDialogButtons(state); state.initialSignature = tableCreateDialogSignature(state); } +function showTableCreateDataMode(state) { + if (!state || state.isSaving || !tableCreateCanInsertRows()) { + return; + } + state.mode = "data"; + clearTableCreateDialogError(state); + updateTableCreateDialogButtons(state); + if (state.dataPreviewReady && state.dataPkSelect) { + state.dataPkSelect.focus(); + } else { + state.dataTextarea.focus(); + } +} + +function showTableCreateManualMode(state) { + if (!state || state.isSaving) { + return; + } + state.mode = "manual"; + clearTableCreateDialogError(state); + updateTableCreateDialogButtons(state); + var firstInput = state.columnList.querySelector(".table-create-column-name"); + if (firstInput) { + firstInput.focus(); + } else { + state.tableName.focus(); + } +} + function collectTableCreatePayload(state) { var payload = { table: state.tableName.value.trim(), @@ -1316,14 +1404,9 @@ function collectTableCreateColumnTypeAssignments(state) { } function validateTableCreatePayload(payload) { - if (!payload.table) { - return "Table name is required."; - } - if (payload.table.indexOf("\n") !== -1) { - return "Table name cannot contain newlines."; - } - if (/^sqlite_/i.test(payload.table)) { - return "Table name cannot start with sqlite_."; + var tableNameError = validateTableCreateTableName(payload.table); + if (tableNameError) { + return tableNameError; } if (!payload.columns.length) { return "At least one column is required."; @@ -1353,6 +1436,19 @@ function validateTableCreatePayload(payload) { return null; } +function validateTableCreateTableName(tableName) { + if (!tableName) { + return "Table name is required."; + } + if (tableName.indexOf("\n") !== -1) { + return "Table name cannot contain newlines."; + } + if (/^sqlite_/i.test(tableName)) { + return "Table name cannot start with sqlite_."; + } + return null; +} + function validateTableCreateColumnTypeAssignments(assignments) { for (var i = 0; i < assignments.length; i += 1) { var assignment = assignments[i]; @@ -1375,6 +1471,476 @@ function validateTableCreateColumnTypeAssignments(assignments) { return null; } +function normalizeCreateTableDataJsonValue(value) { + if (typeof value === "undefined") { + return null; + } + if (Array.isArray(value) || (value && typeof value === "object")) { + return JSON.stringify(value); + } + return value; +} + +function createTableDataColumnObjects(names) { + return names.map(function (name) { + return { name: name }; + }); +} + +function validateCreateTableDataHeaders(headers) { + if (!headers.length) { + throw new Error("No columns found to preview."); + } + var seen = {}; + headers.forEach(function (name, index) { + if (!name) { + throw new Error("Column header " + (index + 1) + " is blank."); + } + if (name.indexOf("\n") !== -1) { + throw new Error("Column names cannot contain newlines."); + } + var key = name.toLowerCase(); + if (seen[key]) { + throw new Error("Duplicate column name: " + name); + } + seen[key] = true; + }); +} + +function jsonRowIsObject(item) { + return !!(item && typeof item === "object" && !Array.isArray(item)); +} + +function extractJsonObjectRows(parsed) { + if (Array.isArray(parsed)) { + return parsed; + } + if (!jsonRowIsObject(parsed)) { + throw new Error( + "JSON must be an array of objects, or an object containing an array of objects.", + ); + } + + var bestRows = null; + Object.keys(parsed).forEach(function (key) { + var value = parsed[key]; + if (!Array.isArray(value) || !value.every(jsonRowIsObject)) { + return; + } + if (!bestRows || value.length > bestRows.length) { + bestRows = value; + } + }); + if (!bestRows) { + throw new Error( + "JSON object must contain at least one root key with an array of objects.", + ); + } + return bestRows; +} + +function parseJsonObjectRows(text) { + var parsed; + try { + parsed = JSON.parse(text); + } catch (error) { + throw new Error("Invalid JSON: " + error.message); + } + var rows = extractJsonObjectRows(parsed); + if (!rows.length) { + throw new Error("No rows found to preview."); + } + return rows; +} + +function parseJsonCreateTableRows(text) { + var parsed = parseJsonObjectRows(text); + + var columnNames = []; + var columnMap = {}; + parsed.forEach(function (item, index) { + if (!item || typeof item !== "object" || Array.isArray(item)) { + throw new Error("JSON row " + (index + 1) + " must be an object."); + } + Object.keys(item).forEach(function (name) { + if (!columnMap[name]) { + columnMap[name] = true; + columnNames.push(name); + } + }); + }); + validateCreateTableDataHeaders(columnNames); + + var rows = parsed.map(function (item) { + var row = {}; + columnNames.forEach(function (name) { + row[name] = Object.prototype.hasOwnProperty.call(item, name) + ? normalizeCreateTableDataJsonValue(item[name]) + : null; + }); + return row; + }); + return { + columns: createTableDataColumnObjects(columnNames), + rows: rows, + }; +} + +function createTableDelimitedValueIsInteger(value) { + return /^[-+]?\d+$/.test(String(value).trim()); +} + +function createTableDelimitedValueIsFloat(value) { + var trimmed = String(value).trim(); + if (!trimmed) { + return false; + } + var numberValue = Number(trimmed); + return Number.isFinite(numberValue); +} + +function inferCreateTableDelimitedColumnType(values) { + var nonBlankValues = values.filter(function (value) { + return String(value).trim() !== ""; + }); + if (!nonBlankValues.length) { + return "text"; + } + if (nonBlankValues.every(createTableDelimitedValueIsInteger)) { + return "integer"; + } + if (nonBlankValues.every(createTableDelimitedValueIsFloat)) { + return "float"; + } + return "text"; +} + +function coerceCreateTableDelimitedValue(value, type) { + var trimmed = String(value).trim(); + if (trimmed === "") { + return type === "integer" || type === "float" ? null : ""; + } + if (type === "integer") { + return parseInt(trimmed, 10); + } + if (type === "float") { + return Number(trimmed); + } + return value; +} + +function detectCreateTableDataDelimiter(text) { + var firstLine = + text.split(/\r\n|\n|\r/).find(function (line) { + return line.trim() !== ""; + }) || ""; + var csvRows = delimiterPreviewRows(firstLine, ","); + var tsvRows = delimiterPreviewRows(firstLine, "\t"); + var csvColumns = csvRows.length ? csvRows[0].length : 0; + var tsvColumns = tsvRows.length ? tsvRows[0].length : 0; + + if (firstLine.indexOf("\t") !== -1 && firstLine.indexOf(",") === -1) { + return "\t"; + } + if (tsvColumns > csvColumns) { + return "\t"; + } + if (csvColumns > 1) { + return ","; + } + if (tsvColumns > 1) { + return "\t"; + } + return null; +} + +function parseDelimitedCreateTableRows(text) { + var delimiter = detectCreateTableDataDelimiter(text); + var rows = ( + delimiter === null + ? splitSingleColumnRows(text) + : splitDelimitedRows(text, delimiter) + ).filter(function (row) { + return !bulkInsertDelimitedRowIsBlank(row); + }); + if (!rows.length) { + throw new Error("No rows found to preview."); + } + + var headers = rows[0].map(function (value) { + return value.trim(); + }); + validateCreateTableDataHeaders(headers); + var dataRows = rows.slice(1); + if (!dataRows.length) { + throw new Error("No data rows found to preview."); + } + + dataRows.forEach(function (row, index) { + if (row.length > headers.length) { + throw new Error( + "Row " + + (index + 1) + + " has " + + row.length + + " values, but only " + + headers.length + + " columns were provided.", + ); + } + }); + + var columnTypes = headers.map(function (_name, columnIndex) { + return inferCreateTableDelimitedColumnType( + dataRows.map(function (row) { + return row[columnIndex] || ""; + }), + ); + }); + + return { + columns: createTableDataColumnObjects(headers), + rows: dataRows.map(function (row) { + var rowObject = {}; + headers.forEach(function (name, columnIndex) { + rowObject[name] = coerceCreateTableDelimitedValue( + row[columnIndex] || "", + columnTypes[columnIndex], + ); + }); + return rowObject; + }), + }; +} + +function parseCreateTableDataRows(text) { + var trimmed = text.trim(); + if (!trimmed) { + throw new Error("Paste rows before previewing."); + } + if (trimmed[0] === "[" || trimmed[0] === "{") { + return parseJsonCreateTableRows(trimmed); + } + return parseDelimitedCreateTableRows(trimmed); +} + +function tableCreateDataRecommendedPrimaryKey(columns, rows) { + var candidates = []; + columns.forEach(function (column, columnIndex) { + var distinctValues = {}; + var maxLength = 0; + var valid = rows.length > 0; + rows.forEach(function (row) { + if (!valid) { + return; + } + var value = row[column.name]; + if (value === null || typeof value === "undefined") { + valid = false; + return; + } + var text = String(value).trim(); + if (!text || text.length >= 20 || /\s/.test(text)) { + valid = false; + return; + } + if (distinctValues[text]) { + valid = false; + return; + } + distinctValues[text] = true; + maxLength = Math.max(maxLength, text.length); + }); + if (valid) { + candidates.push({ + name: column.name, + maxLength: maxLength, + columnIndex: columnIndex, + }); + } + }); + candidates.sort(function (left, right) { + if (left.maxLength !== right.maxLength) { + return left.maxLength - right.maxLength; + } + return left.columnIndex - right.columnIndex; + }); + return candidates.length ? candidates[0].name : TABLE_CREATE_AUTOMATIC_PK; +} + +function renderTableCreateDataPreview(state, preview) { + state.dataPreview.textContent = ""; + + var summary = document.createElement("p"); + summary.className = "table-create-data-preview-summary"; + summary.textContent = + "Previewing " + + preview.rows.length + + " row" + + (preview.rows.length === 1 ? "." : "s."); + state.dataPreview.appendChild(summary); + + var pkField = document.createElement("div"); + pkField.className = "table-create-data-pk-field"; + var pkLabel = document.createElement("label"); + pkLabel.className = "table-create-data-label"; + pkLabel.setAttribute("for", "table-create-data-primary-key"); + pkLabel.textContent = "Primary key"; + var pkSelect = document.createElement("select"); + pkSelect.id = "table-create-data-primary-key"; + pkSelect.className = "table-create-input table-create-data-primary-key"; + var automaticOption = document.createElement("option"); + automaticOption.value = TABLE_CREATE_AUTOMATIC_PK; + automaticOption.textContent = "Automatic ID column"; + pkSelect.appendChild(automaticOption); + preview.columns.forEach(function (column) { + var option = document.createElement("option"); + option.value = column.name; + option.textContent = column.name; + pkSelect.appendChild(option); + }); + pkSelect.value = tableCreateDataRecommendedPrimaryKey( + preview.columns, + preview.rows, + ); + pkSelect.addEventListener("change", function () { + clearTableCreateDialogError(state); + }); + state.dataPkSelect = pkSelect; + pkField.appendChild(pkLabel); + pkField.appendChild(pkSelect); + state.dataPreview.appendChild(pkField); + + var tableWrap = document.createElement("div"); + tableWrap.className = "table-create-data-preview-table-wrap"; + var table = document.createElement("table"); + table.className = "table-create-data-preview-table"; + var thead = document.createElement("thead"); + var headerRow = document.createElement("tr"); + preview.columns.forEach(function (column) { + var th = document.createElement("th"); + th.scope = "col"; + th.textContent = column.name; + headerRow.appendChild(th); + }); + thead.appendChild(headerRow); + table.appendChild(thead); + + var tbody = document.createElement("tbody"); + preview.rows.forEach(function (row) { + var tr = document.createElement("tr"); + preview.columns.forEach(function (column) { + var td = document.createElement("td"); + var value = row[column.name]; + td.textContent = bulkInsertPreviewValue(value); + if (value === null) { + td.className = "table-create-data-preview-null"; + } + tr.appendChild(td); + }); + tbody.appendChild(tr); + }); + table.appendChild(tbody); + tableWrap.appendChild(table); + state.dataPreview.appendChild(tableWrap); + state.dataPreview.hidden = false; +} + +function resetTableCreateDataPreview(state) { + state.dataPreviewRows = null; + state.dataPreviewColumns = []; + state.dataPreviewReady = false; + state.dataPkSelect = null; + state.dataPreview.hidden = true; + state.dataPreview.textContent = ""; + syncTableCreateModeUi(state); +} + +function tableCreateTableNameFromFileName(fileName) { + var baseName = (fileName || "").replace(/^.*[\\/]/, ""); + var nameWithoutExtension = baseName.replace(/\.[^.]*$/, ""); + return nameWithoutExtension + .trim() + .replace(/\s+/g, "_") + .toLowerCase() + .replace(/[^a-z0-9_]/g, ""); +} + +async function loadTableCreateDataTextFile(state, file) { + if (!file) { + return; + } + try { + var text = await readTextFile(file); + var tableName = tableCreateTableNameFromFileName(file.name); + if (tableName) { + state.tableName.value = tableName; + state.tableName.dispatchEvent(new Event("input", { bubbles: true })); + } + state.dataTextarea.value = text; + state.dataTextarea.dispatchEvent(new Event("input", { bubbles: true })); + clearTableCreateDialogError(state); + state.dataTextarea.focus(); + } catch (_error) { + showTableCreateDialogError(state, "Could not read that text file."); + } +} + +function previewTableCreateDataRows(state) { + clearTableCreateDialogError(state); + resetTableCreateDataPreview(state); + try { + var preview = parseCreateTableDataRows(state.dataTextarea.value); + state.dataPreviewRows = preview.rows; + state.dataPreviewColumns = preview.columns; + state.dataPreviewReady = true; + renderTableCreateDataPreview(state, preview); + updateTableCreateDialogButtons(state); + } catch (error) { + showTableCreateDialogError( + state, + error.message || "Could not preview rows.", + ); + updateTableCreateDialogButtons(state); + } +} + +function collectTableCreateDataPayload(state) { + var payload = { + table: state.tableName.value.trim(), + rows: state.dataPreviewRows || [], + }; + var primaryKey = state.dataPkSelect + ? state.dataPkSelect.value + : TABLE_CREATE_AUTOMATIC_PK; + payload.pk = + primaryKey === TABLE_CREATE_AUTOMATIC_PK ? "id" : primaryKey || undefined; + return payload; +} + +function validateTableCreateDataPayload(payload, state) { + var tableNameError = validateTableCreateTableName(payload.table); + if (tableNameError) { + return tableNameError; + } + if (!payload.rows.length) { + return "No rows found to create."; + } + if ( + state.dataPkSelect && + state.dataPkSelect.value === TABLE_CREATE_AUTOMATIC_PK && + payload.rows.some(function (row) { + return Object.prototype.hasOwnProperty.call(row, "id"); + }) + ) { + return ( + "Automatic ID column cannot be used because the pasted data " + + "already has an id column." + ); + } + return null; +} + function fallbackTableUrl(tableName) { var data = databaseCreateTableData() || {}; if (!data.path) { @@ -1442,6 +2008,57 @@ async function assignTableCreateColumnTypes( } } +async function createTableFromDataPreview(state) { + var data = databaseCreateTableData(); + if (!data || !data.path) { + showTableCreateDialogError(state, "Could not find the create table URL."); + return; + } + var payload = collectTableCreateDataPayload(state); + var validationError = validateTableCreateDataPayload(payload, state); + if (validationError) { + showTableCreateDialogError(state, validationError); + return; + } + clearTableCreateDialogError(state); + setTableCreateDialogSaving(state, true); + try { + var response = await fetch(data.path, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(payload), + }); + var responseData = null; + try { + responseData = await response.json(); + } catch (_error) { + responseData = null; + } + if (!response.ok || (responseData && responseData.ok === false)) { + throw rowMutationRequestError(response, responseData); + } + var tableUrl = + responseData.table_url || + fallbackTableUrl(responseData.table || payload.table); + state.shouldRestoreFocus = false; + state.dialog.close(); + if (tableUrl) { + location.href = tableUrl; + } else { + location.reload(); + } + } catch (error) { + setTableCreateDialogSaving(state, false); + showTableCreateDialogError( + state, + error.message || "Could not create table", + ); + } +} + async function saveTableCreateDialog(state) { if (state.isSaving) { return; @@ -1451,6 +2068,14 @@ async function saveTableCreateDialog(state) { showTableCreateDialogError(state, "Could not find the create table URL."); return; } + if (tableCreateIsDataMode(state)) { + if (!state.dataPreviewReady) { + previewTableCreateDataRows(state); + } else { + await createTableFromDataPreview(state); + } + return; + } clearTableCreateDialogError(state); var payload = collectTableCreatePayload(state); var columnTypeAssignments = collectTableCreateColumnTypeAssignments(state); @@ -1561,8 +2186,18 @@ function ensureTableCreateDialog(manager) {
+
+ Create table from data +
@@ -1577,13 +2212,27 @@ function ensureTableCreateDialog(manager) { error: dialog.querySelector(".table-create-error"), fields: dialog.querySelector(".table-create-fields"), tableName: dialog.querySelector(".table-create-table-name"), + columnsPanel: dialog.querySelector(".table-create-columns"), columnList: dialog.querySelector(".table-create-column-list"), addColumnButton: dialog.querySelector(".table-create-add-column"), + dataPanel: dialog.querySelector(".table-create-data"), + dataEditor: dialog.querySelector(".table-create-data-editor"), + dataTextarea: dialog.querySelector(".table-create-data-textarea"), + dataOpenFileButton: dialog.querySelector(".table-create-data-open-file"), + dataFileInput: dialog.querySelector(".table-create-data-file-input"), + dataPreview: dialog.querySelector(".table-create-data-preview"), + createFromDataLink: dialog.querySelector(".table-create-from-data"), + manualCreateLink: dialog.querySelector(".table-create-manual"), cancelButton: dialog.querySelector(".table-create-cancel"), saveButton: dialog.querySelector(".table-create-save"), currentButton: null, shouldRestoreFocus: true, isSaving: false, + mode: "manual", + dataPreviewRows: null, + dataPreviewColumns: [], + dataPreviewReady: false, + dataPkSelect: null, initialSignature: "", nextColumnIndex: 0, foreignKeyTargets: [], @@ -1607,13 +2256,114 @@ function ensureTableCreateDialog(manager) { }); tableCreateDialogState.cancelButton.addEventListener("click", function () { + if ( + tableCreateIsDataMode(tableCreateDialogState) && + tableCreateDialogState.dataPreviewReady && + !tableCreateDialogState.isSaving + ) { + resetTableCreateDataPreview(tableCreateDialogState); + updateTableCreateDialogButtons(tableCreateDialogState); + tableCreateDialogState.dataTextarea.focus(); + return; + } closeTableCreateDialogIfConfirmed(tableCreateDialogState); }); + tableCreateDialogState.createFromDataLink.addEventListener( + "click", + function (ev) { + ev.preventDefault(); + showTableCreateDataMode(tableCreateDialogState); + }, + ); + + tableCreateDialogState.manualCreateLink.addEventListener( + "click", + function (ev) { + ev.preventDefault(); + showTableCreateManualMode(tableCreateDialogState); + }, + ); + tableCreateDialogState.tableName.addEventListener("input", function () { clearTableCreateDialogError(tableCreateDialogState); }); + tableCreateDialogState.dataOpenFileButton.addEventListener( + "click", + function () { + tableCreateDialogState.dataFileInput.click(); + }, + ); + + tableCreateDialogState.dataFileInput.addEventListener( + "change", + async function (ev) { + var files = ev.target.files; + await loadTableCreateDataTextFile( + tableCreateDialogState, + files && files.length ? files[0] : null, + ); + ev.target.value = ""; + }, + ); + + tableCreateDialogState.dataTextarea.addEventListener( + "dragenter", + function (ev) { + ev.preventDefault(); + tableCreateDialogState.dataTextarea.classList.add( + "table-create-data-drop-target", + ); + }, + ); + + tableCreateDialogState.dataTextarea.addEventListener( + "dragover", + function (ev) { + ev.preventDefault(); + tableCreateDialogState.dataTextarea.classList.add( + "table-create-data-drop-target", + ); + }, + ); + + tableCreateDialogState.dataTextarea.addEventListener( + "dragleave", + function () { + tableCreateDialogState.dataTextarea.classList.remove( + "table-create-data-drop-target", + ); + }, + ); + + tableCreateDialogState.dataTextarea.addEventListener( + "drop", + async function (ev) { + ev.preventDefault(); + tableCreateDialogState.dataTextarea.classList.remove( + "table-create-data-drop-target", + ); + var files = ev.dataTransfer && ev.dataTransfer.files; + if (!files || !files.length) { + return; + } + await loadTableCreateDataTextFile(tableCreateDialogState, files[0]); + }, + ); + + tableCreateDialogState.dataTextarea.addEventListener("dragend", function () { + tableCreateDialogState.dataTextarea.classList.remove( + "table-create-data-drop-target", + ); + }); + + tableCreateDialogState.dataTextarea.addEventListener("input", function () { + resetTableCreateDataPreview(tableCreateDialogState); + clearTableCreateDialogError(tableCreateDialogState); + updateTableCreateDialogButtons(tableCreateDialogState); + }); + dialog.addEventListener("click", function (ev) { if (ev.target === dialog) { closeTableCreateDialogIfConfirmed(tableCreateDialogState); @@ -3583,6 +4333,14 @@ function tableInsertUrl() { return url.toString(); } +function tableUpsertUrl() { + var data = tableInsertData(); + if (data && data.upsertPath) { + return new URL(data.upsertPath, location.href).toString(); + } + return null; +} + function rowResourceUrl(row) { var rowId = row.getAttribute("data-row"); if (!rowId) { @@ -4864,18 +5622,64 @@ function clearRowEditDialogError(state) { state.error.textContent = ""; } -function showRowEditDialogError(state, message) { +function showRowEditDialogError(state, message, options) { state.error.hidden = false; state.error.textContent = message; - state.error.focus(); + if (!options || options.focus !== false) { + state.error.focus(); + } +} + +function rowEditIsMultipleInsert(state) { + return state.mode === "insert" && state.insertMode === "multiple"; +} + +function syncRowEditInsertModeUi(state) { + var isInsert = state.mode === "insert"; + var isMultiple = rowEditIsMultipleInsert(state); + state.fields.hidden = isMultiple; + state.bulkInsertPanel.hidden = !isMultiple; + state.bulkInsertEditor.hidden = !isMultiple || state.bulkInsertPreviewReady; + state.bulkInsertPreview.hidden = !isMultiple || !state.bulkInsertPreviewReady; + state.bulkInsertLink.hidden = !isInsert || isMultiple; + state.singleInsertLink.hidden = !isInsert || !isMultiple; } function updateRowEditDialogButtons(state) { state.saveButton.disabled = - state.isLoading || state.isSaving || !state.hasLoaded; + state.isLoading || + state.isSaving || + !state.hasLoaded || + state.bulkInsertInserted || + (rowEditIsMultipleInsert(state) && + !state.bulkInsertPreviewReady && + !!state.bulkInsertLiveValidationError); state.cancelButton.disabled = state.isSaving; - var saveLabel = state.mode === "insert" ? "Insert row" : "Save"; - state.saveButton.textContent = state.isSaving ? "Saving..." : saveLabel; + syncRowEditInsertModeUi(state); + state.cancelButton.textContent = + rowEditIsMultipleInsert(state) && + state.bulkInsertPreviewReady && + !state.bulkInsertInserted + ? "Back" + : state.bulkInsertInserted + ? "Close and view table" + : "Cancel"; + var saveLabel = rowEditIsMultipleInsert(state) + ? state.bulkInsertInserted + ? "Inserted" + : state.bulkInsertPreviewReady + ? bulkInsertSaveLabel(state) + : "Preview rows" + : state.mode === "insert" + ? "Insert row" + : "Save"; + state.saveButton.textContent = state.isSaving + ? rowEditIsMultipleInsert(state) + ? bulkInsertConflictMode(state) === "upsert" + ? "Updating..." + : "Inserting..." + : "Saving..." + : saveLabel; state.form.setAttribute( "aria-busy", state.isLoading || state.isSaving ? "true" : "false", @@ -5076,6 +5880,16 @@ function rowEditDialogHasChanges(state) { if (!state || !state.hasLoaded || state.isLoading) { return false; } + if (state.bulkInsertInserted) { + return false; + } + if ( + state.mode === "insert" && + state.bulkInsertTextarea && + state.bulkInsertTextarea.value.trim() + ) { + return true; + } var fields = state.fields.querySelectorAll(".row-edit-field"); for (var i = 0; i < fields.length; i += 1) { var fieldApi = fields[i]._datasetteColumnFormField; @@ -5109,6 +5923,831 @@ function closeRowEditDialogIfConfirmed(state) { return true; } +function setRowInsertDialogTitle(state) { + var insertData = tableInsertData() || {}; + var title = rowEditIsMultipleInsert(state) + ? "Insert multiple rows" + : "Insert row"; + setRowDialogTitle( + state.title, + insertData.tableName ? title + " into " + insertData.tableName : title, + ); +} + +function showMultipleRowInsert(state) { + if (!state || state.mode !== "insert" || state.isSaving) { + return; + } + state.insertMode = "multiple"; + if (state.bulkInsertPreviewReady || state.bulkInsertInserted) { + resetBulkInsertPreview(state); + } + clearRowEditDialogError(state); + setRowInsertDialogTitle(state); + syncBulkInsertConflictUi(state); + syncBulkInsertTextareaValidation(state); + updateRowEditDialogButtons(state); + state.bulkInsertTextarea.focus(); +} + +function showSingleRowInsert(state) { + if (!state || state.mode !== "insert" || state.isSaving) { + return; + } + state.insertMode = "single"; + clearRowEditDialogError(state); + setRowInsertDialogTitle(state); + updateRowEditDialogButtons(state); + if (!focusFirstRowEditControl(state, { skipReadonly: true })) { + state.saveButton.focus(); + } +} + +function bulkInsertConflictMode(state) { + if (!state || !state.bulkInsertHasPrimaryKeyColumns) { + return "insert"; + } + return state.bulkInsertConflictMode || "ignore"; +} + +function syncBulkInsertConflictUi(state) { + if (!state || !state.bulkInsertConflictField) { + return; + } + var insertData = tableInsertData() || {}; + var primaryKeys = insertData.primaryKeys || []; + var hasPrimaryKeys = primaryKeys.length > 0; + var hasPrimaryKeyColumns = + hasPrimaryKeys && + bulkInsertTextIncludesPrimaryKeyColumns( + state.bulkInsertTextarea.value, + state.bulkInsertColumnDetails, + primaryKeys, + ); + state.bulkInsertHasPrimaryKeyColumns = hasPrimaryKeyColumns; + var canUpsert = hasPrimaryKeyColumns && !!state.currentUpsertUrl; + var upsertOption = state.bulkInsertConflictSelect.querySelector( + 'option[value="upsert"]', + ); + if (upsertOption) { + upsertOption.disabled = !canUpsert; + upsertOption.hidden = !canUpsert; + } + if ( + !hasPrimaryKeyColumns || + (!canUpsert && bulkInsertConflictMode(state) === "upsert") + ) { + state.bulkInsertConflictMode = hasPrimaryKeys ? "ignore" : "insert"; + state.bulkInsertConflictSelect.value = state.bulkInsertConflictMode; + } + state.bulkInsertConflictField.hidden = !hasPrimaryKeyColumns; + state.bulkInsertConflictSelect.value = bulkInsertConflictMode(state); + var helpText = ""; + if (bulkInsertConflictMode(state) === "upsert") { + helpText = + "Rows with existing primary keys will be updated; new primary keys will be inserted."; + } else if (bulkInsertConflictMode(state) === "ignore") { + helpText = "Rows with existing primary keys will be skipped."; + } else { + helpText = "Rows with existing primary keys will stop the import."; + } + state.bulkInsertConflictHelp.textContent = helpText; +} + +function bulkInsertSaveLabel(state) { + if (!state.bulkInsertPreviewReady) { + return "Preview rows"; + } + if (bulkInsertConflictMode(state) === "upsert") { + return "Update or insert rows"; + } + return "Insert these rows"; +} + +function readTextFile(file) { + if (file.text) { + return file.text(); + } + return new Promise(function (resolve, reject) { + var reader = new FileReader(); + reader.onload = function () { + resolve(reader.result || ""); + }; + reader.onerror = function () { + reject(reader.error); + }; + reader.readAsText(file); + }); +} + +async function loadBulkInsertTextFile(state, file) { + if (!file) { + return; + } + try { + state.bulkInsertTextarea.value = await readTextFile(file); + state.bulkInsertTextarea.dispatchEvent( + new Event("input", { bubbles: true }), + ); + state.bulkInsertTextarea.focus(); + } catch (_error) { + showRowEditDialogError(state, "Could not read that text file."); + } +} + +function bulkInsertTemplateText(state) { + return (state.bulkInsertTemplateColumns || []).join("\t"); +} + +async function copyTextToClipboard(text) { + if (navigator.clipboard && navigator.clipboard.writeText) { + await navigator.clipboard.writeText(text); + return; + } + var textarea = document.createElement("textarea"); + textarea.value = text; + textarea.setAttribute("readonly", ""); + textarea.style.position = "fixed"; + textarea.style.top = "-1000px"; + textarea.style.left = "-1000px"; + document.body.appendChild(textarea); + textarea.select(); + var copied = document.execCommand("copy"); + textarea.remove(); + if (!copied) { + throw new Error("copy failed"); + } +} + +function setBulkInsertCopyButtonReady(state) { + state.copyTemplateButton.textContent = ""; + var wideLabel = document.createElement("span"); + wideLabel.className = "row-edit-copy-template-label-wide"; + wideLabel.textContent = "Copy spreadsheet template"; + state.copyTemplateButton.appendChild(wideLabel); + var narrowLabel = document.createElement("span"); + narrowLabel.className = "row-edit-copy-template-label-narrow"; + narrowLabel.textContent = "Copy template"; + state.copyTemplateButton.appendChild(narrowLabel); +} + +function setBulkInsertCopyButtonCopied(state) { + state.copyTemplateButton.textContent = "Copied"; + clearTimeout(state.copyTemplateResetTimer); + state.copyTemplateResetTimer = setTimeout(function () { + setBulkInsertCopyButtonReady(state); + }, 1500); +} + +function resetBulkInsertPreview(state) { + state.bulkInsertPreviewRows = null; + state.bulkInsertPreviewReady = false; + state.bulkInsertInserted = false; + state.bulkInsertInsertedCount = 0; + state.bulkInsertPreview.hidden = true; + state.bulkInsertPreview.textContent = ""; + state.bulkInsertProgress.hidden = true; + state.bulkInsertProgressBar.value = 0; + state.bulkInsertProgressBar.max = 1; + state.bulkInsertProgressStatus.textContent = ""; + syncBulkInsertConflictUi(state); + syncRowEditInsertModeUi(state); +} + +function normalizeBulkInsertCell(column, value) { + if (typeof value === "undefined") { + return column.notnull ? "" : null; + } + if (value === null) { + return column.notnull ? "" : null; + } + if (value === "" && column.notnull) { + return ""; + } + if (column.value_kind === "number" && typeof value === "string") { + return valueFromRowEditText(column.name, value, "number"); + } + return value; +} + +function rowObjectForBulkInsert(valuesByColumn, columns) { + var row = {}; + columns.forEach(function (column) { + var hasValue = Object.prototype.hasOwnProperty.call( + valuesByColumn, + column.name, + ); + if (!hasValue) { + return; + } + row[column.name] = normalizeBulkInsertCell( + column, + valuesByColumn[column.name], + ); + }); + return row; +} + +function splitDelimitedRows(text, delimiter) { + var rows = []; + var row = []; + var cell = ""; + var inQuotes = false; + + for (var i = 0; i < text.length; i += 1) { + var character = text[i]; + if (inQuotes) { + if (character === '"') { + if (text[i + 1] === '"') { + cell += '"'; + i += 1; + } else { + inQuotes = false; + } + } else { + cell += character; + } + continue; + } + + if (character === '"') { + inQuotes = true; + } else if (character === delimiter) { + row.push(cell); + cell = ""; + } else if (character === "\n" || character === "\r") { + row.push(cell); + rows.push(row); + row = []; + cell = ""; + if (character === "\r" && text[i + 1] === "\n") { + i += 1; + } + } else { + cell += character; + } + } + + if (inQuotes) { + throw new Error("Unclosed quoted value."); + } + row.push(cell); + rows.push(row); + + while (rows.length && bulkInsertDelimitedRowIsBlank(rows[rows.length - 1])) { + rows.pop(); + } + return rows; +} + +function bulkInsertDelimitedRowIsBlank(row) { + return row.every(function (value) { + return value.trim() === ""; + }); +} + +function delimiterPreviewRows(text, delimiter) { + try { + return splitDelimitedRows(text, delimiter); + } catch (_error) { + return []; + } +} + +function splitSingleColumnRows(text) { + var rows = text.split(/\r\n|\n|\r/).map(function (line) { + return [line]; + }); + while (rows.length && bulkInsertDelimitedRowIsBlank(rows[rows.length - 1])) { + rows.pop(); + } + return rows; +} + +function detectBulkInsertDelimiter(text, columns) { + var firstLine = + text.split(/\r\n|\n|\r/).find(function (line) { + return line.trim() !== ""; + }) || ""; + var csvRows = delimiterPreviewRows(firstLine, ","); + var tsvRows = delimiterPreviewRows(firstLine, "\t"); + var csvColumns = csvRows.length ? csvRows[0].length : 0; + var tsvColumns = tsvRows.length ? tsvRows[0].length : 0; + + if (firstLine.indexOf("\t") !== -1 && firstLine.indexOf(",") === -1) { + return "\t"; + } + if (tsvColumns > csvColumns) { + return "\t"; + } + if (csvColumns > 1) { + return ","; + } + if (tsvColumns > 1) { + return "\t"; + } + if (columns.length === 1 || bulkInsertColumnMap(columns)[firstLine.trim()]) { + return null; + } + throw new Error("Could not detect CSV or TSV columns."); +} + +function bulkInsertColumnMap(columns) { + var map = {}; + columns.forEach(function (column) { + map[column.name] = column; + }); + return map; +} + +function bulkInsertTextIncludesPrimaryKeyColumns(text, columns, primaryKeys) { + if (!primaryKeys.length || !text.trim()) { + return false; + } + var trimmed = text.trim(); + try { + if (trimmed[0] === "[" || trimmed[0] === "{") { + return jsonBulkInsertTextIncludesPrimaryKeyColumns(trimmed, primaryKeys); + } + return delimitedBulkInsertTextIncludesPrimaryKeyColumns( + trimmed, + columns, + primaryKeys, + ); + } catch (_error) { + return false; + } +} + +function jsonBulkInsertTextIncludesPrimaryKeyColumns(text, primaryKeys) { + var rows = parseJsonObjectRows(text); + var seenKeys = {}; + rows.forEach(function (row) { + Object.keys(row).forEach(function (key) { + seenKeys[key] = true; + }); + }); + return primaryKeys.every(function (key) { + return !!seenKeys[key]; + }); +} + +function delimitedBulkInsertTextIncludesPrimaryKeyColumns( + text, + columns, + primaryKeys, +) { + var delimiter = detectBulkInsertDelimiter(text, columns); + var rows = ( + delimiter === null + ? splitSingleColumnRows(text) + : splitDelimitedRows(text, delimiter) + ).filter(function (row) { + return !bulkInsertDelimitedRowIsBlank(row); + }); + if (!rows.length) { + return false; + } + + var columnMap = bulkInsertColumnMap(columns); + var header = rows[0].map(function (value) { + return value.trim(); + }); + var headerMatches = header.filter(function (name) { + return !!columnMap[name]; + }).length; + if (headerMatches > 0) { + return primaryKeys.every(function (key) { + return header.indexOf(key) !== -1; + }); + } + + var headers = columns.map(function (column) { + return column.name; + }); + var suppliedColumnCount = rows.reduce(function (count, row) { + return Math.max(count, row.length); + }, 0); + return primaryKeys.every(function (key) { + var index = headers.indexOf(key); + return index !== -1 && index < suppliedColumnCount; + }); +} + +function bulkInsertLiveValidationShouldWait(message) { + return ( + message === "Paste rows before previewing." || + message === "No data rows found to preview." || + message.indexOf("Invalid JSON:") === 0 + ); +} + +function bulkInsertLiveValidationError(state) { + var text = state.bulkInsertTextarea.value; + if (!text.trim()) { + return null; + } + try { + parseBulkInsertRows(text, state.bulkInsertColumnDetails); + } catch (error) { + var message = error.message || "Could not preview rows."; + return bulkInsertLiveValidationShouldWait(message) ? null : message; + } + return null; +} + +function syncBulkInsertTextareaValidation(state) { + if (!rowEditIsMultipleInsert(state) || state.bulkInsertPreviewReady) { + state.bulkInsertLiveValidationError = null; + return; + } + state.bulkInsertLiveValidationError = bulkInsertLiveValidationError(state); + if (state.bulkInsertLiveValidationError) { + showRowEditDialogError(state, state.bulkInsertLiveValidationError, { + focus: false, + }); + } else { + clearRowEditDialogError(state); + } +} + +function parseJsonBulkInsertRows(text, columns) { + var parsed = parseJsonObjectRows(text); + + var columnMap = bulkInsertColumnMap(columns); + return parsed.map(function (item, index) { + if (!item || typeof item !== "object" || Array.isArray(item)) { + throw new Error("JSON row " + (index + 1) + " must be an object."); + } + Object.keys(item).forEach(function (key) { + if (!columnMap[key]) { + throw new Error( + "JSON row " + (index + 1) + " has unknown column " + key + ".", + ); + } + }); + return rowObjectForBulkInsert(item, columns); + }); +} + +function parseDelimitedBulkInsertRows(text, columns) { + var delimiter = detectBulkInsertDelimiter(text, columns); + var rows = ( + delimiter === null + ? splitSingleColumnRows(text) + : splitDelimitedRows(text, delimiter) + ).filter(function (row) { + return !bulkInsertDelimitedRowIsBlank(row); + }); + if (!rows.length) { + throw new Error("No rows found to preview."); + } + + var columnMap = bulkInsertColumnMap(columns); + var header = rows[0].map(function (value) { + return value.trim(); + }); + var headerMatches = header.filter(function (name) { + return !!columnMap[name]; + }).length; + var hasHeader = headerMatches > 0; + var dataRows = hasHeader ? rows.slice(1) : rows; + var headers = hasHeader + ? header + : columns.map(function (column) { + return column.name; + }); + var seenHeaders = {}; + + if (hasHeader) { + headers.forEach(function (name) { + if (!name) { + return; + } + if (!columnMap[name]) { + throw new Error("Unknown column " + name + " in header row."); + } + if (seenHeaders[name]) { + throw new Error("Duplicate column " + name + " in header row."); + } + seenHeaders[name] = true; + }); + } + + if (!dataRows.length) { + throw new Error("No data rows found to preview."); + } + + return dataRows.map(function (row, rowIndex) { + if (row.length > headers.length) { + throw new Error( + "Row " + + (rowIndex + 1) + + " has " + + row.length + + " values, but only " + + headers.length + + " columns were provided.", + ); + } + var valuesByColumn = {}; + row.forEach(function (value, index) { + var columnName = headers[index]; + if (columnMap[columnName]) { + valuesByColumn[columnName] = value; + } + }); + return rowObjectForBulkInsert(valuesByColumn, columns); + }); +} + +function parseBulkInsertRows(text, columns) { + var trimmed = text.trim(); + if (!trimmed) { + throw new Error("Paste rows before previewing."); + } + if (trimmed[0] === "[" || trimmed[0] === "{") { + return parseJsonBulkInsertRows(trimmed, columns); + } + return parseDelimitedBulkInsertRows(trimmed, columns); +} + +function bulkInsertPreviewValue(value) { + if (value === null) { + return "null"; + } + if (typeof value === "object") { + return JSON.stringify(value); + } + return String(value); +} + +function bulkInsertPreviewCell(column, hasValue, value) { + if (!hasValue && column.is_auto_pk) { + return { + text: "auto", + className: "row-edit-bulk-preview-auto", + }; + } + if (value === null) { + return { + text: bulkInsertPreviewValue(value), + className: "row-edit-bulk-preview-null", + }; + } + return { + text: hasValue ? bulkInsertPreviewValue(value) : "", + className: "", + }; +} + +function renderBulkInsertPreview(state, rows) { + state.bulkInsertPreview.textContent = ""; + var summary = document.createElement("p"); + summary.className = "row-edit-bulk-preview-summary"; + summary.textContent = + "Previewing " + rows.length + " row" + (rows.length === 1 ? "." : "s."); + state.bulkInsertPreview.appendChild(summary); + + var tableWrap = document.createElement("div"); + tableWrap.className = "row-edit-bulk-preview-table-wrap"; + var table = document.createElement("table"); + table.className = "row-edit-bulk-preview-table"; + var thead = document.createElement("thead"); + var headerRow = document.createElement("tr"); + state.bulkInsertColumnDetails.forEach(function (column) { + var th = document.createElement("th"); + th.scope = "col"; + th.textContent = column.name; + headerRow.appendChild(th); + }); + thead.appendChild(headerRow); + table.appendChild(thead); + + var tbody = document.createElement("tbody"); + rows.forEach(function (row) { + var tr = document.createElement("tr"); + state.bulkInsertColumnDetails.forEach(function (column) { + var td = document.createElement("td"); + var hasValue = Object.prototype.hasOwnProperty.call(row, column.name); + var value = hasValue ? row[column.name] : ""; + var cell = bulkInsertPreviewCell(column, hasValue, value); + td.textContent = cell.text; + if (cell.className) { + td.className = cell.className; + } + tr.appendChild(td); + }); + tbody.appendChild(tr); + }); + table.appendChild(tbody); + tableWrap.appendChild(table); + state.bulkInsertPreview.appendChild(tableWrap); + state.bulkInsertPreview.hidden = false; +} + +function previewBulkInsertRows(state) { + clearRowEditDialogError(state); + resetBulkInsertPreview(state); + syncBulkInsertConflictUi(state); + try { + var rows = parseBulkInsertRows( + state.bulkInsertTextarea.value, + state.bulkInsertColumnDetails, + ); + state.bulkInsertPreviewRows = rows; + state.bulkInsertPreviewReady = true; + renderBulkInsertPreview(state, rows); + updateRowEditDialogButtons(state); + } catch (error) { + showRowEditDialogError(state, error.message || "Could not preview rows."); + updateRowEditDialogButtons(state); + } +} + +function updateBulkInsertProgress(state, inserted, total) { + var words = bulkInsertProgressWords(state); + state.bulkInsertProgress.hidden = false; + state.bulkInsertProgressBar.max = total || 1; + state.bulkInsertProgressBar.value = inserted; + state.bulkInsertProgressStatus.textContent = + inserted >= total + ? total + " row" + (total === 1 ? " " : "s ") + words.complete + "." + : words.active + " " + inserted + " of " + total + " rows..."; +} + +function bulkInsertBatches(rows, batchSize) { + var batches = []; + var size = Math.max(1, batchSize || 1); + for (var index = 0; index < rows.length; index += size) { + batches.push(rows.slice(index, index + size)); + } + return batches; +} + +function animateBulkInsertProgress(state, from, to, total, duration) { + state.bulkInsertProgress.hidden = false; + state.bulkInsertProgressBar.max = total || 1; + if (duration <= 0 || !window.requestAnimationFrame) { + updateBulkInsertProgress(state, to, total); + return Promise.resolve(); + } + + return new Promise(function (resolve) { + var startTime = null; + var step = function (timestamp) { + if (startTime === null) { + startTime = timestamp; + } + var progress = Math.min((timestamp - startTime) / duration, 1); + var easedProgress = 1 - Math.pow(1 - progress, 3); + var value = from + (to - from) * easedProgress; + var displayValue = progress === 1 ? to : Math.floor(value); + var words = bulkInsertProgressWords(state); + state.bulkInsertProgressBar.value = value; + state.bulkInsertProgressStatus.textContent = + displayValue >= total + ? total + " row" + (total === 1 ? " " : "s ") + words.complete + "." + : words.active + " " + displayValue + " of " + total + " rows..."; + if (progress < 1) { + window.requestAnimationFrame(step); + } else { + updateBulkInsertProgress(state, to, total); + resolve(); + } + }; + window.requestAnimationFrame(step); + }); +} + +function bulkInsertProgressWords(state) { + var mode = bulkInsertConflictMode(state); + if (mode === "upsert") { + return { + active: "Upserting", + complete: "upserted", + }; + } + if (mode === "ignore") { + return { + active: "Processing", + complete: "processed", + }; + } + return { + active: "Inserting", + complete: "inserted", + }; +} + +function validateBulkInsertConflictRows(state, rows) { + if (bulkInsertConflictMode(state) !== "upsert") { + return null; + } + var insertData = tableInsertData() || {}; + var primaryKeys = insertData.primaryKeys || []; + for (var index = 0; index < rows.length; index += 1) { + var row = rows[index]; + var missing = primaryKeys.filter(function (key) { + return ( + !Object.prototype.hasOwnProperty.call(row, key) || + row[key] === null || + typeof row[key] === "undefined" + ); + }); + if (missing.length) { + return ( + "Row " + + (index + 1) + + " is missing primary key " + + missing.join(", ") + + ". Upsert requires primary key values for every row." + ); + } + } + return null; +} + +async function insertBulkPreviewRows(state) { + if (!state.bulkInsertPreviewRows || state.bulkInsertInserted) { + return; + } + var conflictMode = bulkInsertConflictMode(state); + var url = + conflictMode === "upsert" ? state.currentUpsertUrl : state.currentInsertUrl; + if (!url) { + showRowEditDialogError( + state, + conflictMode === "upsert" + ? "Could not find the row upsert URL." + : "Could not find the row insert URL.", + ); + return; + } + + var rows = state.bulkInsertPreviewRows; + var validationError = validateBulkInsertConflictRows(state, rows); + if (validationError) { + showRowEditDialogError(state, validationError); + return; + } + var total = rows.length; + var inserted = state.bulkInsertInsertedCount || 0; + var batches = bulkInsertBatches( + rows.slice(inserted), + state.bulkInsertMaxRows, + ); + var progressAnimationDuration = 500 / Math.max(batches.length, 1); + + clearRowEditDialogError(state); + updateBulkInsertProgress(state, inserted, total); + setRowEditDialogSaving(state, true); + try { + for (var batchIndex = 0; batchIndex < batches.length; batchIndex += 1) { + var batch = batches[batchIndex]; + var payload = { rows: batch }; + if (conflictMode === "ignore") { + payload.ignore = true; + } + var response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(payload), + }); + var data = null; + try { + data = await response.json(); + } catch (_error) { + data = null; + } + if (!response.ok || (data && data.ok === false)) { + throw rowMutationRequestError(response, data); + } + var previousInserted = inserted; + inserted += batch.length; + state.bulkInsertInsertedCount = inserted; + await animateBulkInsertProgress( + state, + previousInserted, + inserted, + total, + progressAnimationDuration, + ); + } + state.bulkInsertInserted = true; + state.shouldReloadOnClose = true; + state.redirectOnCloseUrl = tableBaseUrl().toString(); + updateBulkInsertProgress(state, inserted, total); + } catch (error) { + showRowEditDialogError(state, error.message || "Could not insert rows."); + } finally { + setRowEditDialogSaving(state, false); + } +} + function scheduleCloseRowEditDialogIfConfirmed(state) { // Fix for an issue in Safari where hitting Esc would show // the confirm() prompt asking if state should be discarded @@ -5207,6 +6846,14 @@ async function saveRowEditDialog(state) { if (state.isLoading || state.isSaving || !state.hasLoaded) { return; } + if (rowEditIsMultipleInsert(state)) { + if (!state.bulkInsertPreviewReady) { + previewBulkInsertRows(state); + } else if (!state.bulkInsertInserted) { + await insertBulkPreviewRows(state); + } + return; + } clearRowEditDialogError(state); setRowEditDialogSaving(state, true); @@ -5366,6 +7013,7 @@ function renderRowEditFields(state, data) { var columnTypes = data.column_types || {}; var columnDetails = data.column_details || {}; + state.insertMode = "single"; destroyRowEditFields(state); columns.forEach(function (column, index) { var columnDetail = columnDetails[column] || {}; @@ -5399,7 +7047,23 @@ function renderRowEditFields(state, data) { function renderRowInsertFields(state, data) { var columns = data.columns || []; + var bulkColumns = data.bulkColumns || columns; + state.insertMode = "single"; + state.bulkInsertColumnDetails = bulkColumns.slice(); + state.bulkInsertMaxRows = data.maxInsertRows || 100; + state.bulkInsertColumns = bulkColumns.map(function (column) { + return column.name; + }); + state.bulkInsertTemplateColumns = columns.map(function (column) { + return column.name; + }); + state.copyTemplateButton.disabled = !state.bulkInsertTemplateColumns.length; + setBulkInsertCopyButtonReady(state); + syncBulkInsertConflictUi(state); + clearTimeout(state.copyTemplateResetTimer); + state.copyTemplateResetTimer = null; + resetBulkInsertPreview(state); destroyRowEditFields(state); columns.forEach(function (column, index) { state.fields.appendChild( @@ -5489,7 +7153,36 @@ function ensureRowEditDialog(manager) {

Loading row...

+
+ +
@@ -5505,6 +7198,27 @@ function ensureRowEditDialog(manager) { loading: dialog.querySelector(".row-edit-loading"), error: dialog.querySelector(".row-edit-error"), fields: dialog.querySelector(".row-edit-fields"), + bulkInsertPanel: dialog.querySelector(".row-edit-bulk"), + bulkInsertEditor: dialog.querySelector(".row-edit-bulk-editor"), + bulkInsertTextarea: dialog.querySelector(".row-edit-bulk-textarea"), + bulkInsertPreview: dialog.querySelector(".row-edit-bulk-preview"), + bulkInsertProgress: dialog.querySelector(".row-edit-bulk-progress"), + bulkInsertProgressBar: dialog.querySelector(".row-edit-bulk-progress-bar"), + bulkInsertProgressStatus: dialog.querySelector( + ".row-edit-bulk-progress-status", + ), + bulkInsertConflictField: dialog.querySelector(".row-edit-bulk-conflict"), + bulkInsertConflictSelect: dialog.querySelector( + ".row-edit-bulk-conflict-mode", + ), + bulkInsertConflictHelp: dialog.querySelector( + ".row-edit-bulk-conflict-help", + ), + copyTemplateButton: dialog.querySelector(".row-edit-copy-template"), + bulkInsertOpenFileButton: dialog.querySelector(".row-edit-bulk-open-file"), + bulkInsertFileInput: dialog.querySelector(".row-edit-bulk-file-input"), + bulkInsertLink: dialog.querySelector(".row-edit-bulk-insert"), + singleInsertLink: dialog.querySelector(".row-edit-single-insert"), cancelButton: dialog.querySelector(".row-edit-cancel"), saveButton: dialog.querySelector(".row-edit-save"), currentButton: null, @@ -5512,9 +7226,25 @@ function ensureRowEditDialog(manager) { currentRowId: null, currentPkPath: null, currentInsertUrl: null, + currentUpsertUrl: null, currentUpdateUrl: null, currentFragmentUrl: null, mode: "edit", + insertMode: "single", + bulkInsertConflictMode: "ignore", + bulkInsertHasPrimaryKeyColumns: false, + bulkInsertLiveValidationError: null, + bulkInsertColumns: [], + bulkInsertTemplateColumns: [], + bulkInsertColumnDetails: [], + bulkInsertPreviewRows: null, + bulkInsertPreviewReady: false, + bulkInsertInserted: false, + bulkInsertInsertedCount: 0, + bulkInsertMaxRows: 100, + shouldReloadOnClose: false, + redirectOnCloseUrl: null, + copyTemplateResetTimer: null, loadId: 0, manager: manager, isLoading: false, @@ -5530,12 +7260,139 @@ function ensureRowEditDialog(manager) { }); rowEditDialogState.cancelButton.addEventListener("click", function () { + if ( + rowEditIsMultipleInsert(rowEditDialogState) && + rowEditDialogState.bulkInsertPreviewReady && + !rowEditDialogState.bulkInsertInserted && + !rowEditDialogState.isSaving + ) { + resetBulkInsertPreview(rowEditDialogState); + updateRowEditDialogButtons(rowEditDialogState); + rowEditDialogState.bulkInsertTextarea.focus(); + return; + } if (!rowEditDialogState.isSaving) { rowEditDialogState.shouldRestoreFocus = true; dialog.close(); } }); + rowEditDialogState.bulkInsertLink.addEventListener("click", function (ev) { + ev.preventDefault(); + showMultipleRowInsert(rowEditDialogState); + }); + + rowEditDialogState.singleInsertLink.addEventListener("click", function (ev) { + ev.preventDefault(); + showSingleRowInsert(rowEditDialogState); + }); + + rowEditDialogState.copyTemplateButton.addEventListener( + "click", + async function () { + try { + await copyTextToClipboard(bulkInsertTemplateText(rowEditDialogState)); + clearRowEditDialogError(rowEditDialogState); + setBulkInsertCopyButtonCopied(rowEditDialogState); + } catch (_error) { + showRowEditDialogError( + rowEditDialogState, + "Could not copy the spreadsheet template.", + ); + } + }, + ); + + rowEditDialogState.bulkInsertOpenFileButton.addEventListener( + "click", + function () { + rowEditDialogState.bulkInsertFileInput.click(); + }, + ); + + rowEditDialogState.bulkInsertFileInput.addEventListener( + "change", + async function (ev) { + var files = ev.target.files; + await loadBulkInsertTextFile( + rowEditDialogState, + files && files.length ? files[0] : null, + ); + ev.target.value = ""; + }, + ); + + rowEditDialogState.bulkInsertTextarea.addEventListener( + "dragenter", + function (ev) { + ev.preventDefault(); + rowEditDialogState.bulkInsertTextarea.classList.add( + "row-edit-bulk-drop-target", + ); + }, + ); + + rowEditDialogState.bulkInsertTextarea.addEventListener( + "dragover", + function (ev) { + ev.preventDefault(); + rowEditDialogState.bulkInsertTextarea.classList.add( + "row-edit-bulk-drop-target", + ); + }, + ); + + rowEditDialogState.bulkInsertTextarea.addEventListener( + "dragleave", + function () { + rowEditDialogState.bulkInsertTextarea.classList.remove( + "row-edit-bulk-drop-target", + ); + }, + ); + + rowEditDialogState.bulkInsertTextarea.addEventListener( + "drop", + async function (ev) { + ev.preventDefault(); + rowEditDialogState.bulkInsertTextarea.classList.remove( + "row-edit-bulk-drop-target", + ); + var files = ev.dataTransfer && ev.dataTransfer.files; + if (!files || !files.length) { + return; + } + await loadBulkInsertTextFile(rowEditDialogState, files[0]); + }, + ); + + rowEditDialogState.bulkInsertTextarea.addEventListener( + "dragend", + function () { + rowEditDialogState.bulkInsertTextarea.classList.remove( + "row-edit-bulk-drop-target", + ); + }, + ); + + rowEditDialogState.bulkInsertTextarea.addEventListener("input", function () { + resetBulkInsertPreview(rowEditDialogState); + syncBulkInsertTextareaValidation(rowEditDialogState); + updateRowEditDialogButtons(rowEditDialogState); + }); + + rowEditDialogState.bulkInsertConflictSelect.addEventListener( + "change", + function () { + rowEditDialogState.bulkInsertConflictMode = + rowEditDialogState.bulkInsertConflictSelect.value; + syncBulkInsertConflictUi(rowEditDialogState); + resetBulkInsertPreview(rowEditDialogState); + syncBulkInsertTextareaValidation(rowEditDialogState); + updateRowEditDialogButtons(rowEditDialogState); + }, + ); + dialog.addEventListener("click", function (ev) { if (ev.target === dialog) { closeRowEditDialogIfConfirmed(rowEditDialogState); @@ -5557,8 +7414,17 @@ function ensureRowEditDialog(manager) { dialog.addEventListener("close", function () { var state = rowEditDialogState; + var shouldReloadOnClose = state.shouldReloadOnClose; + var redirectOnCloseUrl = state.redirectOnCloseUrl; state.loadId += 1; state.isClosePending = false; + state.bulkInsertLiveValidationError = null; + state.shouldReloadOnClose = false; + state.redirectOnCloseUrl = null; + clearTimeout(state.copyTemplateResetTimer); + state.copyTemplateResetTimer = null; + setBulkInsertCopyButtonReady(state); + resetBulkInsertPreview(state); clearRowEditDialogError(state); state.hasLoaded = false; destroyRowEditFields(state); @@ -5571,6 +7437,13 @@ function ensureRowEditDialog(manager) { ) { state.currentButton.focus(); } + if (shouldReloadOnClose) { + if (redirectOnCloseUrl) { + location.href = redirectOnCloseUrl; + } else { + location.reload(); + } + } }); return rowEditDialogState; @@ -5593,8 +7466,10 @@ async function openRowEditDialog(button, manager) { state.currentRowId = row.getAttribute("data-row") || ""; state.currentPkPath = rowDisplayLabel(row); state.currentInsertUrl = null; + state.currentUpsertUrl = null; state.currentUpdateUrl = rowUpdateUrl(row); state.currentFragmentUrl = rowFragmentUrl(row); + state.insertMode = "single"; if (state.currentUpdateUrl) { state.form.action = new URL( state.currentUpdateUrl, @@ -5620,6 +7495,7 @@ async function openRowEditDialog(button, manager) { ); state.summary.hidden = true; state.summary.textContent = ""; + syncRowEditInsertModeUi(state); if (!state.dialog.open) { state.dialog.showModal(); @@ -5668,8 +7544,16 @@ function openRowInsertDialog(button, manager) { state.currentRowId = null; state.currentPkPath = null; state.currentInsertUrl = tableInsertUrl(); + state.currentUpsertUrl = tableUpsertUrl(); state.currentUpdateUrl = null; state.currentFragmentUrl = null; + state.insertMode = "single"; + state.bulkInsertConflictMode = "ignore"; + state.bulkInsertLiveValidationError = null; + state.bulkInsertTextarea.value = ""; + state.shouldReloadOnClose = false; + state.redirectOnCloseUrl = null; + resetBulkInsertPreview(state); state.shouldRestoreFocus = true; state.hasLoaded = false; state.loadId += 1; @@ -5687,14 +7571,10 @@ function openRowInsertDialog(button, manager) { setRowEditDialogLoading(state, false); destroyRowEditFields(state); state.dialog.removeAttribute("aria-describedby"); - setRowDialogTitle( - state.title, - insertData.tableName - ? "Insert row into " + insertData.tableName - : "Insert row", - ); + setRowInsertDialogTitle(state); state.summary.hidden = true; state.summary.textContent = ""; + syncRowEditInsertModeUi(state); if (!state.dialog.open) { state.dialog.showModal(); diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py index 0a3a947c..1be28982 100644 --- a/datasette/utils/sql_analysis.py +++ b/datasette/utils/sql_analysis.py @@ -150,7 +150,6 @@ _SQLITE_INTERNAL_SCHEMA_FUNCTIONS = { "sqlite_rename_test", "substr", } - _AUTHORIZER_ACTION_NAMES = { getattr(sqlite3, name): name for name in ( @@ -391,6 +390,10 @@ def analyze_sql_tables( ) return sqlite3.SQLITE_OK + if action == sqlite3.SQLITE_RECURSIVE: + # Recursive CTE bookkeeping; table reads are reported separately. + return sqlite3.SQLITE_OK + if action == sqlite3.SQLITE_FUNCTION and arg2 is not None: record( "function", @@ -485,17 +488,17 @@ def analyze_sql_tables( and key.operation in {"create", "alter", "drop"} for key in operations ) - dropped_tables = { + dropped_tables_and_views = { (key.database, key.table) for key in operations - if key.operation == "drop" and key.target_type == "table" + if key.operation == "drop" and key.target_type in {"table", "view"} } def key_is_drop_table_delete(key: OperationKey) -> bool: return ( key.operation == "delete" and key.target_type == "table" - and (key.database, key.table) in dropped_tables + and (key.database, key.table) in dropped_tables_and_views ) has_user_table_access_in_schema_operation = any( diff --git a/datasette/views/database.py b/datasette/views/database.py index e02de657..c6ea57b8 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -325,7 +325,7 @@ class DatabaseContext(Context): database_color: str = field(metadata={"help": "The color assigned to the database"}) database_page_data: dict = field( metadata={ - "help": 'JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions`` and optional ``customColumnTypes``.' + "help": 'JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions``, ``canInsertRows`` and optional ``customColumnTypes``.' } ) database_actions: callable = field( diff --git a/datasette/views/table.py b/datasette/views/table.py index 7899877d..73652f05 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -208,7 +208,7 @@ class TableContext(Context): ) table_insert_ui: dict = field( metadata={ - "help": "Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns`` and ``primaryKeys`` keys; each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``value_kind`` and ``column_type`` keys." + "help": "Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns``, ``bulkColumns``, ``primaryKeys`` and ``maxInsertRows`` keys, plus optional ``upsertPath`` if the current actor has permission to update rows. ``columns`` lists columns for the single-row insert form, while ``bulkColumns`` lists columns for the bulk insert form. Each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``is_auto_pk``, ``value_kind`` and ``column_type`` keys." } ) table_alter_ui: dict = field( @@ -483,8 +483,15 @@ async def _table_insert_ui( ): return None + can_update = await datasette.allowed( + action="update-row", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ) + column_types_map = await datasette.get_column_types(database_name, table_name) columns = [] + bulk_columns = [] column_details = await db.table_column_details(table_name) for column in column_details: if column.hidden: @@ -495,32 +502,40 @@ async def _table_insert_ui( and len(pks) == 1 and SQLiteType.from_declared_type(column.type) == SQLiteType.INTEGER ) + column_type = column_types_map.get(column.name) + column_data = { + "name": column.name, + "sqlite_type": _column_sqlite_type_for_insert_form(column), + "notnull": column.notnull, + "default": column.default_value, + "has_default": column.default_value is not None, + "is_pk": is_pk, + "is_auto_pk": is_auto_pk, + "value_kind": _column_value_kind_for_insert_form(column), + "column_type": ( + {"type": column_type.name, "config": column_type.config} + if column_type is not None + else None + ), + } + bulk_columns.append(column_data) if is_auto_pk: continue - column_type = column_types_map.get(column.name) - columns.append( - { - "name": column.name, - "sqlite_type": _column_sqlite_type_for_insert_form(column), - "notnull": column.notnull, - "default": column.default_value, - "has_default": column.default_value is not None, - "is_pk": is_pk, - "value_kind": _column_value_kind_for_insert_form(column), - "column_type": ( - {"type": column_type.name, "config": column_type.config} - if column_type is not None - else None - ), - } - ) + columns.append(column_data) - return { + data = { "path": "{}/-/insert".format(datasette.urls.table(database_name, table_name)), "tableName": table_name, "columns": columns, + "bulkColumns": bulk_columns, "primaryKeys": pks, + "maxInsertRows": datasette.setting("max_insert_rows"), } + if can_update: + data["upsertPath"] = "{}/-/upsert".format( + datasette.urls.table(database_name, table_name) + ) + return data async def _table_alter_ui( diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index 5feb1fa1..12b8e9f9 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -269,6 +269,11 @@ async def _create_table_ui_context( "databaseName": database_name, "columnTypes": CREATE_TABLE_COLUMN_TYPES, "defaultExpressions": default_expression_options(), + "canInsertRows": await datasette.allowed( + action="insert-row", + resource=DatabaseResource(database=database_name), + actor=request.actor, + ), } can_set_column_type = await datasette.allowed( action="set-column-type", diff --git a/datasette/write_sql.py b/datasette/write_sql.py index cdc0c6d3..ca144a72 100644 --- a/datasette/write_sql.py +++ b/datasette/write_sql.py @@ -138,6 +138,33 @@ def decision_for_write_sql_operation( ), ) ) + if operation.operation == "create" and operation.target_type == "view": + if operation.database is None: + return UnsupportedWriteSqlOperation(unsupported_message) + return RequireWriteSqlPermissions( + ( + PermissionRequirement( + action="create-view", + resource=DatabaseResource(database=operation.database), + ), + ) + ) + if ( + operation.operation == "drop" + and operation.target_type == "view" + and operation.database is not None + and operation.table is not None + ): + return RequireWriteSqlPermissions( + ( + PermissionRequirement( + action="drop-view", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), + ) + ) if ( operation.operation == "alter" and operation.target_type == "table" diff --git a/docs/authentication.rst b/docs/authentication.rst index 8101699c..304b589f 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -50,7 +50,7 @@ The one exception is the "root" account, which you can sign into while using Dat The ``--root`` flag is designed for local development and testing. When you start Datasette with ``--root``, the root user automatically receives every permission, including: * All view permissions (``view-instance``, ``view-database``, ``view-table``, etc.) -* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``alter-table``, ``set-column-type``, ``drop-table``) +* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``create-view``, ``alter-table``, ``set-column-type``, ``drop-table``, ``drop-view``) * Debug permissions (``permissions-debug``, ``debug-menu``) * Any custom permissions defined by plugins @@ -1386,6 +1386,16 @@ create-table Actor is allowed to create a database table. +``resource`` - ``datasette.resources.DatabaseResource(database)`` + ``database`` is the name of the database (string) + +.. _actions_create_view: + +create-view +----------- + +Actor is allowed to create a database view. + ``resource`` - ``datasette.resources.DatabaseResource(database)`` ``database`` is the name of the database (string) @@ -1425,6 +1435,18 @@ Actor is allowed to drop a database table. ``table`` is the name of the table (string) +.. _actions_drop_view: + +drop-view +--------- + +Actor is allowed to drop a database view. + +``resource`` - ``datasette.resources.TableResource(database, table)`` + ``database`` is the name of the database (string) + + ``table`` is the name of the view (string) + .. _actions_execute_sql: execute-sql diff --git a/docs/changelog.rst b/docs/changelog.rst index 0cf1ec0b..4e8f33d8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,11 +4,14 @@ Changelog ========= -.. _v1_0_a36: +.. _unreleased: -1.0a36 (in development) ------------------------ +Unreleased +---------- +- Table pages now offer an "Insert multiple rows" mode in the row insertion dialog. This accepts pasted TSV, CSV or JSON, previews the parsed rows before inserting them, validates unknown columns as data is pasted and displays omitted auto integer primary keys as ``auto`` in the preview. (:pr:`2813`) +- The bulk insert UI can skip rows with existing primary keys, or update existing rows and insert new rows using the existing ``///-/upsert`` API when the actor has both :ref:`insert-row ` and :ref:`update-row ` permissions. (:pr:`2813`) +- The "Create table" dialog now includes a "Create table from data" mode. Paste TSV, CSV or JSON rows to preview inferred columns and types, choose the table name and primary key, then create the table and insert those rows in one step. (:pr:`2813`) - Datasette's JSON APIs now consistently encode every ``BLOB`` value using the documented :ref:`binary value JSON format `, even when the bytes could be decoded as UTF-8 text. - The insert and edit row dialogs now provide a dedicated control for ``BLOB`` values. Existing binary values are shown by byte size, image values under 10MB are previewed as thumbnails, and replacements can be attached, dropped or pasted into the control. - The table and row JSON APIs now support ``?_extra=column_details`` for returning SQLite schema details for columns, including declared type, SQLite affinity, primary key, ``NOT NULL``, default and hidden-column metadata. diff --git a/docs/template_context.rst b/docs/template_context.rst index 5c6b1567..e9447058 100644 --- a/docs/template_context.rst +++ b/docs/template_context.rst @@ -98,7 +98,7 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re The color assigned to the database ``database_page_data`` - ``dict`` - JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions`` and optional ``customColumnTypes``. + JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions``, ``canInsertRows`` and optional ``customColumnTypes``. ``editable`` - ``bool`` Boolean indicating if the database is editable @@ -389,7 +389,7 @@ Many of these keys are shared with the :ref:`JSON API ` for this page. SQL definition for this table ``table_insert_ui`` - ``dict`` - Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns`` and ``primaryKeys`` keys; each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``value_kind`` and ``column_type`` keys. + Information needed to enable the row insertion UI, or ``None`` if row insertion is not available to the current actor. When present it has ``path``, ``tableName``, ``columns``, ``bulkColumns``, ``primaryKeys`` and ``maxInsertRows`` keys, plus optional ``upsertPath`` if the current actor has permission to update rows. ``columns`` lists columns for the single-row insert form, while ``bulkColumns`` lists columns for the bulk insert form. Each column includes ``name``, ``sqlite_type``, ``notnull``, ``default``, ``has_default``, ``is_pk``, ``is_auto_pk``, ``value_kind`` and ``column_type`` keys. ``table_page_data`` - ``dict`` JSON data used by JavaScript on the table page. Includes ``database``, ``table`` and ``tableUrl``, plus optional ``foreignKeys`` mapping column names to autocomplete URLs, optional ``insertRow`` data and optional ``alterTable`` data. diff --git a/tests/test_playwright.py b/tests/test_playwright.py index 63e06b29..15c642fe 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -22,7 +22,7 @@ def find_free_port(): return sock.getsockname()[1] -def wait_for_server(process, url, timeout=10): +def wait_for_server(process, url, timeout=30): deadline = time.monotonic() + timeout last_error = None while time.monotonic() < deadline: @@ -41,7 +41,20 @@ def wait_for_server(process, url, timeout=10): except httpx.HTTPError as ex: last_error = repr(ex) time.sleep(0.1) - raise AssertionError(f"Timed out waiting for {url}: {last_error}") + if process.poll() is None: + process.terminate() + try: + stdout, stderr = process.communicate(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + stdout, stderr = process.communicate() + else: + stdout, stderr = process.communicate() + raise AssertionError( + f"Timed out waiting for {url}: {last_error}\n" + f"stdout:\n{stdout}\n" + f"stderr:\n{stderr}" + ) @pytest.fixture @@ -112,6 +125,17 @@ def write_playwright_database(db_path): name text not null, data blob ); + create table bulk_defaults ( + id integer primary key, + title text not null, + status text not null default 'todo', + score integer default 5 + ); + create table upsert_items ( + id text primary key, + title text, + metadata text + ); insert into projects (title, metadata, logo, notes, score) values ( 'Build Datasette', @@ -120,6 +144,8 @@ def write_playwright_database(db_path): 'Initial notes', 5 ); + insert into upsert_items (id, title, metadata) values + ('existing', 'Existing title', '{"old": true}'); """) conn.execute( "insert into binary_files (name, data) values (?, ?)", @@ -146,6 +172,7 @@ def write_playwright_config(config_path): "data": { "permissions": { "create-table": True, + "insert-row": True, "set-column-type": True, }, "tables": { @@ -176,6 +203,17 @@ def write_playwright_config(config_path): "delete-row": True, }, }, + "bulk_defaults": { + "permissions": { + "insert-row": True, + }, + }, + "upsert_items": { + "permissions": { + "insert-row": True, + "update-row": True, + }, + }, }, }, }, @@ -318,6 +356,41 @@ def binary_file_blob(datasette_server, pk): return response.content +def bulk_default_rows(datasette_server, **filters): + params = { + "_shape": "objects", + **{key: str(value) for key, value in filters.items()}, + } + response = httpx.get(f"{datasette_server}data/bulk_defaults.json", params=params) + response.raise_for_status() + return response.json()["rows"] + + +def upsert_item_rows(datasette_server, **filters): + params = { + "_shape": "objects", + **{key: str(value) for key, value in filters.items()}, + } + response = httpx.get(f"{datasette_server}data/upsert_items.json", params=params) + response.raise_for_status() + return response.json()["rows"] + + +def upsert_item_row(datasette_server, pk): + rows = upsert_item_rows(datasette_server, id=pk) + assert len(rows) == 1 + return rows[0] + + +def open_bulk_insert_dialog(page, url): + page.goto(url) + page.locator('button[data-table-action="insert-row"]').click() + dialog = page.locator("#row-edit-dialog") + dialog.wait_for() + dialog.locator(".row-edit-bulk-insert").click() + return dialog + + def open_jump_menu(page): page.keyboard.press("/") page.locator("navigation-search .search-input").wait_for() @@ -421,6 +494,165 @@ def test_create_table_flow(page, datasette_server): assert "NOT NULL DEFAULT 'Untitled'" in schema +@pytest.mark.playwright +def test_create_table_from_data_flow(page, datasette_server): + page.goto(f"{datasette_server}data") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-database-action="create-table"]').click() + + dialog = page.locator("#table-create-dialog") + dialog.wait_for() + from_data = dialog.locator(".table-create-from-data") + assert from_data.inner_text() == "Create table from data" + from_data.click() + + assert dialog.locator(".table-create-columns").is_hidden() + assert dialog.locator(".table-create-data").is_visible() + assert dialog.locator(".table-create-save").inner_text() == "Preview rows" + assert ( + dialog.locator(".table-create-data-note").inner_text() + == "Paste TSV, CSV, or JSON. You can also open a file or drop it onto this textarea" + ) + assert dialog.locator(".table-create-data-open-file").inner_text() == "open a file" + assert ( + dialog.locator(".table-create-data-editor").evaluate( + """node => Array.from(node.children) + .filter((child) => !child.hidden) + .map((child) => child.className) + .join(" ")""" + ) + == ("table-create-data-note table-create-input table-create-data-textarea") + ) + assert dialog.locator(".table-create-manual").inner_text() == ( + "Create table manually" + ) + assert ( + dialog.get_by_label("Paste TSV, CSV, or JSON").get_attribute("id") + == "table-create-data-textarea" + ) + + textarea = dialog.locator(".table-create-data-textarea") + dropped_value = textarea.evaluate("""node => new Promise((resolve) => { + node.addEventListener("input", () => resolve(node.value), { once: true }); + const file = new File(["short_id,name\\nx,Ada"], "Repo Export 2026!!.CSV", { type: "text/csv" }); + const dataTransfer = new DataTransfer(); + dataTransfer.items.add(file); + node.dispatchEvent(new DragEvent("dragenter", { bubbles: true, cancelable: true, dataTransfer })); + node.dispatchEvent(new DragEvent("dragover", { bubbles: true, cancelable: true, dataTransfer })); + node.dispatchEvent(new DragEvent("drop", { bubbles: true, cancelable: true, dataTransfer })); + })""") + assert dropped_value == "short_id,name\nx,Ada" + assert dialog.locator(".table-create-table-name").input_value() == ( + "repo_export_2026" + ) + + dialog.locator(".table-create-table-name").fill("playwright_from_data") + textarea.fill( + json.dumps( + { + "metadata": [{"short_id": "a", "name": "Ignored", "score": 9}], + "people": [ + {"short_id": "b", "name": "Ada", "score": 1}, + {"short_id": "c", "name": "Bob", "score": 2.5}, + ], + } + ) + ) + dialog.locator(".table-create-save").click() + + assert dialog.locator(".table-create-data-textarea").is_hidden() + assert dialog.locator(".table-create-save").inner_text() == "Create table" + assert dialog.locator(".table-create-cancel").inner_text() == "Back" + assert dialog.locator(".table-create-data-preview-summary").inner_text() == ( + "Previewing 2 rows." + ) + assert dialog.locator(".table-create-data-primary-key").input_value() == "short_id" + preview_text = dialog.locator(".table-create-data-preview-table").inner_text() + assert "short_id" in preview_text + assert "Ada" in preview_text + assert "2.5" in preview_text + preview_cell_style = dialog.locator( + ".table-create-data-preview-table td" + ).first.evaluate( + """node => ({ + overflowWrap: getComputedStyle(node).overflowWrap, + whiteSpace: getComputedStyle(node).whiteSpace + })""" + ) + assert preview_cell_style == { + "overflowWrap": "anywhere", + "whiteSpace": "normal", + } + + dialog.locator(".table-create-cancel").click() + assert dialog.evaluate("node => node.open") + assert dialog.locator(".table-create-data-textarea").is_visible() + assert '"people"' in dialog.locator(".table-create-data-textarea").input_value() + assert dialog.locator(".table-create-save").inner_text() == "Preview rows" + + dialog.locator(".table-create-save").click() + assert dialog.locator(".table-create-save").inner_text() == "Create table" + dialog.locator(".table-create-save").click() + page.wait_for_url("**/data/playwright_from_data") + + response = httpx.get( + f"{datasette_server}data/playwright_from_data.json?_shape=objects" + ) + response.raise_for_status() + data = response.json() + assert data["rows"] == [ + {"short_id": "b", "name": "Ada", "score": 1}, + {"short_id": "c", "name": "Bob", "score": 2.5}, + ] + + +@pytest.mark.playwright +def test_create_table_from_csv_keeps_numeric_type_when_values_are_blank( + page, datasette_server +): + page.goto(f"{datasette_server}data") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-database-action="create-table"]').click() + + dialog = page.locator("#table-create-dialog") + dialog.wait_for() + dialog.locator(".table-create-from-data").click() + dialog.locator(".table-create-table-name").fill("playwright_numeric_blanks") + dialog.locator(".table-create-data-textarea").fill("name,score\nA,1\nB,") + dialog.locator(".table-create-save").click() + + assert dialog.locator(".table-create-save").inner_text() == "Create table" + preview_text = dialog.locator(".table-create-data-preview-table").inner_text() + assert "A" in preview_text + assert "1" in preview_text + assert "B" in preview_text + assert "null" in preview_text + + dialog.locator(".table-create-save").click() + page.wait_for_url("**/data/playwright_numeric_blanks") + + response = httpx.get( + f"{datasette_server}data/playwright_numeric_blanks.json?_shape=objects" + ) + response.raise_for_status() + assert response.json()["rows"] == [ + {"name": "A", "score": 1}, + {"name": "B", "score": None}, + ] + + schema_response = httpx.get( + f"{datasette_server}data/-/query.json", + params={ + "sql": ( + "select type from pragma_table_info('playwright_numeric_blanks') " + "where name = 'score'" + ) + }, + ) + schema_response.raise_for_status() + assert schema_response.json()["rows"] == [{"type": "INTEGER"}] + + @pytest.mark.playwright def test_create_table_foreign_key_selection_updates_column_type(page, datasette_server): page.goto(f"{datasette_server}data") @@ -841,11 +1073,128 @@ def test_navigation_search_renders_jump_sections_from_javascript_plugins( @pytest.mark.playwright def test_insert_row_flow_uses_custom_column_field(page, datasette_server): + page.add_init_script(""" + (() => { + let clipboardText = ""; + Object.defineProperty(navigator, "clipboard", { + configurable: true, + get: () => ({ + writeText: async (text) => { + clipboardText = String(text); + }, + readText: async () => clipboardText, + }), + }); + })(); + """) page.goto(f"{datasette_server}data/projects") page.locator('button[data-table-action="insert-row"]').click() dialog = page.locator("#row-edit-dialog") dialog.wait_for() + bulk_insert = dialog.locator(".row-edit-bulk-insert") + bulk_insert.wait_for() + assert bulk_insert.inner_text() == "Insert multiple rows" + current_url = page.url + bulk_insert.click() + assert page.url == current_url + assert dialog.evaluate("node => node.open") + assert dialog.locator(".row-edit-fields").is_hidden() + assert dialog.locator(".row-edit-bulk").is_visible() + assert dialog.locator(".row-edit-save").inner_text() == "Preview rows" + assert ( + dialog.locator(".row-edit-bulk-note").inner_text() + == "Paste TSV, CSV, or JSON. You can also open a file or drop it onto this textarea" + ) + assert dialog.locator(".row-edit-bulk-open-file").inner_text() == "open a file" + assert ( + dialog.locator(".row-edit-bulk-editor").evaluate( + """node => Array.from(node.children) + .filter((child) => !child.hidden) + .map((child) => child.className) + .join(" ")""" + ) + == ( + "row-edit-bulk-note row-edit-input row-edit-bulk-textarea " + "row-edit-bulk-actions" + ) + ) + copy_template = dialog.locator(".row-edit-copy-template") + assert copy_template.inner_text() == "Copy spreadsheet template" + assert ( + dialog.get_by_label("Paste TSV, CSV, or JSON").get_attribute("id") + == "row-edit-bulk-textarea" + ) + assert dialog.locator(".row-edit-copy-template-label-narrow").text_content() == ( + "Copy template" + ) + assert dialog.locator(".row-edit-bulk-template-note").inner_text() == ( + "You can paste the template into Google Sheets or Excel." + ) + assert dialog.locator(".row-edit-bulk-template-note-narrow").text_content() == ( + "Paste into Google Sheets or Excel" + ) + copy_template.click() + page.wait_for_function( + """() => document.querySelector(".row-edit-copy-template").textContent === "Copied" """ + ) + assert page.evaluate("navigator.clipboard.readText()") == ( + "title\tmetadata\tlogo\tnotes\tscore" + ) + textarea = dialog.locator(".row-edit-bulk-textarea") + textarea.fill("title\tmetadata\nFrom TSV\t{}") + assert textarea.input_value() == "title\tmetadata\nFrom TSV\t{}" + dropped_value = textarea.evaluate("""node => new Promise((resolve) => { + node.addEventListener("input", () => resolve(node.value), { once: true }); + const file = new File(["\\n\\ntitle,metadata\\nFrom CSV,{}\\n,\\n , \\n"], "rows.csv", { type: "text/csv" }); + const dataTransfer = new DataTransfer(); + dataTransfer.items.add(file); + node.dispatchEvent(new DragEvent("dragenter", { bubbles: true, cancelable: true, dataTransfer })); + node.dispatchEvent(new DragEvent("dragover", { bubbles: true, cancelable: true, dataTransfer })); + node.dispatchEvent(new DragEvent("drop", { bubbles: true, cancelable: true, dataTransfer })); + })""") + assert dropped_value == "\n\ntitle,metadata\nFrom CSV,{}\n,\n , \n" + dialog.locator(".row-edit-save").click() + assert dialog.evaluate("node => node.open") + assert dialog.locator(".row-edit-bulk-textarea").is_hidden() + assert dialog.locator(".row-edit-save").inner_text() == "Insert these rows" + assert dialog.locator(".row-edit-bulk-preview-summary").inner_text() == ( + "Previewing 1 row." + ) + preview_text = dialog.locator(".row-edit-bulk-preview-table").inner_text() + assert "id" in preview_text + assert "title" in preview_text + assert "metadata" in preview_text + assert "From CSV" in preview_text + assert dialog.locator(".row-edit-bulk-preview-auto").first.text_content() == "auto" + assert "null" not in preview_text + assert "undefined" not in preview_text + preview_cell_style = dialog.locator( + ".row-edit-bulk-preview-table td" + ).first.evaluate( + """node => ({ + overflowWrap: getComputedStyle(node).overflowWrap, + whiteSpace: getComputedStyle(node).whiteSpace + })""" + ) + assert preview_cell_style == { + "overflowWrap": "anywhere", + "whiteSpace": "normal", + } + assert dialog.locator(".row-edit-cancel").inner_text() == "Back" + dialog.locator(".row-edit-cancel").click() + assert dialog.evaluate("node => node.open") + assert dialog.locator(".row-edit-bulk-textarea").is_visible() + assert dialog.locator(".row-edit-bulk-textarea").input_value() == ( + "\n\ntitle,metadata\nFrom CSV,{}\n,\n , \n" + ) + assert dialog.locator(".row-edit-save").inner_text() == "Preview rows" + single_insert = dialog.locator(".row-edit-single-insert") + assert single_insert.inner_text() == "Insert single row" + single_insert.click() + assert dialog.locator(".row-edit-bulk").is_hidden() + assert dialog.locator(".row-edit-fields").is_visible() + assert dialog.locator(".row-edit-save").inner_text() == "Insert row" dialog.locator('input[name="title"]').fill("Launch Datasette Cloud") dialog.locator('textarea[name="metadata"]').fill( '{"ok": false, "source": "playwright"}' @@ -875,6 +1224,206 @@ def test_insert_row_flow_uses_custom_column_field(page, datasette_server): assert data["score"] == 5 +@pytest.mark.playwright +def test_bulk_insert_preview_inserts_rows(page, datasette_server): + page.goto(f"{datasette_server}data/projects") + page.locator('button[data-table-action="insert-row"]').click() + + dialog = page.locator("#row-edit-dialog") + dialog.wait_for() + dialog.locator(".row-edit-bulk-insert").click() + dialog.locator(".row-edit-bulk-textarea").fill( + json.dumps( + { + "metadata": [{"title": "Ignored", "metadata": "{}"}], + "projects": [ + {"title": "Bulk one", "metadata": "{}"}, + {"title": "Bulk two", "metadata": "{}"}, + ], + } + ) + ) + dialog.locator(".row-edit-save").click() + assert dialog.locator(".row-edit-save").inner_text() == "Insert these rows" + dialog.locator(".row-edit-save").click() + dialog.locator( + ".row-edit-bulk-progress-status", has_text="2 rows inserted." + ).wait_for() + assert dialog.locator(".row-edit-cancel").inner_text() == "Close and view table" + dialog.locator(".row-edit-cancel").click() + page.wait_for_load_state("domcontentloaded") + assert page.url == f"{datasette_server}data/projects" + + assert project_rows(datasette_server, title="Bulk one") + assert project_rows(datasette_server, title="Bulk two") + + +@pytest.mark.playwright +def test_bulk_insert_upsert_option_updates_existing_and_inserts_new( + page, datasette_server +): + dialog = open_bulk_insert_dialog(page, f"{datasette_server}data/upsert_items") + textarea = dialog.locator(".row-edit-bulk-textarea") + conflict_field = dialog.locator(".row-edit-bulk-conflict") + conflict_select = dialog.locator(".row-edit-bulk-conflict-mode") + + assert conflict_field.is_hidden() + textarea.fill("title\nNo primary key") + assert conflict_field.is_hidden() + + textarea.fill( + "id,title,metadata\nexisting,Updated by upsert,{}\nnew,Inserted by upsert,{}" + ) + assert conflict_field.is_visible() + assert ( + dialog.locator(".row-edit-bulk-conflict-label").inner_text() + == "If the row exists already" + ) + assert conflict_select.input_value() == "ignore" + assert ( + conflict_select.evaluate("""node => Array.from(node.options) + .filter((option) => !option.hidden) + .map((option) => option.textContent.trim())""") + == [ + "Stop with an error", + "Skip existing rows", + "Update existing and insert new", + ] + ) + + conflict_select.select_option("upsert") + assert conflict_select.input_value() == "upsert" + dialog.locator(".row-edit-save").click() + + assert dialog.locator(".row-edit-save").inner_text() == "Update or insert rows" + preview_text = dialog.locator(".row-edit-bulk-preview-table").inner_text() + assert "Updated by upsert" in preview_text + assert "Inserted by upsert" in preview_text + + dialog.locator(".row-edit-save").click() + dialog.locator( + ".row-edit-bulk-progress-status", has_text="2 rows upserted." + ).wait_for() + + assert upsert_item_row(datasette_server, "existing")["title"] == ( + "Updated by upsert" + ) + assert upsert_item_row(datasette_server, "new")["title"] == "Inserted by upsert" + + +@pytest.mark.playwright +def test_bulk_insert_conflicts_hide_upsert_without_update_permission( + page, datasette_server +): + dialog = open_bulk_insert_dialog(page, f"{datasette_server}data/bulk_defaults") + textarea = dialog.locator(".row-edit-bulk-textarea") + conflict_field = dialog.locator(".row-edit-bulk-conflict") + conflict_select = dialog.locator(".row-edit-bulk-conflict-mode") + + textarea.fill("title\nOnly title") + assert conflict_field.is_hidden() + + textarea.fill("id,title\n1,Only title") + assert conflict_field.is_visible() + assert conflict_select.input_value() == "ignore" + assert ( + conflict_select.evaluate("""node => Array.from(node.options) + .filter((option) => !option.hidden) + .map((option) => option.textContent.trim())""") + == [ + "Stop with an error", + "Skip existing rows", + ] + ) + assert conflict_select.locator('option[value="upsert"]').evaluate( + "node => node.hidden && node.disabled" + ) + + +@pytest.mark.playwright +def test_bulk_insert_live_validation_reports_unknown_columns(page, datasette_server): + dialog = open_bulk_insert_dialog(page, f"{datasette_server}data/projects") + textarea = dialog.locator(".row-edit-bulk-textarea") + error = dialog.locator(".row-edit-error") + save = dialog.locator(".row-edit-save") + + textarea.fill(json.dumps([{"id2": 1, "title": "Unknown column"}])) + error.wait_for() + assert error.inner_text() == "JSON row 1 has unknown column id2." + assert save.is_disabled() + assert textarea.evaluate("node => document.activeElement === node") + + textarea.fill("id2,title\n1,Unknown column") + assert error.inner_text() == "Unknown column id2 in header row." + assert save.is_disabled() + + textarea.fill("[") + assert error.is_hidden() + assert not save.is_disabled() + + textarea.fill(json.dumps([{"id": 1, "title": "Known column"}])) + assert error.is_hidden() + assert not save.is_disabled() + assert dialog.locator(".row-edit-bulk-conflict").is_visible() + assert dialog.locator(".row-edit-bulk-conflict-mode").input_value() == "ignore" + + +@pytest.mark.playwright +def test_bulk_insert_omits_columns_absent_from_pasted_input(page, datasette_server): + page.goto(f"{datasette_server}data/bulk_defaults") + page.locator('button[data-table-action="insert-row"]').click() + + dialog = page.locator("#row-edit-dialog") + dialog.wait_for() + dialog.locator(".row-edit-bulk-insert").click() + dialog.locator(".row-edit-bulk-textarea").fill("title\nOnly title") + dialog.locator(".row-edit-save").click() + + assert dialog.locator(".row-edit-save").inner_text() == "Insert these rows" + preview_text = dialog.locator(".row-edit-bulk-preview-table").inner_text() + assert "Only title" in preview_text + assert "undefined" not in preview_text + assert dialog.locator(".row-edit-bulk-preview-auto").inner_text() == "auto" + + dialog.locator(".row-edit-save").click() + dialog.locator( + ".row-edit-bulk-progress-status", has_text="1 row inserted." + ).wait_for() + + rows = bulk_default_rows(datasette_server, title="Only title") + assert rows == [ + { + "id": 1, + "title": "Only title", + "status": "todo", + "score": 5, + } + ] + + +@pytest.mark.playwright +def test_bulk_insert_preview_accepts_single_column_input(page, datasette_server): + page.goto(f"{datasette_server}data/projects") + page.locator('button[data-table-action="insert-row"]').click() + + dialog = page.locator("#row-edit-dialog") + dialog.wait_for() + dialog.locator(".row-edit-bulk-insert").click() + dialog.locator(".row-edit-bulk-textarea").fill("title\none\ntwo\nthree") + dialog.locator(".row-edit-save").click() + + assert dialog.locator(".row-edit-save").inner_text() == "Insert these rows" + assert dialog.locator(".row-edit-bulk-preview-summary").inner_text() == ( + "Previewing 3 rows." + ) + preview_text = dialog.locator(".row-edit-bulk-preview-table").inner_text() + assert "one" in preview_text + assert "two" in preview_text + assert "three" in preview_text + assert "null" not in preview_text + assert "undefined" not in preview_text + + @pytest.mark.playwright def test_edit_row_flow_validates_json_and_saves_changes(page, datasette_server): page.goto(f"{datasette_server}data/projects") @@ -882,6 +1431,9 @@ def test_edit_row_flow_validates_json_and_saves_changes(page, datasette_server): dialog = page.locator("#row-edit-dialog") dialog.wait_for() + assert dialog.locator(".row-edit-bulk-insert").is_hidden() + assert dialog.locator(".row-edit-single-insert").is_hidden() + assert dialog.locator(".row-edit-bulk").is_hidden() title = dialog.locator('input[name="title"]') title.wait_for() title.fill("Build Datasette, edited") diff --git a/tests/test_queries.py b/tests/test_queries.py index b757f221..6d4275a3 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1585,6 +1585,66 @@ async def test_create_query_analyze_endpoint_uses_sql_only(): assert old_analyze_response.status_code == 404 +@pytest.mark.asyncio +async def test_create_query_supports_recursive_cte(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("query_create_recursive_cte", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + sql = """ + with recursive dog_tree(id, name) as ( + select id, name from dogs + union all + select id + 1, name from dog_tree where id < 3 + ) + select name from dog_tree + """.strip() + + analysis_response = await ds.client.get( + "/data/-/queries/analyze", + actor={"id": "root"}, + params={"sql": sql}, + ) + form_response = await ds.client.get( + "/data/-/queries/store", + actor={"id": "root"}, + params={"sql": sql}, + ) + store_response = await ds.client.post( + "/data/-/queries/store", + actor={"id": "root"}, + data={ + "name": "dog-tree", + "title": "Dog tree", + "sql": sql, + "is_private": "1", + }, + ) + + assert analysis_response.status_code == 200 + analysis_data = analysis_response.json() + assert analysis_data["ok"] is True + assert analysis_data["analysis_error"] is None + assert analysis_data["analysis_is_write"] is False + assert analysis_data["save_disabled"] is False + + assert form_response.status_code == 200 + soup = Soup(form_response.text, "html.parser") + submit = soup.select_one("[data-query-create-submit]") + assert submit is not None + assert not submit.has_attr("disabled") + assert "This is a read-only query." in form_response.text + + assert store_response.status_code == 302 + assert store_response.headers["location"] == "/data/dog-tree" + query = await ds.get_query("data", "dog-tree") + assert query is not None + assert query.sql == sql + assert query.is_write is False + + @pytest.mark.asyncio async def test_create_query_form_error_redisplays_form_with_values(): ds = Datasette(memory=True, default_deny=True) @@ -3154,6 +3214,74 @@ async def test_execute_write_create_table_uses_create_table_permission(): assert not await db.table_exists("should_not_exist") +@pytest.mark.asyncio +async def test_execute_write_create_view_uses_create_view_permission(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "permissions": { + "insert-row": {"id": "row-writer"}, + "update-row": {"id": "row-writer"}, + }, + "databases": { + "data": { + "permissions": { + "view-database": {"id": ["creator", "row-writer"]}, + "execute-write-sql": {"id": ["creator", "row-writer"]}, + "create-view": {"id": "creator"}, + } + } + }, + }, + ) + db = ds.add_memory_database("execute_write_create_view", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + analysis_response = await ds.client.get( + "/data/-/execute-write/analyze", + actor={"id": "creator"}, + params={"sql": "create view dog_names as select id, name from dogs"}, + ) + allowed_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "creator"}, + json={"sql": "create view dog_names as select id, name from dogs"}, + ) + row_permission_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "row-writer"}, + json={"sql": "create view should_not_exist as select id from dogs"}, + ) + + assert analysis_response.status_code == 200 + analysis_data = analysis_response.json() + assert analysis_data["ok"] is True + assert analysis_data["execute_disabled"] is False + assert analysis_data["analysis_rows"] == [ + { + "operation": "create", + "database": "data", + "table": "dog_names", + "required_permission": "create-view", + "source": None, + "allowed": True, + } + ] + + assert allowed_response.status_code == 200 + assert allowed_response.json()["ok"] is True + assert allowed_response.json()["message"] == "Query executed" + assert await db.view_exists("dog_names") + + assert row_permission_response.status_code == 403 + assert row_permission_response.json()["errors"] == [ + "Permission denied: need create-view on data" + ] + assert not await db.view_exists("should_not_exist") + + @pytest.mark.parametrize( ( "database_name", @@ -3204,8 +3332,20 @@ async def test_execute_write_create_table_uses_create_table_permission(): (), "drop-table", ), + ( + "execute_write_drop_view", + "dropper", + "drop view dogs_view", + "drop view cats_view", + "Permission denied: need drop-view on data/cats_view", + ( + "create view dogs_view as select * from dogs", + "create view cats_view as select * from cats", + ), + "drop-view", + ), ), - ids=("alter-table", "create-index", "drop-index", "drop-table"), + ids=("alter-table", "create-index", "drop-index", "drop-table", "drop-view"), ) @pytest.mark.asyncio async def test_execute_write_schema_operations_use_schema_permissions( @@ -3240,7 +3380,12 @@ async def test_execute_write_schema_operations_use_schema_permissions( "drop-table": {"id": "dropper"}, "view-table": {"id": "alterer"}, } - } + }, + "dogs_view": { + "permissions": { + "drop-view": {"id": "dropper"}, + } + }, }, } }, @@ -3293,6 +3438,9 @@ async def test_execute_write_schema_operations_use_schema_permissions( elif expected_state == "drop-table": assert not await db.table_exists("dogs") assert await db.table_exists("cats") + elif expected_state == "drop-view": + assert not await db.view_exists("dogs_view") + assert await db.view_exists("cats_view") @pytest.mark.asyncio diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 3af2bb08..df1db1a3 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1027,6 +1027,7 @@ async def test_database_create_table_action_button_and_data(): "databaseName": "data", "columnTypes": ["text", "integer", "float", "blob"], "defaultExpressions": DEFAULT_EXPRESSION_OPTIONS, + "canInsertRows": False, }, } assert "customColumnTypes" not in database_data_from_soup(soup)["createTable"] @@ -1050,6 +1051,40 @@ async def test_database_create_table_action_button_and_data(): ds.close() +@pytest.mark.asyncio +async def test_database_create_table_data_includes_insert_row_permission(): + ds = Datasette( + [], + config={ + "databases": { + "data": { + "permissions": { + "create-table": {"id": "root"}, + "insert-row": {"id": "root"}, + }, + }, + }, + }, + ) + try: + db = ds.add_database( + Database(ds, memory_name="test_database_create_table_insert_permission"), + name="data", + ) + await db.execute_write_script(""" + create table items (id integer primary key, name text); + """) + + response = await ds.client.get("/data", actor={"id": "root"}) + assert response.status_code == 200 + create_table_data = database_data_from_soup(Soup(response.text, "html.parser"))[ + "createTable" + ] + assert create_table_data["canInsertRows"] is True + finally: + ds.close() + + @pytest.mark.asyncio async def test_database_create_table_data_includes_custom_column_types(): ds = Datasette( @@ -1316,6 +1351,7 @@ async def test_table_insert_action_button_and_data(): assert insert_data["path"] == "/data/items/-/insert" assert insert_data["tableName"] == "items" assert insert_data["primaryKeys"] == ["id"] + assert insert_data["maxInsertRows"] == 100 assert [column["name"] for column in insert_data["columns"]] == [ "name", "score",