mirror of
https://github.com/simonw/datasette.git
synced 2026-07-09 17:14:35 +02:00
Add bulk insert upsert controls
Expose conflict handling in the bulk insert UI when pasted rows include primary key columns, and route update-or-insert selections through the existing upsert API. Add live validation for bulk textarea column errors and Playwright coverage for upsert permissions, conflict options, and validation behavior. Refs shttps://github.com/simonw/datasette/pull/2813#issuecomment-4878320713
This commit is contained in:
Simon Willison
parent
ed6235e59a
commit
c864bc866d
4 changed files with 536 additions and 36 deletions
|
|
@ -1832,6 +1832,34 @@ textarea.row-edit-input {
|
|||
padding-right: 12px;
|
||||
}
|
||||
|
||||
.row-edit-bulk-conflict {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(120px, 180px) minmax(0, 1fr);
|
||||
gap: 8px 12px;
|
||||
align-items: start;
|
||||
}
|
||||
|
||||
.row-edit-bulk-conflict[hidden] {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.row-edit-bulk-conflict-label {
|
||||
color: var(--ink);
|
||||
font-size: 0.82rem;
|
||||
padding-top: 8px;
|
||||
}
|
||||
|
||||
.row-edit-bulk-conflict-control {
|
||||
display: grid;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.row-edit-bulk-conflict-help {
|
||||
color: var(--muted);
|
||||
font-size: 0.78rem;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.row-edit-copy-template-label-narrow {
|
||||
display: none;
|
||||
}
|
||||
|
|
@ -3343,6 +3371,15 @@ select.table-alter-input {
|
|||
padding-top: 0;
|
||||
}
|
||||
|
||||
.row-edit-bulk-conflict {
|
||||
grid-template-columns: 1fr;
|
||||
gap: 5px;
|
||||
}
|
||||
|
||||
.row-edit-bulk-conflict-label {
|
||||
padding-top: 0;
|
||||
}
|
||||
|
||||
.row-edit-dialog .modal-footer {
|
||||
padding-left: 18px;
|
||||
padding-right: 18px;
|
||||
|
|
|
|||
|
|
@ -4332,6 +4332,14 @@ function tableInsertUrl() {
|
|||
return url.toString();
|
||||
}
|
||||
|
||||
function tableUpsertUrl() {
|
||||
var data = tableInsertData();
|
||||
if (data && data.upsertPath) {
|
||||
return new URL(data.upsertPath, location.href).toString();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function rowResourceUrl(row) {
|
||||
var rowId = row.getAttribute("data-row");
|
||||
if (!rowId) {
|
||||
|
|
@ -5179,10 +5187,12 @@ function clearRowEditDialogError(state) {
|
|||
state.error.textContent = "";
|
||||
}
|
||||
|
||||
function showRowEditDialogError(state, message) {
|
||||
function showRowEditDialogError(state, message, options) {
|
||||
state.error.hidden = false;
|
||||
state.error.textContent = message;
|
||||
state.error.focus();
|
||||
if (!options || options.focus !== false) {
|
||||
state.error.focus();
|
||||
}
|
||||
}
|
||||
|
||||
function rowEditIsMultipleInsert(state) {
|
||||
|
|
@ -5205,7 +5215,10 @@ function updateRowEditDialogButtons(state) {
|
|||
state.isLoading ||
|
||||
state.isSaving ||
|
||||
!state.hasLoaded ||
|
||||
state.bulkInsertInserted;
|
||||
state.bulkInsertInserted ||
|
||||
(rowEditIsMultipleInsert(state) &&
|
||||
!state.bulkInsertPreviewReady &&
|
||||
!!state.bulkInsertLiveValidationError);
|
||||
state.cancelButton.disabled = state.isSaving;
|
||||
syncRowEditInsertModeUi(state);
|
||||
state.cancelButton.textContent =
|
||||
|
|
@ -5220,14 +5233,16 @@ function updateRowEditDialogButtons(state) {
|
|||
? state.bulkInsertInserted
|
||||
? "Inserted"
|
||||
: state.bulkInsertPreviewReady
|
||||
? "Insert these rows"
|
||||
? bulkInsertSaveLabel(state)
|
||||
: "Preview rows"
|
||||
: state.mode === "insert"
|
||||
? "Insert row"
|
||||
: "Save";
|
||||
state.saveButton.textContent = state.isSaving
|
||||
? rowEditIsMultipleInsert(state)
|
||||
? "Inserting..."
|
||||
? bulkInsertConflictMode(state) === "upsert"
|
||||
? "Updating..."
|
||||
: "Inserting..."
|
||||
: "Saving..."
|
||||
: saveLabel;
|
||||
state.form.setAttribute(
|
||||
|
|
@ -5484,6 +5499,8 @@ function showMultipleRowInsert(state) {
|
|||
}
|
||||
clearRowEditDialogError(state);
|
||||
setRowInsertDialogTitle(state);
|
||||
syncBulkInsertConflictUi(state);
|
||||
syncBulkInsertTextareaValidation(state);
|
||||
updateRowEditDialogButtons(state);
|
||||
state.bulkInsertTextarea.focus();
|
||||
}
|
||||
|
|
@ -5501,6 +5518,67 @@ function showSingleRowInsert(state) {
|
|||
}
|
||||
}
|
||||
|
||||
function bulkInsertConflictMode(state) {
|
||||
if (!state || !state.bulkInsertHasPrimaryKeyColumns) {
|
||||
return "insert";
|
||||
}
|
||||
return state.bulkInsertConflictMode || "ignore";
|
||||
}
|
||||
|
||||
function syncBulkInsertConflictUi(state) {
|
||||
if (!state || !state.bulkInsertConflictField) {
|
||||
return;
|
||||
}
|
||||
var insertData = tableInsertData() || {};
|
||||
var primaryKeys = insertData.primaryKeys || [];
|
||||
var hasPrimaryKeys = primaryKeys.length > 0;
|
||||
var hasPrimaryKeyColumns =
|
||||
hasPrimaryKeys &&
|
||||
bulkInsertTextIncludesPrimaryKeyColumns(
|
||||
state.bulkInsertTextarea.value,
|
||||
state.bulkInsertColumnDetails,
|
||||
primaryKeys,
|
||||
);
|
||||
state.bulkInsertHasPrimaryKeyColumns = hasPrimaryKeyColumns;
|
||||
var canUpsert = hasPrimaryKeyColumns && !!state.currentUpsertUrl;
|
||||
var upsertOption = state.bulkInsertConflictSelect.querySelector(
|
||||
'option[value="upsert"]',
|
||||
);
|
||||
if (upsertOption) {
|
||||
upsertOption.disabled = !canUpsert;
|
||||
upsertOption.hidden = !canUpsert;
|
||||
}
|
||||
if (
|
||||
!hasPrimaryKeyColumns ||
|
||||
(!canUpsert && bulkInsertConflictMode(state) === "upsert")
|
||||
) {
|
||||
state.bulkInsertConflictMode = hasPrimaryKeys ? "ignore" : "insert";
|
||||
state.bulkInsertConflictSelect.value = state.bulkInsertConflictMode;
|
||||
}
|
||||
state.bulkInsertConflictField.hidden = !hasPrimaryKeyColumns;
|
||||
state.bulkInsertConflictSelect.value = bulkInsertConflictMode(state);
|
||||
var helpText = "";
|
||||
if (bulkInsertConflictMode(state) === "upsert") {
|
||||
helpText =
|
||||
"Rows with existing primary keys will be updated; new primary keys will be inserted.";
|
||||
} else if (bulkInsertConflictMode(state) === "ignore") {
|
||||
helpText = "Rows with existing primary keys will be skipped.";
|
||||
} else {
|
||||
helpText = "Rows with existing primary keys will stop the import.";
|
||||
}
|
||||
state.bulkInsertConflictHelp.textContent = helpText;
|
||||
}
|
||||
|
||||
function bulkInsertSaveLabel(state) {
|
||||
if (!state.bulkInsertPreviewReady) {
|
||||
return "Preview rows";
|
||||
}
|
||||
if (bulkInsertConflictMode(state) === "upsert") {
|
||||
return "Update or insert rows";
|
||||
}
|
||||
return "Insert these rows";
|
||||
}
|
||||
|
||||
function readTextFile(file) {
|
||||
if (file.text) {
|
||||
return file.text();
|
||||
|
|
@ -5526,7 +5604,6 @@ async function loadBulkInsertTextFile(state, file) {
|
|||
state.bulkInsertTextarea.dispatchEvent(
|
||||
new Event("input", { bubbles: true }),
|
||||
);
|
||||
clearRowEditDialogError(state);
|
||||
state.bulkInsertTextarea.focus();
|
||||
} catch (_error) {
|
||||
showRowEditDialogError(state, "Could not read that text file.");
|
||||
|
|
@ -5588,6 +5665,7 @@ function resetBulkInsertPreview(state) {
|
|||
state.bulkInsertProgressBar.value = 0;
|
||||
state.bulkInsertProgressBar.max = 1;
|
||||
state.bulkInsertProgressStatus.textContent = "";
|
||||
syncBulkInsertConflictUi(state);
|
||||
syncRowEditInsertModeUi(state);
|
||||
}
|
||||
|
||||
|
|
@ -5617,7 +5695,10 @@ function rowObjectForBulkInsert(valuesByColumn, columns) {
|
|||
if (!hasValue) {
|
||||
return;
|
||||
}
|
||||
row[column.name] = normalizeBulkInsertCell(column, valuesByColumn[column.name]);
|
||||
row[column.name] = normalizeBulkInsertCell(
|
||||
column,
|
||||
valuesByColumn[column.name],
|
||||
);
|
||||
});
|
||||
return row;
|
||||
}
|
||||
|
|
@ -5734,6 +5815,118 @@ function bulkInsertColumnMap(columns) {
|
|||
return map;
|
||||
}
|
||||
|
||||
function bulkInsertTextIncludesPrimaryKeyColumns(text, columns, primaryKeys) {
|
||||
if (!primaryKeys.length || !text.trim()) {
|
||||
return false;
|
||||
}
|
||||
var trimmed = text.trim();
|
||||
try {
|
||||
if (trimmed[0] === "[" || trimmed[0] === "{") {
|
||||
return jsonBulkInsertTextIncludesPrimaryKeyColumns(trimmed, primaryKeys);
|
||||
}
|
||||
return delimitedBulkInsertTextIncludesPrimaryKeyColumns(
|
||||
trimmed,
|
||||
columns,
|
||||
primaryKeys,
|
||||
);
|
||||
} catch (_error) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function jsonBulkInsertTextIncludesPrimaryKeyColumns(text, primaryKeys) {
|
||||
var rows = parseJsonObjectRows(text);
|
||||
var seenKeys = {};
|
||||
rows.forEach(function (row) {
|
||||
Object.keys(row).forEach(function (key) {
|
||||
seenKeys[key] = true;
|
||||
});
|
||||
});
|
||||
return primaryKeys.every(function (key) {
|
||||
return !!seenKeys[key];
|
||||
});
|
||||
}
|
||||
|
||||
function delimitedBulkInsertTextIncludesPrimaryKeyColumns(
|
||||
text,
|
||||
columns,
|
||||
primaryKeys,
|
||||
) {
|
||||
var delimiter = detectBulkInsertDelimiter(text, columns);
|
||||
var rows = (
|
||||
delimiter === null
|
||||
? splitSingleColumnRows(text)
|
||||
: splitDelimitedRows(text, delimiter)
|
||||
).filter(function (row) {
|
||||
return !bulkInsertDelimitedRowIsBlank(row);
|
||||
});
|
||||
if (!rows.length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
var columnMap = bulkInsertColumnMap(columns);
|
||||
var header = rows[0].map(function (value) {
|
||||
return value.trim();
|
||||
});
|
||||
var headerMatches = header.filter(function (name) {
|
||||
return !!columnMap[name];
|
||||
}).length;
|
||||
if (headerMatches > 0) {
|
||||
return primaryKeys.every(function (key) {
|
||||
return header.indexOf(key) !== -1;
|
||||
});
|
||||
}
|
||||
|
||||
var headers = columns.map(function (column) {
|
||||
return column.name;
|
||||
});
|
||||
var suppliedColumnCount = rows.reduce(function (count, row) {
|
||||
return Math.max(count, row.length);
|
||||
}, 0);
|
||||
return primaryKeys.every(function (key) {
|
||||
var index = headers.indexOf(key);
|
||||
return index !== -1 && index < suppliedColumnCount;
|
||||
});
|
||||
}
|
||||
|
||||
function bulkInsertLiveValidationShouldWait(message) {
|
||||
return (
|
||||
message === "Paste rows before previewing." ||
|
||||
message === "No data rows found to preview." ||
|
||||
(message.indexOf("Invalid JSON:") === 0 &&
|
||||
message.indexOf("Unexpected end") !== -1)
|
||||
);
|
||||
}
|
||||
|
||||
function bulkInsertLiveValidationError(state) {
|
||||
var text = state.bulkInsertTextarea.value;
|
||||
if (!text.trim()) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
parseBulkInsertRows(text, state.bulkInsertColumnDetails);
|
||||
} catch (error) {
|
||||
var message = error.message || "Could not preview rows.";
|
||||
return bulkInsertLiveValidationShouldWait(message) ? null : message;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function syncBulkInsertTextareaValidation(state) {
|
||||
if (!rowEditIsMultipleInsert(state) || state.bulkInsertPreviewReady) {
|
||||
state.bulkInsertLiveValidationError = null;
|
||||
return;
|
||||
}
|
||||
state.bulkInsertLiveValidationError = bulkInsertLiveValidationError(state);
|
||||
if (state.bulkInsertLiveValidationError) {
|
||||
showRowEditDialogError(state, state.bulkInsertLiveValidationError, {
|
||||
focus: false,
|
||||
});
|
||||
} else {
|
||||
clearRowEditDialogError(state);
|
||||
}
|
||||
}
|
||||
|
||||
function parseJsonBulkInsertRows(text, columns) {
|
||||
var parsed = parseJsonObjectRows(text);
|
||||
|
||||
|
|
@ -5892,6 +6085,7 @@ function renderBulkInsertPreview(state, rows) {
|
|||
function previewBulkInsertRows(state) {
|
||||
clearRowEditDialogError(state);
|
||||
resetBulkInsertPreview(state);
|
||||
syncBulkInsertConflictUi(state);
|
||||
try {
|
||||
var rows = parseBulkInsertRows(
|
||||
state.bulkInsertTextarea.value,
|
||||
|
|
@ -5908,13 +6102,14 @@ function previewBulkInsertRows(state) {
|
|||
}
|
||||
|
||||
function updateBulkInsertProgress(state, inserted, total) {
|
||||
var words = bulkInsertProgressWords(state);
|
||||
state.bulkInsertProgress.hidden = false;
|
||||
state.bulkInsertProgressBar.max = total || 1;
|
||||
state.bulkInsertProgressBar.value = inserted;
|
||||
state.bulkInsertProgressStatus.textContent =
|
||||
inserted >= total
|
||||
? total + " row" + (total === 1 ? " inserted." : "s inserted.")
|
||||
: "Inserting " + inserted + " of " + total + " rows...";
|
||||
? total + " row" + (total === 1 ? " " : "s ") + words.complete + "."
|
||||
: words.active + " " + inserted + " of " + total + " rows...";
|
||||
}
|
||||
|
||||
function bulkInsertBatches(rows, batchSize) {
|
||||
|
|
@ -5944,11 +6139,12 @@ function animateBulkInsertProgress(state, from, to, total, duration) {
|
|||
var easedProgress = 1 - Math.pow(1 - progress, 3);
|
||||
var value = from + (to - from) * easedProgress;
|
||||
var displayValue = progress === 1 ? to : Math.floor(value);
|
||||
var words = bulkInsertProgressWords(state);
|
||||
state.bulkInsertProgressBar.value = value;
|
||||
state.bulkInsertProgressStatus.textContent =
|
||||
displayValue >= total
|
||||
? total + " row" + (total === 1 ? " inserted." : "s inserted.")
|
||||
: "Inserting " + displayValue + " of " + total + " rows...";
|
||||
? total + " row" + (total === 1 ? " " : "s ") + words.complete + "."
|
||||
: words.active + " " + displayValue + " of " + total + " rows...";
|
||||
if (progress < 1) {
|
||||
window.requestAnimationFrame(step);
|
||||
} else {
|
||||
|
|
@ -5960,16 +6156,77 @@ function animateBulkInsertProgress(state, from, to, total, duration) {
|
|||
});
|
||||
}
|
||||
|
||||
function bulkInsertProgressWords(state) {
|
||||
var mode = bulkInsertConflictMode(state);
|
||||
if (mode === "upsert") {
|
||||
return {
|
||||
active: "Upserting",
|
||||
complete: "upserted",
|
||||
};
|
||||
}
|
||||
if (mode === "ignore") {
|
||||
return {
|
||||
active: "Processing",
|
||||
complete: "processed",
|
||||
};
|
||||
}
|
||||
return {
|
||||
active: "Inserting",
|
||||
complete: "inserted",
|
||||
};
|
||||
}
|
||||
|
||||
function validateBulkInsertConflictRows(state, rows) {
|
||||
if (bulkInsertConflictMode(state) !== "upsert") {
|
||||
return null;
|
||||
}
|
||||
var insertData = tableInsertData() || {};
|
||||
var primaryKeys = insertData.primaryKeys || [];
|
||||
for (var index = 0; index < rows.length; index += 1) {
|
||||
var row = rows[index];
|
||||
var missing = primaryKeys.filter(function (key) {
|
||||
return (
|
||||
!Object.prototype.hasOwnProperty.call(row, key) ||
|
||||
row[key] === null ||
|
||||
typeof row[key] === "undefined"
|
||||
);
|
||||
});
|
||||
if (missing.length) {
|
||||
return (
|
||||
"Row " +
|
||||
(index + 1) +
|
||||
" is missing primary key " +
|
||||
missing.join(", ") +
|
||||
". Upsert requires primary key values for every row."
|
||||
);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function insertBulkPreviewRows(state) {
|
||||
if (!state.bulkInsertPreviewRows || state.bulkInsertInserted) {
|
||||
return;
|
||||
}
|
||||
if (!state.currentInsertUrl) {
|
||||
showRowEditDialogError(state, "Could not find the row insert URL.");
|
||||
var conflictMode = bulkInsertConflictMode(state);
|
||||
var url =
|
||||
conflictMode === "upsert" ? state.currentUpsertUrl : state.currentInsertUrl;
|
||||
if (!url) {
|
||||
showRowEditDialogError(
|
||||
state,
|
||||
conflictMode === "upsert"
|
||||
? "Could not find the row upsert URL."
|
||||
: "Could not find the row insert URL.",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
var rows = state.bulkInsertPreviewRows;
|
||||
var validationError = validateBulkInsertConflictRows(state, rows);
|
||||
if (validationError) {
|
||||
showRowEditDialogError(state, validationError);
|
||||
return;
|
||||
}
|
||||
var total = rows.length;
|
||||
var inserted = state.bulkInsertInsertedCount || 0;
|
||||
var batches = bulkInsertBatches(
|
||||
|
|
@ -5984,13 +6241,17 @@ async function insertBulkPreviewRows(state) {
|
|||
try {
|
||||
for (var batchIndex = 0; batchIndex < batches.length; batchIndex += 1) {
|
||||
var batch = batches[batchIndex];
|
||||
var response = await fetch(state.currentInsertUrl, {
|
||||
var payload = { rows: batch };
|
||||
if (conflictMode === "ignore") {
|
||||
payload.ignore = true;
|
||||
}
|
||||
var response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Accept: "application/json",
|
||||
},
|
||||
body: JSON.stringify({ rows: batch }),
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
var data = null;
|
||||
try {
|
||||
|
|
@ -6318,15 +6579,17 @@ function renderRowEditFields(state, data) {
|
|||
|
||||
function renderRowInsertFields(state, data) {
|
||||
var columns = data.columns || [];
|
||||
var bulkColumns = data.bulkColumns || columns;
|
||||
|
||||
state.insertMode = "single";
|
||||
state.bulkInsertColumnDetails = columns.slice();
|
||||
state.bulkInsertColumnDetails = bulkColumns.slice();
|
||||
state.bulkInsertMaxRows = data.maxInsertRows || 100;
|
||||
state.bulkInsertColumns = columns.map(function (column) {
|
||||
state.bulkInsertColumns = bulkColumns.map(function (column) {
|
||||
return column.name;
|
||||
});
|
||||
state.copyTemplateButton.disabled = !state.bulkInsertColumns.length;
|
||||
setBulkInsertCopyButtonReady(state);
|
||||
syncBulkInsertConflictUi(state);
|
||||
clearTimeout(state.copyTemplateResetTimer);
|
||||
state.copyTemplateResetTimer = null;
|
||||
resetBulkInsertPreview(state);
|
||||
|
|
@ -6424,6 +6687,17 @@ function ensureRowEditDialog(manager) {
|
|||
<p class="row-edit-bulk-note"><label for="row-edit-bulk-textarea">Paste TSV, CSV, or JSON</label>. You can also <button type="button" class="button-as-link row-edit-bulk-open-file">open a file</button> or drop it onto this textarea</p>
|
||||
<input class="row-edit-bulk-file-input" type="file" accept=".csv,.tsv,.json,.txt,text/csv,text/tab-separated-values,application/json,text/plain" hidden>
|
||||
<textarea class="row-edit-input row-edit-bulk-textarea" id="row-edit-bulk-textarea" name="_bulk_rows" rows="12" spellcheck="false"></textarea>
|
||||
<div class="row-edit-bulk-conflict" hidden>
|
||||
<label class="row-edit-bulk-conflict-label" for="row-edit-bulk-conflict-mode">If the row exists already</label>
|
||||
<div class="row-edit-bulk-conflict-control">
|
||||
<select class="row-edit-input row-edit-bulk-conflict-mode" id="row-edit-bulk-conflict-mode" aria-describedby="row-edit-bulk-conflict-help">
|
||||
<option value="insert">Stop with an error</option>
|
||||
<option value="ignore">Skip existing rows</option>
|
||||
<option value="upsert">Update existing and insert new</option>
|
||||
</select>
|
||||
<p class="row-edit-bulk-conflict-help" id="row-edit-bulk-conflict-help"></p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row-edit-bulk-actions">
|
||||
<button type="button" class="btn btn-ghost row-edit-copy-template"><span class="row-edit-copy-template-label-wide">Copy spreadsheet template</span><span class="row-edit-copy-template-label-narrow">Copy template</span></button>
|
||||
<span class="row-edit-bulk-template-note"><span class="row-edit-bulk-template-note-wide">You can paste the template into Google Sheets or Excel.</span><span class="row-edit-bulk-template-note-narrow">Paste into Google Sheets or Excel</span></span>
|
||||
|
|
@ -6462,6 +6736,13 @@ function ensureRowEditDialog(manager) {
|
|||
bulkInsertProgressStatus: dialog.querySelector(
|
||||
".row-edit-bulk-progress-status",
|
||||
),
|
||||
bulkInsertConflictField: dialog.querySelector(".row-edit-bulk-conflict"),
|
||||
bulkInsertConflictSelect: dialog.querySelector(
|
||||
".row-edit-bulk-conflict-mode",
|
||||
),
|
||||
bulkInsertConflictHelp: dialog.querySelector(
|
||||
".row-edit-bulk-conflict-help",
|
||||
),
|
||||
copyTemplateButton: dialog.querySelector(".row-edit-copy-template"),
|
||||
bulkInsertOpenFileButton: dialog.querySelector(".row-edit-bulk-open-file"),
|
||||
bulkInsertFileInput: dialog.querySelector(".row-edit-bulk-file-input"),
|
||||
|
|
@ -6474,10 +6755,14 @@ function ensureRowEditDialog(manager) {
|
|||
currentRowId: null,
|
||||
currentPkPath: null,
|
||||
currentInsertUrl: null,
|
||||
currentUpsertUrl: null,
|
||||
currentUpdateUrl: null,
|
||||
currentFragmentUrl: null,
|
||||
mode: "edit",
|
||||
insertMode: "single",
|
||||
bulkInsertConflictMode: "ignore",
|
||||
bulkInsertHasPrimaryKeyColumns: false,
|
||||
bulkInsertLiveValidationError: null,
|
||||
bulkInsertColumns: [],
|
||||
bulkInsertColumnDetails: [],
|
||||
bulkInsertPreviewRows: null,
|
||||
|
|
@ -6620,10 +6905,22 @@ function ensureRowEditDialog(manager) {
|
|||
|
||||
rowEditDialogState.bulkInsertTextarea.addEventListener("input", function () {
|
||||
resetBulkInsertPreview(rowEditDialogState);
|
||||
clearRowEditDialogError(rowEditDialogState);
|
||||
syncBulkInsertTextareaValidation(rowEditDialogState);
|
||||
updateRowEditDialogButtons(rowEditDialogState);
|
||||
});
|
||||
|
||||
rowEditDialogState.bulkInsertConflictSelect.addEventListener(
|
||||
"change",
|
||||
function () {
|
||||
rowEditDialogState.bulkInsertConflictMode =
|
||||
rowEditDialogState.bulkInsertConflictSelect.value;
|
||||
syncBulkInsertConflictUi(rowEditDialogState);
|
||||
resetBulkInsertPreview(rowEditDialogState);
|
||||
syncBulkInsertTextareaValidation(rowEditDialogState);
|
||||
updateRowEditDialogButtons(rowEditDialogState);
|
||||
},
|
||||
);
|
||||
|
||||
dialog.addEventListener("click", function (ev) {
|
||||
if (ev.target === dialog) {
|
||||
closeRowEditDialogIfConfirmed(rowEditDialogState);
|
||||
|
|
@ -6649,6 +6946,7 @@ function ensureRowEditDialog(manager) {
|
|||
var redirectOnCloseUrl = state.redirectOnCloseUrl;
|
||||
state.loadId += 1;
|
||||
state.isClosePending = false;
|
||||
state.bulkInsertLiveValidationError = null;
|
||||
state.shouldReloadOnClose = false;
|
||||
state.redirectOnCloseUrl = null;
|
||||
clearTimeout(state.copyTemplateResetTimer);
|
||||
|
|
@ -6696,6 +6994,7 @@ async function openRowEditDialog(button, manager) {
|
|||
state.currentRowId = row.getAttribute("data-row") || "";
|
||||
state.currentPkPath = rowDisplayLabel(row);
|
||||
state.currentInsertUrl = null;
|
||||
state.currentUpsertUrl = null;
|
||||
state.currentUpdateUrl = rowUpdateUrl(row);
|
||||
state.currentFragmentUrl = rowFragmentUrl(row);
|
||||
state.insertMode = "single";
|
||||
|
|
@ -6773,9 +7072,12 @@ function openRowInsertDialog(button, manager) {
|
|||
state.currentRowId = null;
|
||||
state.currentPkPath = null;
|
||||
state.currentInsertUrl = tableInsertUrl();
|
||||
state.currentUpsertUrl = tableUpsertUrl();
|
||||
state.currentUpdateUrl = null;
|
||||
state.currentFragmentUrl = null;
|
||||
state.insertMode = "single";
|
||||
state.bulkInsertConflictMode = "ignore";
|
||||
state.bulkInsertLiveValidationError = null;
|
||||
state.bulkInsertTextarea.value = "";
|
||||
state.shouldReloadOnClose = false;
|
||||
state.redirectOnCloseUrl = null;
|
||||
|
|
|
|||
|
|
@ -480,8 +480,15 @@ async def _table_insert_ui(
|
|||
):
|
||||
return None
|
||||
|
||||
can_update = await datasette.allowed(
|
||||
action="update-row",
|
||||
resource=TableResource(database=database_name, table=table_name),
|
||||
actor=request.actor,
|
||||
)
|
||||
|
||||
column_types_map = await datasette.get_column_types(database_name, table_name)
|
||||
columns = []
|
||||
bulk_columns = []
|
||||
column_details = await db.table_column_details(table_name)
|
||||
for column in column_details:
|
||||
if column.hidden:
|
||||
|
|
@ -492,33 +499,39 @@ async def _table_insert_ui(
|
|||
and len(pks) == 1
|
||||
and SQLiteType.from_declared_type(column.type) == SQLiteType.INTEGER
|
||||
)
|
||||
column_type = column_types_map.get(column.name)
|
||||
column_data = {
|
||||
"name": column.name,
|
||||
"sqlite_type": _column_sqlite_type_for_insert_form(column),
|
||||
"notnull": column.notnull,
|
||||
"default": column.default_value,
|
||||
"has_default": column.default_value is not None,
|
||||
"is_pk": is_pk,
|
||||
"value_kind": _column_value_kind_for_insert_form(column),
|
||||
"column_type": (
|
||||
{"type": column_type.name, "config": column_type.config}
|
||||
if column_type is not None
|
||||
else None
|
||||
),
|
||||
}
|
||||
bulk_columns.append(column_data)
|
||||
if is_auto_pk:
|
||||
continue
|
||||
column_type = column_types_map.get(column.name)
|
||||
columns.append(
|
||||
{
|
||||
"name": column.name,
|
||||
"sqlite_type": _column_sqlite_type_for_insert_form(column),
|
||||
"notnull": column.notnull,
|
||||
"default": column.default_value,
|
||||
"has_default": column.default_value is not None,
|
||||
"is_pk": is_pk,
|
||||
"value_kind": _column_value_kind_for_insert_form(column),
|
||||
"column_type": (
|
||||
{"type": column_type.name, "config": column_type.config}
|
||||
if column_type is not None
|
||||
else None
|
||||
),
|
||||
}
|
||||
)
|
||||
columns.append(column_data)
|
||||
|
||||
return {
|
||||
data = {
|
||||
"path": "{}/-/insert".format(datasette.urls.table(database_name, table_name)),
|
||||
"tableName": table_name,
|
||||
"columns": columns,
|
||||
"bulkColumns": bulk_columns,
|
||||
"primaryKeys": pks,
|
||||
"maxInsertRows": datasette.setting("max_insert_rows"),
|
||||
}
|
||||
if can_update:
|
||||
data["upsertPath"] = "{}/-/upsert".format(
|
||||
datasette.urls.table(database_name, table_name)
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
async def _table_alter_ui(
|
||||
|
|
|
|||
|
|
@ -108,6 +108,11 @@ def write_playwright_database(db_path):
|
|||
status text not null default 'todo',
|
||||
score integer default 5
|
||||
);
|
||||
create table upsert_items (
|
||||
id text primary key,
|
||||
title text,
|
||||
metadata text
|
||||
);
|
||||
insert into projects (title, metadata, logo, notes, score) values
|
||||
(
|
||||
'Build Datasette',
|
||||
|
|
@ -116,6 +121,8 @@ def write_playwright_database(db_path):
|
|||
'Initial notes',
|
||||
5
|
||||
);
|
||||
insert into upsert_items (id, title, metadata) values
|
||||
('existing', 'Existing title', '{"old": true}');
|
||||
""")
|
||||
finally:
|
||||
conn.close()
|
||||
|
|
@ -157,6 +164,12 @@ def write_playwright_config(config_path):
|
|||
"insert-row": True,
|
||||
},
|
||||
},
|
||||
"upsert_items": {
|
||||
"permissions": {
|
||||
"insert-row": True,
|
||||
"update-row": True,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -300,6 +313,31 @@ def bulk_default_rows(datasette_server, **filters):
|
|||
return response.json()["rows"]
|
||||
|
||||
|
||||
def upsert_item_rows(datasette_server, **filters):
|
||||
params = {
|
||||
"_shape": "objects",
|
||||
**{key: str(value) for key, value in filters.items()},
|
||||
}
|
||||
response = httpx.get(f"{datasette_server}data/upsert_items.json", params=params)
|
||||
response.raise_for_status()
|
||||
return response.json()["rows"]
|
||||
|
||||
|
||||
def upsert_item_row(datasette_server, pk):
|
||||
rows = upsert_item_rows(datasette_server, id=pk)
|
||||
assert len(rows) == 1
|
||||
return rows[0]
|
||||
|
||||
|
||||
def open_bulk_insert_dialog(page, url):
|
||||
page.goto(url)
|
||||
page.locator('button[data-table-action="insert-row"]').click()
|
||||
dialog = page.locator("#row-edit-dialog")
|
||||
dialog.wait_for()
|
||||
dialog.locator(".row-edit-bulk-insert").click()
|
||||
return dialog
|
||||
|
||||
|
||||
def open_jump_menu(page):
|
||||
page.keyboard.press("/")
|
||||
page.locator("navigation-search .search-input").wait_for()
|
||||
|
|
@ -1166,6 +1204,116 @@ def test_bulk_insert_preview_inserts_rows(page, datasette_server):
|
|||
assert project_rows(datasette_server, title="Bulk two")
|
||||
|
||||
|
||||
@pytest.mark.playwright
|
||||
def test_bulk_insert_upsert_option_updates_existing_and_inserts_new(
|
||||
page, datasette_server
|
||||
):
|
||||
dialog = open_bulk_insert_dialog(page, f"{datasette_server}data/upsert_items")
|
||||
textarea = dialog.locator(".row-edit-bulk-textarea")
|
||||
conflict_field = dialog.locator(".row-edit-bulk-conflict")
|
||||
conflict_select = dialog.locator(".row-edit-bulk-conflict-mode")
|
||||
|
||||
assert conflict_field.is_hidden()
|
||||
textarea.fill("title\nNo primary key")
|
||||
assert conflict_field.is_hidden()
|
||||
|
||||
textarea.fill(
|
||||
"id,title,metadata\nexisting,Updated by upsert,{}\nnew,Inserted by upsert,{}"
|
||||
)
|
||||
assert conflict_field.is_visible()
|
||||
assert (
|
||||
dialog.locator(".row-edit-bulk-conflict-label").inner_text()
|
||||
== "If the row exists already"
|
||||
)
|
||||
assert conflict_select.input_value() == "ignore"
|
||||
assert (
|
||||
conflict_select.evaluate("""node => Array.from(node.options)
|
||||
.filter((option) => !option.hidden)
|
||||
.map((option) => option.textContent.trim())""")
|
||||
== [
|
||||
"Stop with an error",
|
||||
"Skip existing rows",
|
||||
"Update existing and insert new",
|
||||
]
|
||||
)
|
||||
|
||||
conflict_select.select_option("upsert")
|
||||
assert conflict_select.input_value() == "upsert"
|
||||
dialog.locator(".row-edit-save").click()
|
||||
|
||||
assert dialog.locator(".row-edit-save").inner_text() == "Update or insert rows"
|
||||
preview_text = dialog.locator(".row-edit-bulk-preview-table").inner_text()
|
||||
assert "Updated by upsert" in preview_text
|
||||
assert "Inserted by upsert" in preview_text
|
||||
|
||||
dialog.locator(".row-edit-save").click()
|
||||
dialog.locator(
|
||||
".row-edit-bulk-progress-status", has_text="2 rows upserted."
|
||||
).wait_for()
|
||||
|
||||
assert upsert_item_row(datasette_server, "existing")["title"] == (
|
||||
"Updated by upsert"
|
||||
)
|
||||
assert upsert_item_row(datasette_server, "new")["title"] == "Inserted by upsert"
|
||||
|
||||
|
||||
@pytest.mark.playwright
|
||||
def test_bulk_insert_conflicts_hide_upsert_without_update_permission(
|
||||
page, datasette_server
|
||||
):
|
||||
dialog = open_bulk_insert_dialog(page, f"{datasette_server}data/bulk_defaults")
|
||||
textarea = dialog.locator(".row-edit-bulk-textarea")
|
||||
conflict_field = dialog.locator(".row-edit-bulk-conflict")
|
||||
conflict_select = dialog.locator(".row-edit-bulk-conflict-mode")
|
||||
|
||||
textarea.fill("title\nOnly title")
|
||||
assert conflict_field.is_hidden()
|
||||
|
||||
textarea.fill("id,title\n1,Only title")
|
||||
assert conflict_field.is_visible()
|
||||
assert conflict_select.input_value() == "ignore"
|
||||
assert (
|
||||
conflict_select.evaluate("""node => Array.from(node.options)
|
||||
.filter((option) => !option.hidden)
|
||||
.map((option) => option.textContent.trim())""")
|
||||
== [
|
||||
"Stop with an error",
|
||||
"Skip existing rows",
|
||||
]
|
||||
)
|
||||
assert conflict_select.locator('option[value="upsert"]').evaluate(
|
||||
"node => node.hidden && node.disabled"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.playwright
|
||||
def test_bulk_insert_live_validation_reports_unknown_columns(page, datasette_server):
|
||||
dialog = open_bulk_insert_dialog(page, f"{datasette_server}data/projects")
|
||||
textarea = dialog.locator(".row-edit-bulk-textarea")
|
||||
error = dialog.locator(".row-edit-error")
|
||||
save = dialog.locator(".row-edit-save")
|
||||
|
||||
textarea.fill(json.dumps([{"id2": 1, "title": "Unknown column"}]))
|
||||
error.wait_for()
|
||||
assert error.inner_text() == "JSON row 1 has unknown column id2."
|
||||
assert save.is_disabled()
|
||||
assert textarea.evaluate("node => document.activeElement === node")
|
||||
|
||||
textarea.fill("id2,title\n1,Unknown column")
|
||||
assert error.inner_text() == "Unknown column id2 in header row."
|
||||
assert save.is_disabled()
|
||||
|
||||
textarea.fill("[")
|
||||
assert error.is_hidden()
|
||||
assert not save.is_disabled()
|
||||
|
||||
textarea.fill(json.dumps([{"id": 1, "title": "Known column"}]))
|
||||
assert error.is_hidden()
|
||||
assert not save.is_disabled()
|
||||
assert dialog.locator(".row-edit-bulk-conflict").is_visible()
|
||||
assert dialog.locator(".row-edit-bulk-conflict-mode").input_value() == "ignore"
|
||||
|
||||
|
||||
@pytest.mark.playwright
|
||||
def test_bulk_insert_omits_columns_absent_from_pasted_input(page, datasette_server):
|
||||
page.goto(f"{datasette_server}data/bulk_defaults")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue