mirror of
https://github.com/simonw/sqlite-utils.git
synced 2026-09-16 13:34:10 +02:00
Extract build_insert_queries_and_params
This commit is contained in:
Simon Wiles
parent
1b1e0ccfd3
commit
c241162fa9
1 changed files with 106 additions and 79 deletions
|
|
@ -1031,6 +1031,100 @@ class Table(Queryable):
|
|||
self.last_pk = pk_values[0] if len(self.pks) == 1 else pk_values
|
||||
return self
|
||||
|
||||
def build_insert_queries_and_params(
|
||||
self,
|
||||
extracts,
|
||||
chunk,
|
||||
all_columns,
|
||||
hash_id,
|
||||
upsert,
|
||||
pk,
|
||||
conversions,
|
||||
num_records_processed,
|
||||
replace,
|
||||
ignore,
|
||||
):
|
||||
# values is the list of insert data that is passed to the
|
||||
# .execute() method - but some of them may be replaced by
|
||||
# new primary keys if we are extracting any columns.
|
||||
values = []
|
||||
extracts = resolve_extracts(extracts)
|
||||
for record in chunk:
|
||||
record_values = []
|
||||
for key in all_columns:
|
||||
value = jsonify_if_needed(
|
||||
record.get(key, None if key != hash_id else _hash(record))
|
||||
)
|
||||
if key in extracts:
|
||||
extract_table = extracts[key]
|
||||
value = self.db[extract_table].lookup({"value": value})
|
||||
record_values.append(value)
|
||||
values.append(record_values)
|
||||
|
||||
queries_and_params = []
|
||||
if upsert:
|
||||
if isinstance(pk, str):
|
||||
pks = [pk]
|
||||
else:
|
||||
pks = pk
|
||||
self.last_pk = None
|
||||
for record_values in values:
|
||||
# TODO: make more efficient:
|
||||
record = dict(zip(all_columns, record_values))
|
||||
sql = "INSERT OR IGNORE INTO [{table}]({pks}) VALUES({pk_placeholders});".format(
|
||||
table=self.name,
|
||||
pks=", ".join(["[{}]".format(p) for p in pks]),
|
||||
pk_placeholders=", ".join(["?" for p in pks]),
|
||||
)
|
||||
queries_and_params.append((sql, [record[col] for col in pks]))
|
||||
# UPDATE [book] SET [name] = 'Programming' WHERE [id] = 1001;
|
||||
set_cols = [col for col in all_columns if col not in pks]
|
||||
sql2 = "UPDATE [{table}] SET {pairs} WHERE {wheres}".format(
|
||||
table=self.name,
|
||||
pairs=", ".join(
|
||||
"[{}] = {}".format(col, conversions.get(col, "?"))
|
||||
for col in set_cols
|
||||
),
|
||||
wheres=" AND ".join("[{}] = ?".format(pk) for pk in pks),
|
||||
)
|
||||
queries_and_params.append(
|
||||
(
|
||||
sql2,
|
||||
[record[col] for col in set_cols] + [record[pk] for pk in pks],
|
||||
)
|
||||
)
|
||||
# We can populate .last_pk right here
|
||||
if num_records_processed == 1:
|
||||
self.last_pk = tuple(record[pk] for pk in pks)
|
||||
if len(self.last_pk) == 1:
|
||||
self.last_pk = self.last_pk[0]
|
||||
|
||||
else:
|
||||
or_what = ""
|
||||
if replace:
|
||||
or_what = "OR REPLACE "
|
||||
elif ignore:
|
||||
or_what = "OR IGNORE "
|
||||
sql = """
|
||||
INSERT {or_what}INTO [{table}] ({columns}) VALUES {rows};
|
||||
""".strip().format(
|
||||
or_what=or_what,
|
||||
table=self.name,
|
||||
columns=", ".join("[{}]".format(c) for c in all_columns),
|
||||
rows=", ".join(
|
||||
"({placeholders})".format(
|
||||
placeholders=", ".join(
|
||||
[conversions.get(col, "?") for col in all_columns]
|
||||
)
|
||||
)
|
||||
for record in chunk
|
||||
),
|
||||
)
|
||||
flat_values = list(itertools.chain(*values))
|
||||
queries_and_params = [(sql, flat_values)]
|
||||
|
||||
return queries_and_params
|
||||
|
||||
def insert(
|
||||
self,
|
||||
record,
|
||||
|
|
@ -1164,86 +1258,19 @@ class Table(Queryable):
|
|||
|
||||
validate_column_names(all_columns)
|
||||
first = False
|
||||
# values is the list of insert data that is passed to the
|
||||
# .execute() method - but some of them may be replaced by
|
||||
# new primary keys if we are extracting any columns.
|
||||
values = []
|
||||
extracts = resolve_extracts(extracts)
|
||||
for record in chunk:
|
||||
record_values = []
|
||||
for key in all_columns:
|
||||
value = jsonify_if_needed(
|
||||
record.get(key, None if key != hash_id else _hash(record))
|
||||
)
|
||||
if key in extracts:
|
||||
extract_table = extracts[key]
|
||||
value = self.db[extract_table].lookup({"value": value})
|
||||
record_values.append(value)
|
||||
values.append(record_values)
|
||||
|
||||
queries_and_params = []
|
||||
if upsert:
|
||||
if isinstance(pk, str):
|
||||
pks = [pk]
|
||||
else:
|
||||
pks = pk
|
||||
self.last_pk = None
|
||||
for record_values in values:
|
||||
# TODO: make more efficient:
|
||||
record = dict(zip(all_columns, record_values))
|
||||
params = []
|
||||
sql = "INSERT OR IGNORE INTO [{table}]({pks}) VALUES({pk_placeholders});".format(
|
||||
table=self.name,
|
||||
pks=", ".join(["[{}]".format(p) for p in pks]),
|
||||
pk_placeholders=", ".join(["?" for p in pks]),
|
||||
)
|
||||
queries_and_params.append((sql, [record[col] for col in pks]))
|
||||
# UPDATE [book] SET [name] = 'Programming' WHERE [id] = 1001;
|
||||
set_cols = [col for col in all_columns if col not in pks]
|
||||
sql2 = "UPDATE [{table}] SET {pairs} WHERE {wheres}".format(
|
||||
table=self.name,
|
||||
pairs=", ".join(
|
||||
"[{}] = {}".format(col, conversions.get(col, "?"))
|
||||
for col in set_cols
|
||||
),
|
||||
wheres=" AND ".join("[{}] = ?".format(pk) for pk in pks),
|
||||
)
|
||||
queries_and_params.append(
|
||||
(
|
||||
sql2,
|
||||
[record[col] for col in set_cols]
|
||||
+ [record[pk] for pk in pks],
|
||||
)
|
||||
)
|
||||
# We can populate .last_pk right here
|
||||
if num_records_processed == 1:
|
||||
self.last_pk = tuple(record[pk] for pk in pks)
|
||||
if len(self.last_pk) == 1:
|
||||
self.last_pk = self.last_pk[0]
|
||||
|
||||
else:
|
||||
or_what = ""
|
||||
if replace:
|
||||
or_what = "OR REPLACE "
|
||||
elif ignore:
|
||||
or_what = "OR IGNORE "
|
||||
sql = """
|
||||
INSERT {or_what}INTO [{table}] ({columns}) VALUES {rows};
|
||||
""".strip().format(
|
||||
or_what=or_what,
|
||||
table=self.name,
|
||||
columns=", ".join("[{}]".format(c) for c in all_columns),
|
||||
rows=", ".join(
|
||||
"({placeholders})".format(
|
||||
placeholders=", ".join(
|
||||
[conversions.get(col, "?") for col in all_columns]
|
||||
)
|
||||
)
|
||||
for record in chunk
|
||||
),
|
||||
)
|
||||
flat_values = list(itertools.chain(*values))
|
||||
queries_and_params = [(sql, flat_values)]
|
||||
queries_and_params = self.build_insert_queries_and_params(
|
||||
extracts,
|
||||
chunk,
|
||||
all_columns,
|
||||
hash_id,
|
||||
upsert,
|
||||
pk,
|
||||
conversions,
|
||||
num_records_processed,
|
||||
replace,
|
||||
ignore,
|
||||
)
|
||||
|
||||
with self.db.conn:
|
||||
for query, params in queries_and_params:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue