mirror of
https://github.com/simonw/sqlite-utils.git
synced 2026-07-23 01:14:31 +02:00
hash_id= mechanism for setting ID using content hash, closes #13
This commit is contained in:
Simon Willison
parent
e615d22c55
commit
dbf239e3fc
3 changed files with 93 additions and 7 deletions
|
|
@ -121,6 +121,10 @@ If you want to explicitly set the order of the columns you can do so using the `
|
|||
|
||||
You don't need to pass all of the columns to the ``column_order`` parameter. If you only pass a subset of the columns the remaining columns will be ordered based on the key order of the dictionary.
|
||||
|
||||
After inserting a row like this, the ``dogs.last_rowid`` property will return the SQLite ``rowid`` assigned to the most recently inserted record.
|
||||
|
||||
The ``dogs.last_pk`` property will return the last inserted primary key value, if you specified one. This can be very useful when writing code that creates foreign key or many-to-many relationships.
|
||||
|
||||
Bulk inserts
|
||||
============
|
||||
|
||||
|
|
@ -178,6 +182,31 @@ Note that the ``pk`` and ``column_order`` parameters here are optional if you ar
|
|||
|
||||
An ``upsert_all()`` method is also available, which behaves like ``insert_all()`` but performs upserts instead.
|
||||
|
||||
Setting an ID based on the hash of the row contents
|
||||
===================================================
|
||||
|
||||
Sometimes you will find yourself working with a dataset that includes rows that do not have a provided obvious ID, but where you would like to assign one so that you can later upsert into that table without creating duplicate records.
|
||||
|
||||
In these cases, a useful technique is to create an ID that is derived from the sha1 hash of the row contents.
|
||||
|
||||
``sqlite-utils`` can do this for you using the ``hash_id=`` option. For example::
|
||||
|
||||
db = sqlite_utils.Database("dogs.db")
|
||||
db["dogs"].upsert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id")
|
||||
print(list(db["dogs]))
|
||||
|
||||
Outputs::
|
||||
|
||||
[{'id': 'f501265970505d9825d8d9f590bfab3519fb20b1', 'name': 'Cleo', 'twitter': 'cleopaws'}]
|
||||
|
||||
If you are going to use that ID straight away, you can access it using ``last_pk``::
|
||||
|
||||
dog_id = db["dogs"].upsert({
|
||||
"name": "Cleo",
|
||||
"twitter": "cleopaws"
|
||||
}, hash_id="id").last_pk
|
||||
# dog_id is now "f501265970505d9825d8d9f590bfab3519fb20b1"
|
||||
|
||||
Creating views
|
||||
==============
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import sqlite3
|
||||
from collections import namedtuple
|
||||
import datetime
|
||||
import hashlib
|
||||
import itertools
|
||||
import json
|
||||
import pathlib
|
||||
|
|
@ -53,7 +54,7 @@ class Database:
|
|||
return [dict(zip(keys, row)) for row in cursor.fetchall()]
|
||||
|
||||
def create_table(
|
||||
self, name, columns, pk=None, foreign_keys=None, column_order=None
|
||||
self, name, columns, pk=None, foreign_keys=None, column_order=None, hash_id=None
|
||||
):
|
||||
foreign_keys = foreign_keys or []
|
||||
foreign_keys_by_name = {fk[0]: fk for fk in foreign_keys}
|
||||
|
|
@ -62,6 +63,9 @@ class Database:
|
|||
column_items.sort(
|
||||
key=lambda p: column_order.index(p[0]) if p[0] in column_order else 999
|
||||
)
|
||||
if hash_id:
|
||||
column_items.insert(0, (hash_id, str))
|
||||
pk = hash_id
|
||||
extra = ""
|
||||
col_type_mapping = {
|
||||
float: "FLOAT",
|
||||
|
|
@ -216,7 +220,9 @@ class Table:
|
|||
indexes.append(Index(**row))
|
||||
return indexes
|
||||
|
||||
def create(self, columns, pk=None, foreign_keys=None, column_order=None):
|
||||
def create(
|
||||
self, columns, pk=None, foreign_keys=None, column_order=None, hash_id=None
|
||||
):
|
||||
columns = {name: value for (name, value) in columns.items()}
|
||||
with self.db.conn:
|
||||
self.db.create_table(
|
||||
|
|
@ -225,6 +231,7 @@ class Table:
|
|||
pk=pk,
|
||||
foreign_keys=foreign_keys,
|
||||
column_order=column_order,
|
||||
hash_id=hash_id,
|
||||
)
|
||||
self.exists = True
|
||||
return self
|
||||
|
|
@ -358,7 +365,13 @@ class Table:
|
|||
return self.db.conn.execute(sql, (q,)).fetchall()
|
||||
|
||||
def insert(
|
||||
self, record, pk=None, foreign_keys=None, upsert=False, column_order=None
|
||||
self,
|
||||
record,
|
||||
pk=None,
|
||||
foreign_keys=None,
|
||||
upsert=False,
|
||||
column_order=None,
|
||||
hash_id=None,
|
||||
):
|
||||
return self.insert_all(
|
||||
[record],
|
||||
|
|
@ -366,6 +379,7 @@ class Table:
|
|||
foreign_keys=foreign_keys,
|
||||
upsert=upsert,
|
||||
column_order=column_order,
|
||||
hash_id=hash_id,
|
||||
)
|
||||
|
||||
def insert_all(
|
||||
|
|
@ -376,12 +390,14 @@ class Table:
|
|||
upsert=False,
|
||||
batch_size=100,
|
||||
column_order=None,
|
||||
hash_id=None,
|
||||
):
|
||||
"""
|
||||
Like .insert() but takes a list of records and ensures that the table
|
||||
that it creates (if table does not exist) has columns for ALL of that
|
||||
data
|
||||
"""
|
||||
assert not (hash_id and pk), "Use either pk= or hash_id="
|
||||
all_columns = None
|
||||
first = True
|
||||
for chunk in chunks(records, batch_size):
|
||||
|
|
@ -394,11 +410,14 @@ class Table:
|
|||
pk,
|
||||
foreign_keys,
|
||||
column_order=column_order,
|
||||
hash_id=hash_id,
|
||||
)
|
||||
all_columns = set()
|
||||
for record in chunk:
|
||||
all_columns.update(record.keys())
|
||||
all_columns = list(sorted(all_columns))
|
||||
if hash_id:
|
||||
all_columns.insert(0, hash_id)
|
||||
first = False
|
||||
sql = """
|
||||
INSERT {upsert} INTO [{table}] ({columns}) VALUES {rows};
|
||||
|
|
@ -418,24 +437,44 @@ class Table:
|
|||
values = []
|
||||
for record in chunk:
|
||||
values.extend(
|
||||
jsonify_if_needed(record.get(key, None)) for key in all_columns
|
||||
jsonify_if_needed(
|
||||
record.get(key, None if key != hash_id else _hash(record))
|
||||
)
|
||||
for key in all_columns
|
||||
)
|
||||
with self.db.conn:
|
||||
result = self.db.conn.execute(sql, values)
|
||||
self.last_id = result.lastrowid
|
||||
self.last_rowid = result.lastrowid
|
||||
self.lash_pk = None
|
||||
if hash_id or pk:
|
||||
self.last_pk = self.db.conn.execute(
|
||||
"select [{}] from [{}] where rowid = ?".format(
|
||||
hash_id or pk, self.name
|
||||
),
|
||||
(self.last_rowid,),
|
||||
).fetchone()[0]
|
||||
return self
|
||||
|
||||
def upsert(self, record, pk=None, foreign_keys=None, column_order=None):
|
||||
def upsert(
|
||||
self, record, pk=None, foreign_keys=None, column_order=None, hash_id=None
|
||||
):
|
||||
return self.insert(
|
||||
record,
|
||||
pk=pk,
|
||||
foreign_keys=foreign_keys,
|
||||
upsert=True,
|
||||
column_order=column_order,
|
||||
hash_id=hash_id,
|
||||
)
|
||||
|
||||
def upsert_all(
|
||||
self, records, pk=None, foreign_keys=None, column_order=None, batch_size=100
|
||||
self,
|
||||
records,
|
||||
pk=None,
|
||||
foreign_keys=None,
|
||||
column_order=None,
|
||||
batch_size=100,
|
||||
hash_id=None,
|
||||
):
|
||||
return self.insert_all(
|
||||
records,
|
||||
|
|
@ -444,6 +483,7 @@ class Table:
|
|||
column_order=column_order,
|
||||
batch_size=100,
|
||||
upsert=True,
|
||||
hash_id=hash_id,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -458,3 +498,9 @@ def jsonify_if_needed(value):
|
|||
return json.dumps(value)
|
||||
else:
|
||||
return value
|
||||
|
||||
|
||||
def _hash(record):
|
||||
return hashlib.sha1(
|
||||
json.dumps(record, separators=(",", ":"), sort_keys=True).encode("utf8")
|
||||
).hexdigest()
|
||||
|
|
|
|||
|
|
@ -206,6 +206,17 @@ def test_insert_thousands_ignores_extra_columns_after_first_100(fresh_db):
|
|||
assert [{"i": 101, "word": None}] == rows
|
||||
|
||||
|
||||
def test_insert_hash_id(fresh_db):
|
||||
dogs = fresh_db["dogs"]
|
||||
id = dogs.upsert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id").last_pk
|
||||
assert "f501265970505d9825d8d9f590bfab3519fb20b1" == id
|
||||
assert 1 == dogs.count
|
||||
# Upserting a second time should not create a new row
|
||||
id2 = dogs.upsert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id").last_pk
|
||||
assert "f501265970505d9825d8d9f590bfab3519fb20b1" == id2
|
||||
assert 1 == dogs.count
|
||||
|
||||
|
||||
def test_create_view(fresh_db):
|
||||
fresh_db["data"].insert({"foo": "foo", "bar": "bar"})
|
||||
fresh_db.create_view("bar", "select bar from data")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue