remote/sqlite-utils
1
0
Fork 0
mirror of https://github.com/simonw/sqlite-utils.git synced 2026-07-26 02:44:33 +02:00
Code Issues Projects Releases Packages Wiki Activity

CLI insert tool now uses generators, closes #7

Browse source 
Also cleaned up the logic so we commit rows in batches too.
This commit is contained in:
Simon Willison 2019-01-27 22:26:45 -08:00
commit 50589f8523
2 changed files with 15 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

8
sqlite_utils/cli.py
View file

@ -72,17 +72,17 @@ def optimize(path, no_vacuum):
@click.argument("json_file", type=click.File(), required=True)
@click.option("--pk", help="Column to use as the primary key, e.g. id")
@click.option("--nl", is_flag=True, help="Expect newline-delimited JSON")
def insert(path, table, json_file, pk, nl):
@click.option("--batch-size", type=int, default=100, help="Commit every X records")
def insert(path, table, json_file, pk, nl, batch_size):
"Insert records from JSON file into the table, create table if it is missing"
db = sqlite_utils.Database(path)
if nl:
# TODO: Use a generator once #7 is solved
docs = [json_std.loads(line) for line in json_file]
docs = (json_std.loads(line) for line in json_file)
else:
docs = json_std.load(json_file)
if isinstance(docs, dict):
docs = [docs]
db[table].insert_all(docs, pk=pk)
db[table].insert_all(docs, pk=pk, batch_size=batch_size)
@cli.command()