remote/dclient
1
0
Fork 0
mirror of https://github.com/simonw/dclient.git synced 2026-09-28 04:44:28 +02:00
Code Issues Projects Releases Packages Wiki Activity

--batch-size option, refs #8

Browse source
This commit is contained in:
Simon Willison 2023-07-24 14:23:53 -07:00
commit 0d4e5cfc73
1 changed files with 5 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

9
dclient/cli.py
View file

@ -7,9 +7,6 @@ from sqlite_utils.utils import rows_from_file, Format, TypeTracker, progressbar
import sys import sys
from .utils import token_for_url from .utils import token_for_url
INSERT_BATCH_SIZE = 100
def get_config_dir(): def get_config_dir():
return pathlib.Path(click.get_app_dir("io.datasette.dclient")) return pathlib.Path(click.get_app_dir("io.datasette.dclient"))
@ -108,6 +105,9 @@ def query(url_or_alias, sql, token):
multiple=True, multiple=True,
help="Columns to use as the primary key when creating the table", help="Columns to use as the primary key when creating the table",
) )
@click.option(
"--batch-size", type=int, default=100, help="Send rows in batches of this size"
)
@click.option("--token", "-t", help="API token") @click.option("--token", "-t", help="API token")
@click.option("--silent", is_flag=True, help="Don't output progress") @click.option("--silent", is_flag=True, help="Don't output progress")
def insert( def insert(
@ -123,6 +123,7 @@ def insert(
ignore, ignore,
create, create,
pks, pks,
batch_size,
token, token,
silent, silent,
): ):
@ -179,7 +180,7 @@ def insert(
show_percent=True, show_percent=True,
) as bar: ) as bar:
bytes_so_far = 0 bytes_so_far = 0
for batch in _batches(rows, INSERT_BATCH_SIZE): for batch in _batches(rows, batch_size):
if file_size is not None: if file_size is not None:
bytes_consumed_so_far = fp.tell() bytes_consumed_so_far = fp.tell()
new_bytes = bytes_consumed_so_far - bytes_so_far new_bytes = bytes_consumed_so_far - bytes_so_far