From 0d4e5cfc73e8f7b07018fa5d5cab51161a27a67c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 24 Jul 2023 14:23:53 -0700 Subject: [PATCH] --batch-size option, refs #8 --- dclient/cli.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dclient/cli.py b/dclient/cli.py index 4bccac2..9046cb9 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -7,9 +7,6 @@ from sqlite_utils.utils import rows_from_file, Format, TypeTracker, progressbar import sys from .utils import token_for_url -INSERT_BATCH_SIZE = 100 - - def get_config_dir(): return pathlib.Path(click.get_app_dir("io.datasette.dclient")) @@ -108,6 +105,9 @@ def query(url_or_alias, sql, token): multiple=True, help="Columns to use as the primary key when creating the table", ) +@click.option( + "--batch-size", type=int, default=100, help="Send rows in batches of this size" +) @click.option("--token", "-t", help="API token") @click.option("--silent", is_flag=True, help="Don't output progress") def insert( @@ -123,6 +123,7 @@ def insert( ignore, create, pks, + batch_size, token, silent, ): @@ -179,7 +180,7 @@ def insert( show_percent=True, ) as bar: bytes_so_far = 0 - for batch in _batches(rows, INSERT_BATCH_SIZE): + for batch in _batches(rows, batch_size): if file_size is not None: bytes_consumed_so_far = fp.tell() new_bytes = bytes_consumed_so_far - bytes_so_far