remote/sqlite-utils
1
0
Fork 0
mirror of https://github.com/simonw/sqlite-utils.git synced 2026-07-26 02:44:33 +02:00
Code Issues Projects Releases Packages Wiki Activity

Refactor import_options and insert_upsert_options, refs #377

Browse source
This commit is contained in:
Simon Willison 2022-01-10 17:47:04 -08:00
commit de29a10c49
1 changed files with 48 additions and 86 deletions
Download patch file Download diff file Expand all files Collapse all files

134
sqlite_utils/cli.py
View file

@ -660,49 +660,46 @@ def reset_counts(path, load_extension):
db.reset_counts()
_import_options = (
click.option(
"--flatten",
is_flag=True,
help='Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1}',
),
click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"),
click.option("-c", "--csv", is_flag=True, help="Expect CSV input"),
click.option("--tsv", is_flag=True, help="Expect TSV input"),
click.option(
"--lines",
is_flag=True,
help="Treat each line as a single value called 'line'",
),
click.option(
"--text",
is_flag=True,
help="Treat input as a single value called 'text'",
),
click.option("--convert", help="Python code to convert each item"),
click.option(
"--import",
"imports",
type=str,
multiple=True,
help="Python modules to import",
),
click.option("--delimiter", help="Delimiter to use for CSV files"),
click.option("--quotechar", help="Quote character to use for CSV/TSV"),
click.option("--sniff", is_flag=True, help="Detect delimiter and quote character"),
click.option("--no-headers", is_flag=True, help="CSV file has no header row"),
click.option(
"--encoding",
help="Character encoding for input, defaults to utf-8",
),
)
def import_options(fn):
for decorator in reversed(
(
click.option(
"--flatten",
is_flag=True,
help='Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1}',
),
click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"),
click.option("-c", "--csv", is_flag=True, help="Expect CSV input"),
click.option("--tsv", is_flag=True, help="Expect TSV input"),
click.option(
"--lines",
is_flag=True,
help="Treat each line as a single value called 'line'",
),
click.option(
"--text",
is_flag=True,
help="Treat input as a single value called 'text'",
),
click.option("--convert", help="Python code to convert each item"),
click.option(
"--import",
"imports",
type=str,
multiple=True,
help="Python modules to import",
),
click.option("--delimiter", help="Delimiter to use for CSV files"),
click.option("--quotechar", help="Quote character to use for CSV/TSV"),
click.option(
"--sniff", is_flag=True, help="Detect delimiter and quote character"
),
click.option(
"--no-headers", is_flag=True, help="CSV file has no header row"
),
click.option(
"--encoding",
help="Character encoding for input, defaults to utf-8",
),
)
):
for decorator in reversed(_import_options):
fn = decorator(fn)
return fn
@ -720,40 +717,9 @@ def insert_upsert_options(fn):
click.option(
"--pk", help="Columns to use as the primary key, e.g. id", multiple=True
),
click.option(
"--flatten",
is_flag=True,
help='Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1}',
),
click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"),
click.option("-c", "--csv", is_flag=True, help="Expect CSV input"),
click.option("--tsv", is_flag=True, help="Expect TSV input"),
click.option(
"--lines",
is_flag=True,
help="Treat each line as a single value called 'line'",
),
click.option(
"--text",
is_flag=True,
help="Treat input as a single value called 'text'",
),
click.option("--convert", help="Python code to convert each item"),
click.option(
"--import",
"imports",
type=str,
multiple=True,
help="Python modules to import",
),
click.option("--delimiter", help="Delimiter to use for CSV files"),
click.option("--quotechar", help="Quote character to use for CSV/TSV"),
click.option(
"--sniff", is_flag=True, help="Detect delimiter and quote character"
),
click.option(
"--no-headers", is_flag=True, help="CSV file has no header row"
),
)
+ _import_options
+ (
click.option(
"--batch-size", type=int, default=100, help="Commit every X records"
),
@ -773,10 +739,6 @@ def insert_upsert_options(fn):
type=(str, str),
help="Default value that should be set for a column",
),
click.option(
"--encoding",
help="Character encoding for input, defaults to utf-8",
),
click.option(
"-d",
"--detect-types",
@ -814,6 +776,7 @@ def insert_upsert_implementation(
quotechar,
sniff,
no_headers,
encoding,
batch_size,
alter,
upsert,
@ -822,7 +785,6 @@ def insert_upsert_implementation(
truncate=False,
not_null=None,
default=None,
encoding=None,
detect_types=None,
analyze=False,
load_extension=None,
@ -1022,9 +984,9 @@ def insert(
quotechar,
sniff,
no_headers,
encoding,
batch_size,
alter,
encoding,
detect_types,
analyze,
load_extension,
@ -1074,13 +1036,13 @@ def insert(
quotechar,
sniff,
no_headers,
encoding,
batch_size,
alter=alter,
upsert=False,
ignore=ignore,
replace=replace,
truncate=truncate,
encoding=encoding,
detect_types=detect_types,
analyze=analyze,
load_extension=load_extension,
@ -1112,10 +1074,10 @@ def upsert(
quotechar,
sniff,
no_headers,
encoding,
alter,
not_null,
default,
encoding,
detect_types,
analyze,
load_extension,
@ -1144,12 +1106,12 @@ def upsert(
quotechar,
sniff,
no_headers,
encoding,
batch_size,
alter=alter,
upsert=True,
not_null=not_null,
default=default,
encoding=encoding,
detect_types=detect_types,
analyze=analyze,
load_extension=load_extension,
@ -1209,12 +1171,12 @@ def bulk(
quotechar=quotechar,
sniff=sniff,
no_headers=no_headers,
encoding=encoding,
batch_size=1,
alter=False,
upsert=False,
not_null=set(),
default={},
encoding=encoding,
detect_types=False,
load_extension=load_extension,
silent=False,