From 44d7e2bf9e60222f57da2545c7bb0fb8b0120768 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 25 Feb 2024 11:05:13 -0800
Subject: [PATCH] Docs for streaming and --interval, closes #21

---
 dclient/cli.py    |  2 +-
 docs/inserting.md | 25 ++++++++++++++++++++++++-
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/dclient/cli.py b/dclient/cli.py
index 87d7bea..009fb9e 100644
--- a/dclient/cli.py
+++ b/dclient/cli.py
@@ -111,7 +111,7 @@ def query(url_or_alias, sql, token):
     "--batch-size", type=int, default=100, help="Send rows in batches of this size"
 )
 @click.option(
-    "--interval", type=float, default=None, help="Send batch at least every X seconds"
+    "--interval", type=float, default=10, help="Send batch at least every X seconds"
 )
 @click.option("--token", "-t", help="API token")
 @click.option("--silent", is_flag=True, help="Don't output progress")
diff --git a/docs/inserting.md b/docs/inserting.md
index e544aa6..fb01d47 100644
--- a/docs/inserting.md
+++ b/docs/inserting.md
@@ -7,7 +7,9 @@ First you'll need to {ref}`authenticate <authentication>` with the instance.
 To insert data from a `data.csv` file into a table called `my_table`, creating that table if it does not exist:
 
 ```bash
-dclient insert https://my-private-space.datasette.cloud/data my_table data.csv --create
+dclient insert \
+  https://my-private-space.datasette.cloud/data \
+  my_table data.csv --create
 ```
 You can also pipe data into standard input:
 ```bash
@@ -17,6 +19,26 @@ curl -s 'https://api.github.com/repos/simonw/dclient/issues' | \
     issues - --create
 ```
 
+## Streaming data
+
+`dclient insert` works for streaming data as well.
+
+If you have a log file containing newline-delimited JSON you can tail it and send it to a Datasette instance like this:
+```bash
+tail -f log.jsonl | \
+  dclient insert https://my-private-space.datasette.cloud/data logs - --nl
+```
+When reading from standard input (filename `-`) you are required to specify the format. In this example that's `--nl` for newline-delimited JSON. `--csv` and `--tsv` are supported for streaming as well, but `--json` is not.
+
+In streaming mode records default to being sent to the server every 100 records or every 10 seconds, whichever comes first. You can adjust these values using the `--batch-size` and `--interval` settings. For example, here's how to send every 10 records or if 5 seconds has passed since the last time data was sent to the server:
+
+```bash
+tail -f log.jsonl | dclient insert \
+  https://my-private-space.datasette.cloud/data logs - --nl --create \
+  --batch-size 10 \
+  --interval 5
+```
+
 ## Supported formats
 
 Data can be inserted from CSV, TSV, JSON or newline-delimited JSON files.
@@ -106,6 +128,7 @@ Options:
   --pk TEXT             Columns to use as the primary key when creating the
                         table
   --batch-size INTEGER  Send rows in batches of this size
+  --interval FLOAT      Send batch at least every X seconds
   -t, --token TEXT      API token
   --silent              Don't output progress
   --help                Show this message and exit.