diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index cac9700..f8bbcd3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,18 +12,18 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Install dependencies run: | - pip install '.[test]' + pip install . --group dev - name: Run tests run: | pytest @@ -34,13 +34,13 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: - python-version: "3.12" + python-version: "3.14" cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Install dependencies run: | pip install setuptools wheel build diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bbd465f..ea123b5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,18 +10,18 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: pip - cache-dependency-path: setup.py + cache-dependency-path: pyproject.toml - name: Install dependencies run: | - pip install -e '.[test]' + pip install . --group dev - name: Run tests run: | pytest diff --git a/.gitignore b/.gitignore index 53605b7..60686dc 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ venv .pytest_cache *.egg-info .DS_Store +uv.lock +dist/ diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000..3bf34f3 --- /dev/null +++ b/PLAN.md @@ -0,0 +1,529 @@ +# dclient Improvement Plan + +## Feature Gap Analysis: dclient vs Datasette + +### What dclient currently supports + +| Feature | Status | +|---------|--------| +| SQL query execution (`query`) | Supported | +| Bulk data insertion (`insert`) | Supported (CSV, TSV, JSON, JSONL) | +| Actor/token introspection (`actor`) | Supported | +| URL aliases (`alias add/list/remove`) | Supported | +| Token storage (`auth add/list/remove`) | Supported | +| Bearer token authentication | Supported | +| Progress bar for inserts | Supported | +| Verbose/debug mode | Supported | +| Datasette plugin (`datasette dc`) | Supported | + +### What Datasette exposes that dclient does NOT cover + +#### Read Operations (high value) + +| Datasette Feature | API Endpoint | dclient Status | +|-------------------|-------------|----------------| +| List databases | `GET /-/databases.json` | **Missing** | +| List tables/views in a database | `GET /{db}.json` | **Missing** | +| Browse/filter table rows | `GET /{db}/{table}.json?filters...` | **Missing** | +| Get single row by PK | `GET /{db}/{table}/{pk}.json` | **Missing** | +| Table schema inspection | `GET /{db}/{table}/-/schema.json` | **Missing** | +| Database schema inspection | `GET /{db}/-/schema.json` | **Missing** | +| Full-text search | `?_search=term` | **Missing** | +| Column filtering (exact, gt, lt, contains, etc.) | `?col__op=val` | **Missing** | +| Faceted browsing | `?_facet=col` | **Missing** | +| Sorting | `?_sort=col` / `?_sort_desc=col` | **Missing** | +| Pagination | `?_next=cursor&_size=N` | **Missing** | +| Column selection | `?_col=a&_col=b` / `?_nocol=x` | **Missing** | +| CSV/TSV export of table data | `GET /{db}/{table}.csv` | **Missing** | +| Canned queries listing | metadata in `GET /{db}.json` | **Missing** | + +#### Write Operations (high value) + +| Datasette Feature | API Endpoint | dclient Status | +|-------------------|-------------|----------------| +| Row upsert (insert-or-update) | `POST /{db}/{table}/-/upsert` | **Missing** | +| Update a single row | `POST /{db}/{table}/{pk}/-/update` | **Missing** | +| Delete a single row | `POST /{db}/{table}/{pk}/-/delete` | **Missing** | +| Drop a table | `POST /{db}/{table}/-/drop` | **Missing** | +| Create empty table with schema | `POST /{db}/-/create` (columns only) | **Missing** | + +#### Metadata/Admin (medium value) + +| Datasette Feature | API Endpoint | dclient Status | +|-------------------|-------------|----------------| +| Instance version info | `GET /-/versions.json` | **Missing** | +| Installed plugins | `GET /-/plugins.json` | **Missing** | +| Instance settings | `GET /-/settings.json` | **Missing** | +| Token creation | `POST /-/create-token` | **Missing** | +| Permission checks | `GET /-/allowed.json` | **Missing** | + +--- + +## Proposed Plan + +The plan is organized into tiers by impact and complexity, with the most +valuable additions first. + +--- + +### Tier 1 -- Core Table Browsing & Schema Exploration + +These commands turn dclient from "a way to run raw SQL" into a genuine +database exploration tool. They make up the biggest gap today: a user who +wants to see what tables exist or browse rows must either open a browser +or hand-craft SQL. + +#### 1.1 `dclient databases` -- List databases + +``` +dclient databases +``` + +Hits `GET {url}/-/databases.json`. Prints a table of database names, sizes, +and whether they are mutable. Add `--json` for raw JSON output. + +#### 1.2 `dclient tables` -- List tables and views + +``` +dclient tables # e.g. https://latest.datasette.io/fixtures +dclient tables --views # include views +dclient tables --counts # include row counts +dclient tables --schema # include CREATE TABLE SQL +``` + +Hits `GET {db}.json`. Prints table names, row counts, and column lists. +`--json` for raw JSON. + +#### 1.3 `dclient schema` -- Show table/database schema + +``` +dclient schema # full database schema +dclient schema / # single table schema +``` + +Hits `GET {db}/-/schema.json` or `GET {db}/{table}/-/schema.json`. +Outputs the `CREATE TABLE` / `CREATE VIEW` SQL. + +#### 1.4 `dclient rows` -- Browse table rows with filtering + +This is the biggest single feature addition. It exposes Datasette's +powerful table filtering API without requiring the user to write SQL. + +``` +dclient rows [OPTIONS] +``` + +**Filter flags:** + +``` +--where 'column = value' # maps to ?column__exact=value +--where 'column > 100' # maps to ?column__gt=100 +--where 'column contains foo' # maps to ?column__contains=foo +--where 'column is null' # maps to ?column__isnull +--where-sql 'age > 30' # maps to ?_where=age > 30 (raw SQL) +``` + +Or a simpler column-value approach: + +``` +-w column=value # exact +-w column__gt=100 # pass Datasette filter operators directly +``` + +**Other flags:** + +``` +--sort column # ascending sort +--sort-desc column # descending sort +--search 'full text query' # FTS search (?_search=...) +--col name --col email # select specific columns +--nocol description # exclude columns +--facet column # facet counts +--size 50 # page size (default 100) +--all # auto-paginate through all pages +--limit N # stop after N total rows +--csv # output as CSV +--tsv # output as TSV +--nl # output as newline-delimited JSON +--json # output as JSON (default) +--table # human-readable table format +``` + +**Pagination:** +By default, print one page of results. `--all` follows `next_url` to +fetch every page. `--limit N` caps total rows returned. + +#### 1.5 `dclient get` -- Fetch a single row by primary key + +``` +dclient get +dclient get , # compound key +``` + +Hits `GET {db}/{table}/{pk}.json`. Prints the row as JSON. + +--- + +### Tier 2 -- Write Operations + +dclient already supports `insert` and `create`. These additions round out +the full CRUD lifecycle. + +#### 2.1 `dclient upsert` -- Insert or update rows + +``` +dclient upsert
[OPTIONS] +``` + +Same interface as `insert` but hits the `/-/upsert` endpoint. Requires +rows to contain primary key columns. + +Shares most code with `insert`; the main difference is the endpoint URL +and the absence of `--replace`/`--ignore` flags. + +#### 2.2 `dclient update` -- Update a single row + +``` +dclient update [ ...] +dclient update --input file.json +``` + +Hits `POST {db}/{table}/{pk}/-/update` with `{"update": {...}}`. + +#### 2.3 `dclient delete` -- Delete rows + +``` +dclient delete +dclient delete --yes # skip confirmation +``` + +Hits `POST {db}/{table}/{pk}/-/delete`. Prompts for confirmation unless +`--yes` is passed. + +#### 2.4 `dclient drop` -- Drop a table + +``` +dclient drop +dclient drop --yes # skip confirmation +``` + +Hits `POST {db}/{table}/-/drop` with `{"confirm": true}`. Shows row count +and asks for confirmation unless `--yes`. + +#### 2.5 `dclient create-table` -- Create an empty table with explicit schema + +``` +dclient create-table \ + --column id integer \ + --column name text \ + --column score float \ + --pk id +``` + +Hits `POST {db}/-/create` with a `columns` array instead of `rows`. +The existing `insert --create` handles creation-with-data; this handles +the schema-only case. + +--- + +### Tier 3 -- Output & UX Improvements + +#### 3.1 Multiple output formats for `query` and `rows` + +Currently `query` only outputs JSON. Add: + +``` +--csv # CSV output +--tsv # TSV output +--nl # newline-delimited JSON +--table # human-readable ASCII table (like sqlite-utils) +--yaml # YAML output (optional, low priority) +``` + +For `--table` output, use a simple column-aligned format or integrate +with `tabulate` / `rich` (consider keeping dependencies minimal). + +A lightweight approach: `sqlite-utils` already has table-formatting +utilities that could be reused. + +#### 3.2 Pipe-friendly defaults + +Detect whether stdout is a TTY: +- **TTY**: default to `--table` (human-readable) output +- **Pipe**: default to JSON (machine-readable) output + +This matches the UX convention of tools like `gh` and `jq`. + +#### 3.3 `--output` / `-o` flag for writing to a file + +``` +dclient query ... -o results.csv --csv +dclient rows ... -o dump.json +``` + +#### 3.4 Streaming output for large result sets + +For `--all` pagination mode and `--csv`/`--nl` formats, stream rows as +they arrive rather than buffering everything in memory. Write each page +to stdout immediately. + +--- + +### Tier 4 -- Authentication Improvements + +The current auth system works but has friction points. These changes +make it smoother. + +#### 4.1 `dclient auth login` -- Browser-based authentication + +``` +dclient auth login +``` + +Flow: +1. Open the user's browser to `{instance}/-/create-token` +2. Start a temporary local HTTP server to receive the token callback +3. User creates a token in the Datasette UI and pastes it, OR: +4. If the instance supports it, redirect back to the local server with the token +5. Store the token automatically via the existing auth system + +For instances that don't support redirect, fall back to: +1. Open the browser to `/-/create-token` +2. Prompt the user to paste the token into the terminal + +This is similar to how `gh auth login`, `gcloud auth login`, and +`heroku login` work. + +#### 4.2 `dclient auth token` -- Create a token via the API + +``` +dclient auth token +dclient auth token --expires-after 3600 +dclient auth token --read-only +``` + +Hits `POST /-/create-token`. Requires an existing root token or session. +Prints the new token and optionally stores it. + +#### 4.3 `dclient auth status` -- Validate stored credentials + +``` +dclient auth status +``` + +Hits `/-/actor.json` with the stored token and prints: +- Whether the token is valid +- The actor identity (id, permissions) +- Token expiry if available + +This is just a friendlier wrapper around the existing `actor` command, +integrated into the `auth` subgroup. + +#### 4.4 Environment variable support + +``` +export DCLIENT_TOKEN=dstok_xxx +export DCLIENT_URL=https://my-datasette.example.com/mydb +``` + +Token resolution order (highest to lowest priority): +1. `--token` CLI flag +2. `DCLIENT_TOKEN` environment variable +3. Stored token in `auth.json` (existing behavior) + +The `DCLIENT_URL` variable provides a default instance so users don't +have to type it every time: + +``` +export DCLIENT_URL=https://my-datasette.example.com/mydb +dclient tables # uses DCLIENT_URL +dclient query 'select 1' # uses DCLIENT_URL +dclient rows mytable --limit 10 # uses DCLIENT_URL +``` + +#### 4.5 Keyring integration (optional, lower priority) + +Instead of storing tokens in plaintext `auth.json`, optionally use the +system keyring via the `keyring` Python package. This keeps tokens +encrypted at rest on macOS (Keychain), Windows (Credential Vault), and +Linux (Secret Service / KWallet). + +Make this opt-in: `dclient auth add --keyring `. Fall back to the +current file-based storage if `keyring` is not installed. + +--- + +### Tier 5 -- Discoverability & Convenience + +#### 5.1 `dclient info` -- Instance overview + +``` +dclient info +``` + +Hits `/-/versions.json`, `/-/plugins.json`, `/-/databases.json` in +parallel. Prints a summary: + +``` +Datasette 1.0.1 +Python 3.12.1 +SQLite 3.45.0 +Databases: 3 (fixtures, content, ephemeral) +Plugins: 12 installed +``` + +#### 5.2 URL inference and shortcuts + +Allow shorthand for common patterns: + +``` +# These should all work equivalently: +dclient rows https://latest.datasette.io/fixtures/facetable +dclient rows latest fixtures/facetable # if 'latest' is an alias +dclient rows latest facetable # if alias points to a database URL +``` + +The key insight: if an alias points to `https://x.com/dbname`, then +`dclient rows
` should construct +`https://x.com/dbname/table.json`. + +Currently the alias is just a URL prefix that gets used verbatim. Making +the alias system slightly smarter about joining paths would reduce typing +significantly. + +#### 5.3 `dclient config` -- Manage configuration + +``` +dclient config show # print config directory and contents +dclient config path # print config directory path +dclient config edit # open config in $EDITOR +``` + +Useful for debugging when tokens or aliases aren't working as expected. + +#### 5.4 Shell completions + +Use Click's built-in shell completion support to generate completions +for bash, zsh, and fish: + +``` +dclient --install-completion bash +dclient --install-completion zsh +dclient --install-completion fish +``` + +Click 8.x supports this natively via `shell_complete`. This could also +dynamically complete alias names, database names, and table names by +querying the stored aliases. + +--- + +### Tier 6 -- Advanced Features (lower priority) + +#### 6.1 `dclient export` -- Bulk export table data + +``` +dclient export -o data.csv --csv +dclient export -o data.json --json +dclient export -o data.db # export to local SQLite +``` + +This is essentially `dclient rows --all` but optimized for bulk export: +- Streams using `?_stream=1` (CSV format) when available +- For JSON, auto-paginates using `_next` cursor +- For SQLite output, pipes into `sqlite-utils insert` + +#### 6.2 Parallel insert for large files + +When inserting very large files, optionally send batches in parallel: + +``` +dclient insert
huge.csv --parallel 4 +``` + +Uses a thread pool to send N batches concurrently. Would require +switching from synchronous `httpx` to `httpx.AsyncClient` or using +`concurrent.futures.ThreadPoolExecutor`. + +#### 6.3 `dclient diff` -- Compare local and remote data + +``` +dclient diff local.db remote_alias/tablename +``` + +Compare a local SQLite table with a remote Datasette table. Show added, +removed, and modified rows. Useful for sync workflows. + +--- + +## Implementation Notes + +### Shared infrastructure to build first + +Several of the proposed commands share common patterns. Before +implementing individual commands, refactor these into shared utilities: + +1. **URL resolution helper**: Centralize the alias-lookup + URL-construction + logic that is currently duplicated in every command. A single function + like `resolve_url(url_or_alias, suffix=None)` that handles aliases, + adds `.json` where appropriate, and joins path segments. + +2. **Authenticated HTTP client helper**: A function or context manager + that creates an `httpx.Client` with the right `Authorization` header + and timeout, resolved from `--token` / env var / auth.json. This + eliminates the repeated token-resolution boilerplate in every command. + +3. **Output formatter**: A shared output module that handles `--json`, + `--csv`, `--tsv`, `--nl`, `--table` flags consistently across all + commands that return data. + +4. **Pagination helper**: A generator that follows `next_url` links to + yield all pages of results, used by `rows --all` and `export`. + +### Dependency considerations + +The current dependencies are minimal: `click`, `httpx`, `sqlite-utils`. +The proposed changes mostly stay within these. Potential additions: + +- `tabulate` or `rich` for `--table` output (or implement a minimal + version to avoid new deps) +- `keyring` for optional encrypted token storage (optional dependency) + +### Backward compatibility + +All proposed changes are additive (new commands and new flags). No +existing command signatures or behaviors change. The only subtle change +would be if TTY-detection changes the default output format of `query`, +which should be gated behind a major version bump or opt-in flag. + +### Testing strategy + +Each new command should follow the existing test patterns: +- Unit tests with `pytest-httpx` for HTTP mocking +- Integration tests against a real in-memory Datasette instance + (as done in `test_insert.py`) +- `cogapp` for keeping `--help` output in docs in sync + +--- + +## Priority Summary + +| Priority | Feature | Rationale | +|----------|---------|-----------| +| **P0** | `databases`, `tables`, `schema` | Basic exploration; most-requested gap | +| **P0** | `rows` with filtering/sorting/pagination | Replaces need for raw SQL for common tasks | +| **P0** | Environment variable support (`DCLIENT_TOKEN`, `DCLIENT_URL`) | Near-zero effort, big UX win | +| **P1** | `get`, `update`, `delete`, `drop` | Completes CRUD lifecycle | +| **P1** | `upsert` | Natural complement to existing `insert` | +| **P1** | Multiple output formats (CSV, table, NL) | Makes `query`/`rows` output usable in pipelines | +| **P1** | `auth login` (browser-based) | Biggest auth UX improvement | +| **P2** | `create-table` (schema only) | Useful but `insert --create` covers most cases | +| **P2** | TTY-aware default output | Polish | +| **P2** | `auth status`, `auth token` | Convenience wrappers | +| **P2** | `info` | Nice discoverability | +| **P2** | URL inference / smarter aliases | Reduces typing | +| **P3** | `export` (bulk) | Covered by `rows --all` for most cases | +| **P3** | Shell completions | Polish | +| **P3** | Keyring integration | Security hardening | +| **P3** | Parallel insert | Performance optimization | +| **P3** | `diff` | Advanced workflow | diff --git a/README.md b/README.md index ec1925a..a0214fd 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,15 @@ Much of the functionality requires Datasette 1.0a2 or higher. ## Things you can do with dclient -- Run SQL queries against Datasette and returning the results as JSON +- **Explore** databases, tables, schemas, and rows without writing SQL +- **Query** with SQL and get results as JSON, CSV, TSV, or formatted tables +- **Insert** data from CSV, TSV, JSON, or newline-delimited JSON files +- **Upsert** data (insert or update based on primary key) +- **Update** individual rows by primary key +- **Delete** rows or **drop** entire tables +- **Create tables** with explicit schemas - Run queries against authenticated Datasette instances -- Create aliases and store authentication tokens for convenient access to Datasette -- Insert data into Datasette using the [insert API](https://docs.datasette.io/en/latest/json_api.html#the-json-write-api) (Datasette 1.0 alpha or higher) +- Create aliases and store authentication tokens for convenient access ## Installation @@ -26,36 +31,67 @@ If you want to install it in the same virtual environment as Datasette (to use i ```bash datasette install dclient ``` -## Running a query + +## Quick start ```bash +# Explore a Datasette instance +dclient databases https://latest.datasette.io +dclient tables https://latest.datasette.io/fixtures +dclient schema https://latest.datasette.io/fixtures + +# Browse rows with filtering and sorting +dclient rows https://latest.datasette.io/fixtures/facetable --table +dclient rows https://latest.datasette.io/fixtures/facetable -w state=CA --sort city + +# Fetch a single row by primary key +dclient get https://latest.datasette.io/fixtures/facetable 1 + +# Run a SQL query dclient query https://latest.datasette.io/fixtures "select * from facetable limit 1" ``` -To shorten that, create an alias: + +For write operations (requires authentication): + +```bash +# Create a table +dclient create-table https://example.com/db mytable \ + --column id integer --column name text --pk id + +# Insert data from a CSV file +dclient insert https://example.com/db mytable data.csv --create + +# Update a row +dclient update https://example.com/db/mytable 42 name=Alice age=30 + +# Upsert from a JSON file +dclient upsert https://example.com/db mytable data.json --json + +# Delete a row +dclient delete https://example.com/db/mytable 42 --yes + +# Drop a table +dclient drop https://example.com/db/mytable --yes +``` + +To shorten URLs, create an alias: ```bash dclient alias add fixtures https://latest.datasette.io/fixtures -``` -Then run it like this instead: -```bash dclient query fixtures "select * from facetable limit 1" ``` + ## Documentation Visit **[dclient.datasette.io](https://dclient.datasette.io)** for full documentation on using this tool. ## Development -To contribute to this tool, first checkout the code. Then create a new virtual environment: +To contribute to this tool, first checkout the code. Then install dependencies and run tests using [uv](https://docs.astral.sh/uv/): ```bash cd dclient -python -m venv venv -source venv/bin/activate +uv run pytest ``` -Now install the dependencies and test dependencies: +To build the package: ```bash -pip install -e '.[test]' -``` -To run the tests: -```bash -pytest +uv build ``` diff --git a/dclient/cli.py b/dclient/cli.py index a8b3376..678d618 100644 --- a/dclient/cli.py +++ b/dclient/cli.py @@ -1,5 +1,7 @@ import click +import csv import httpx +import io import json import pathlib from sqlite_utils.utils import rows_from_file, Format, TypeTracker, progressbar @@ -14,10 +16,234 @@ def get_config_dir(): return pathlib.Path(click.get_app_dir("io.datasette.dclient")) +# --- Shared infrastructure --- + + +def _load_aliases(aliases_file): + if aliases_file.exists(): + aliases = json.loads(aliases_file.read_text()) + else: + aliases = {} + return aliases + + +def _load_auths(auth_file): + if auth_file.exists(): + auths = json.loads(auth_file.read_text()) + else: + auths = {} + return auths + + +def _resolve_url(url_or_alias): + """Resolve an alias to a URL, or return the input unchanged.""" + aliases_file = get_config_dir() / "aliases.json" + aliases = _load_aliases(aliases_file) + if url_or_alias in aliases: + return aliases[url_or_alias] + return url_or_alias + + +def _resolve_token(url, token=None): + """Resolve a token from CLI arg, env var, or auth.json.""" + if token: + return token + env_token = click.get_current_context().obj.get("env_token") if click.get_current_context().obj else None + if env_token: + return env_token + return token_for_url(url, _load_auths(get_config_dir() / "auth.json")) + + +def _make_headers(token): + """Build request headers with optional auth.""" + headers = {} + if token: + headers["Authorization"] = "Bearer {}".format(token) + return headers + + +def _api_get(url, token=None, params=None, verbose=False): + """Make an authenticated GET request and return parsed JSON.""" + headers = _make_headers(token) + if verbose: + full_url = url + if params: + full_url += "?" + urllib.parse.urlencode(params, doseq=True) + click.echo(full_url, err=True) + response = httpx.get(url, params=params, headers=headers, timeout=40.0, follow_redirects=True) + if response.status_code != 200: + _raise_api_error(response) + try: + data = response.json() + except json.JSONDecodeError: + raise click.ClickException("Response was not valid JSON") + if isinstance(data, dict) and data.get("ok") is False: + _raise_from_data(data) + return data + + +def _api_post(url, data, token=None, verbose=False): + """Make an authenticated POST request and return parsed JSON.""" + headers = _make_headers(token) + headers["Content-Type"] = "application/json" + if verbose: + click.echo("POST {}".format(url), err=True) + click.echo(textwrap.indent(json.dumps(data, indent=2), " "), err=True) + response = httpx.post(url, headers=headers, json=data, timeout=40.0) + if verbose: + click.echo(str(response), err=True) + if str(response.status_code)[0] != "2": + if "json" in response.headers.get("content-type", ""): + resp_data = response.json() + if "errors" in resp_data: + raise click.ClickException("\n".join(resp_data["errors"])) + _raise_from_data(resp_data) + response.raise_for_status() + response_data = response.json() + if verbose: + click.echo(textwrap.indent(json.dumps(response_data, indent=2), " "), err=True) + return response_data + + +def _raise_api_error(response): + """Raise a ClickException from a non-200 response.""" + try: + data = response.json() + except json.JSONDecodeError: + raise click.ClickException( + "{} status code. Response was not valid JSON".format(response.status_code) + ) + bits = [] + if data.get("title"): + bits.append(data["title"]) + if data.get("error"): + bits.append(data["error"]) + raise click.ClickException( + "{} status code. {}".format(response.status_code, ": ".join(bits)) + ) + + +def _raise_from_data(data): + """Raise a ClickException from a JSON error response.""" + bits = [] + if data.get("title"): + bits.append(data["title"]) + if data.get("error"): + bits.append(data["error"]) + if data.get("errors"): + bits.extend(data["errors"]) + if not bits: + bits = [json.dumps(data)] + raise click.ClickException(": ".join(bits)) + + +def _output_rows(rows, output_format, nl=False, cols=None): + """Output rows in the specified format.""" + if output_format == "json": + click.echo(json.dumps(rows, indent=2)) + elif output_format == "nl": + for row in rows: + click.echo(json.dumps(row)) + elif output_format == "csv": + _output_csv(rows, cols) + elif output_format == "tsv": + _output_csv(rows, cols, delimiter="\t") + elif output_format == "table": + _output_table(rows, cols) + else: + click.echo(json.dumps(rows, indent=2)) + + +def _output_csv(rows, cols=None, delimiter=","): + """Write rows as CSV/TSV to stdout.""" + if not rows: + return + if cols is None: + cols = list(rows[0].keys()) + writer = csv.DictWriter(sys.stdout, fieldnames=cols, delimiter=delimiter, extrasaction="ignore") + writer.writeheader() + writer.writerows(rows) + + +def _output_table(rows, cols=None): + """Write rows as a simple aligned text table.""" + if not rows: + return + if cols is None: + cols = list(rows[0].keys()) + # Calculate column widths + widths = {col: len(col) for col in cols} + str_rows = [] + for row in rows: + str_row = {} + for col in cols: + val = str(row.get(col, "")) if row.get(col) is not None else "" + str_row[col] = val + widths[col] = max(widths[col], len(val)) + str_rows.append(str_row) + # Header + header = " ".join(col.ljust(widths[col]) for col in cols) + click.echo(header) + click.echo(" ".join("-" * widths[col] for col in cols)) + for str_row in str_rows: + click.echo(" ".join(str_row[col].ljust(widths[col]) for col in cols)) + + +def _determine_format(fmt_csv, fmt_tsv, fmt_json, fmt_nl, fmt_table, default="json"): + """Determine output format from flags.""" + if fmt_csv: + return "csv" + if fmt_tsv: + return "tsv" + if fmt_json: + return "json" + if fmt_nl: + return "nl" + if fmt_table: + return "table" + return default + + +# --- CLI --- + + @click.group() @click.version_option() -def cli(): +@click.pass_context +def cli(ctx): "A client CLI utility for Datasette instances" + ctx.ensure_object(dict) + import os + + ctx.obj["env_token"] = os.environ.get("DCLIENT_TOKEN") + ctx.obj["env_url"] = os.environ.get("DCLIENT_URL") + + +def _resolve_url_with_env(url_or_alias): + """Resolve URL, falling back to DCLIENT_URL env var.""" + if url_or_alias: + return _resolve_url(url_or_alias) + ctx = click.get_current_context() + env_url = ctx.obj.get("env_url") if ctx.obj else None + if env_url: + return env_url + raise click.ClickException("No URL provided and DCLIENT_URL not set") + + +# --- Output format options (shared decorator) --- + + +def output_format_options(f): + """Add --csv, --tsv, --json, --nl, --table output options.""" + f = click.option("fmt_table", "--table", is_flag=True, help="Output as text table")(f) + f = click.option("fmt_nl", "--nl", is_flag=True, help="Output as newline-delimited JSON")(f) + f = click.option("fmt_json", "--json", is_flag=True, help="Output as JSON (default)")(f) + f = click.option("fmt_tsv", "--tsv", is_flag=True, help="Output as TSV")(f) + f = click.option("fmt_csv", "--csv", is_flag=True, help="Output as CSV")(f) + return f + + +# --- query command --- @cli.command() @@ -25,7 +251,9 @@ def cli(): @click.argument("sql") @click.option("--token", help="API token") @click.option("-v", "--verbose", is_flag=True, help="Verbose output: show HTTP request") -def query(url_or_alias, sql, token, verbose): +@output_format_options +@click.pass_context +def query(ctx, url_or_alias, sql, token, verbose, fmt_csv, fmt_tsv, fmt_json, fmt_nl, fmt_table): """ Run a SQL query against a Datasette database URL @@ -38,62 +266,277 @@ def query(url_or_alias, sql, token, verbose): https://datasette.io/content \\ 'select * from news limit 10' """ - aliases_file = get_config_dir() / "aliases.json" - aliases = _load_aliases(aliases_file) - if url_or_alias in aliases: - url = aliases[url_or_alias] - else: - url = url_or_alias - if not url_or_alias.endswith(".json"): + url = _resolve_url_with_env(url_or_alias) + if not url.endswith(".json"): url += ".json" - if token is None: - # Maybe there's a token in auth.json? - token = token_for_url(url, _load_auths(get_config_dir() / "auth.json")) - headers = {} - if token: - headers["Authorization"] = f"Bearer {token}" - params = {"sql": sql, "_shape": "objects"} - if verbose: - click.echo(url + "?" + urllib.parse.urlencode(params), err=True) - response = httpx.get(url, params=params, headers=headers) + token = _resolve_token(url, token) + data = _api_get(url, token=token, params={"sql": sql, "_shape": "objects"}, verbose=verbose) + output_format = _determine_format(fmt_csv, fmt_tsv, fmt_json, fmt_nl, fmt_table) + _output_rows(data["rows"], output_format) - if response.status_code != 200: - # Is it valid JSON? - try: - data = response.json() - except json.JSONDecodeError: - raise click.ClickException( - "{} status code. Response was not valid JSON".format( - response.status_code - ) - ) - bits = [] - if data.get("title"): - bits.append(data["title"]) - if data.get("error"): - bits.append(data["error"]) - raise click.ClickException( - "{} status code. {}".format(response.status_code, ": ".join(bits)) - ) - # We should have JSON now - try: - data = response.json() - except json.JSONDecodeError: - raise click.ClickException("Response was not valid JSON") - # ... but it may have a {"ok": false} error - if not data.get("ok"): - bits = [] - if data.get("title"): - bits.append(data["title"]) - if data.get("error"): - bits.append(data["error"]) - if not bits: - bits = [json.dumps(data)] - raise click.ClickException(": ".join(bits)) +# --- databases command --- - # Output results - click.echo(json.dumps(response.json()["rows"], indent=2)) + +@cli.command() +@click.argument("url_or_alias", required=False) +@click.option("--token", help="API token") +@click.option("-v", "--verbose", is_flag=True, help="Verbose output") +@output_format_options +@click.pass_context +def databases(ctx, url_or_alias, token, verbose, fmt_csv, fmt_tsv, fmt_json, fmt_nl, fmt_table): + """ + List databases on a Datasette instance + + Example usage: + + \b + dclient databases https://latest.datasette.io + """ + url = _resolve_url_with_env(url_or_alias) + # Ensure we hit the instance root + api_url = url.rstrip("/") + "/-/databases.json" + token = _resolve_token(url, token) + data = _api_get(api_url, token=token, verbose=verbose) + output_format = _determine_format(fmt_csv, fmt_tsv, fmt_json, fmt_nl, fmt_table) + if output_format == "table": + rows = [] + for db in data: + rows.append({ + "name": db["name"], + "path": db.get("path", ""), + "is_mutable": db.get("is_mutable", ""), + }) + _output_rows(rows, "table") + elif output_format == "json": + click.echo(json.dumps(data, indent=2)) + else: + _output_rows(data, output_format) + + +# --- tables command --- + + +@cli.command() +@click.argument("url_or_alias", required=False) +@click.option("--token", help="API token") +@click.option("--views", is_flag=True, help="Include views") +@click.option("--schema", is_flag=True, help="Show CREATE TABLE SQL") +@click.option("-v", "--verbose", is_flag=True, help="Verbose output") +@output_format_options +@click.pass_context +def tables(ctx, url_or_alias, token, views, schema, verbose, fmt_csv, fmt_tsv, fmt_json, fmt_nl, fmt_table): + """ + List tables in a Datasette database + + Example usage: + + \b + dclient tables https://latest.datasette.io/fixtures + """ + url = _resolve_url_with_env(url_or_alias) + api_url = url.rstrip("/") + ".json" + token = _resolve_token(url, token) + data = _api_get(api_url, token=token, verbose=verbose) + output_format = _determine_format(fmt_csv, fmt_tsv, fmt_json, fmt_nl, fmt_table) + + result_tables = data.get("tables", []) if isinstance(data, dict) else data + if not views: + result_tables = [t for t in result_tables if not t.get("hidden")] + + if schema: + # Fetch schema for the database + schema_url = url.rstrip("/") + "/-/schema.json" + schema_data = _api_get(schema_url, token=token, verbose=verbose) + schema_map = {} + if isinstance(schema_data, dict): + for item in schema_data.get("tables", []): + schema_map[item["name"]] = item.get("schema", "") + for t in result_tables: + t["schema"] = schema_map.get(t["name"], "") + + if output_format == "json": + click.echo(json.dumps(result_tables, indent=2)) + elif output_format == "table": + rows = [] + for t in result_tables: + row = { + "name": t["name"], + "rows": str(t.get("count", t.get("rows_count", ""))), + "columns": ", ".join(t.get("columns", [])), + } + if schema: + row["schema"] = t.get("schema", "") + rows.append(row) + _output_rows(rows, "table") + else: + _output_rows(result_tables, output_format) + + +# --- schema command --- + + +@cli.command() +@click.argument("url_or_alias", required=False) +@click.option("--token", help="API token") +@click.option("-v", "--verbose", is_flag=True, help="Verbose output") +@click.pass_context +def schema(ctx, url_or_alias, token, verbose): + """ + Show the SQL schema for a database or table + + Example usage: + + \b + dclient schema https://latest.datasette.io/fixtures + dclient schema https://latest.datasette.io/fixtures/facetable + """ + url = _resolve_url_with_env(url_or_alias) + token = _resolve_token(url, token) + schema_url = url.rstrip("/") + "/-/schema.json" + data = _api_get(schema_url, token=token, verbose=verbose) + if isinstance(data, dict) and "tables" in data: + for t in data["tables"]: + click.echo(t.get("schema", "") + ";") + click.echo() + elif isinstance(data, dict) and "schema" in data: + click.echo(data["schema"] + ";") + else: + click.echo(json.dumps(data, indent=2)) + + +# --- rows command --- + + +@cli.command() +@click.argument("url_or_alias", required=False) +@click.option("--token", help="API token") +@click.option("-w", "--where", "where_clauses", multiple=True, help="Filter: column__op=value (e.g. age__gt=30)") +@click.option("--where-sql", "where_sql", multiple=True, help="Raw SQL WHERE clause") +@click.option("--search", help="Full-text search query") +@click.option("--sort", help="Sort by column (ascending)") +@click.option("--sort-desc", help="Sort by column (descending)") +@click.option("--col", "columns", multiple=True, help="Include only these columns") +@click.option("--nocol", "nocolumns", multiple=True, help="Exclude these columns") +@click.option("--facet", "facets", multiple=True, help="Facet by column") +@click.option("--size", type=int, help="Number of rows per page") +@click.option("--all", "fetch_all", is_flag=True, help="Fetch all pages") +@click.option("--limit", type=int, help="Maximum total rows to return") +@click.option("-v", "--verbose", is_flag=True, help="Verbose output") +@output_format_options +@click.pass_context +def rows(ctx, url_or_alias, token, where_clauses, where_sql, search, sort, + sort_desc, columns, nocolumns, facets, size, fetch_all, limit, verbose, + fmt_csv, fmt_tsv, fmt_json, fmt_nl, fmt_table): + """ + Browse rows in a Datasette table with filtering and sorting + + Example usage: + + \b + dclient rows https://latest.datasette.io/fixtures/facetable + dclient rows https://latest.datasette.io/fixtures/facetable -w state=CA + dclient rows https://latest.datasette.io/fixtures/facetable --sort city + dclient rows https://latest.datasette.io/fixtures/facetable --search text + """ + url = _resolve_url_with_env(url_or_alias) + token = _resolve_token(url, token) + output_format = _determine_format(fmt_csv, fmt_tsv, fmt_json, fmt_nl, fmt_table) + + # Build query params + params = [("_shape", "objects"), ("_extra", "next_url")] + for w in where_clauses: + if "=" in w: + key, _, value = w.partition("=") + params.append((key, value)) + else: + params.append(("_where", w)) + for w in where_sql: + params.append(("_where", w)) + if search: + params.append(("_search", search)) + if sort: + params.append(("_sort", sort)) + if sort_desc: + params.append(("_sort_desc", sort_desc)) + for col in columns: + params.append(("_col", col)) + for col in nocolumns: + params.append(("_nocol", col)) + for facet in facets: + params.append(("_facet", facet)) + if size: + params.append(("_size", str(size))) + + api_url = url.rstrip("/") + ".json" + all_rows = [] + total = 0 + page_url = api_url + + while True: + data = _api_get(page_url, token=token, params=params if page_url == api_url else None, verbose=verbose) + page_rows = data.get("rows", []) + if limit: + remaining = limit - total + page_rows = page_rows[:remaining] + all_rows.extend(page_rows) + total += len(page_rows) + + if limit and total >= limit: + break + + next_url = data.get("next_url") + if not fetch_all or not next_url: + break + page_url = next_url + params = None # next_url includes params + + # Show facets if requested + facet_results = data.get("facet_results") if facets else None + if facet_results: + click.echo("Facets:", err=True) + for facet_name, facet_info in facet_results.items(): + click.echo(" {}:".format(facet_name), err=True) + results_list = facet_info.get("results", facet_info) if isinstance(facet_info, dict) else facet_info + if isinstance(results_list, list): + for item in results_list: + click.echo(" {} ({})".format(item.get("value", ""), item.get("count", "")), err=True) + click.echo(err=True) + + _output_rows(all_rows, output_format) + + +# --- get command --- + + +@cli.command() +@click.argument("url_or_alias", required=False) +@click.argument("pk_values") +@click.option("--token", help="API token") +@click.option("-v", "--verbose", is_flag=True, help="Verbose output") +@click.pass_context +def get(ctx, url_or_alias, pk_values, token, verbose): + """ + Fetch a single row by primary key + + Example usage: + + \b + dclient get https://latest.datasette.io/fixtures/facetable 1 + dclient get https://latest.datasette.io/fixtures/compound_pk a,b + """ + url = _resolve_url_with_env(url_or_alias) + token = _resolve_token(url, token) + api_url = url.rstrip("/") + "/" + pk_values + ".json" + data = _api_get(api_url, token=token, params={"_shape": "objects"}, verbose=verbose) + rows = data.get("rows", [data] if isinstance(data, dict) else data) + if rows: + click.echo(json.dumps(rows[0], indent=2)) + else: + raise click.ClickException("Row not found") + + +# --- insert command --- @cli.command() @@ -167,15 +610,8 @@ def insert( https://private.datasette.cloud/data \\ mytable data.csv --pk id --create """ - aliases_file = get_config_dir() / "aliases.json" - aliases = _load_aliases(aliases_file) - if url_or_alias in aliases: - url = aliases[url_or_alias] - else: - url = url_or_alias - - if token is None: - token = token_for_url(url, _load_auths(get_config_dir() / "auth.json")) + url = _resolve_url(url_or_alias) + token = _resolve_token(url, token) format = None if format_csv: @@ -205,7 +641,6 @@ def insert( raise click.ClickException(str(ex)) if format in (Format.JSON, Format.NL): - # Disable progress bar - it can't handle these formats file_size = None no_detect_types = True @@ -226,15 +661,12 @@ def insert( bar.update(new_bytes) bytes_so_far += new_bytes except ValueError: - # File has likely been closed, so fp.tell() fails pass types = None if first and not no_detect_types: - # Detect types on first batch tracker = TypeTracker() list(tracker.wrap(batch)) types = tracker.types - # Convert types for row in batch: for key, value in row.items(): if value is None: @@ -264,10 +696,311 @@ def insert( ) +# --- upsert command --- + + +@cli.command() +@click.argument("url_or_alias") +@click.argument("table") +@click.argument( + "filepath", type=click.Path("rb", readable=True, allow_dash=True, dir_okay=False) +) +@click.option("format_csv", "--csv", is_flag=True, help="Input is CSV") +@click.option("format_tsv", "--tsv", is_flag=True, help="Input is TSV") +@click.option("format_json", "--json", is_flag=True, help="Input is JSON") +@click.option("format_nl", "--nl", is_flag=True, help="Input is newline-delimited JSON") +@click.option("--encoding", help="Character encoding for CSV/TSV") +@click.option( + "--no-detect-types", is_flag=True, help="Don't detect column types for CSV/TSV" +) +@click.option("--alter", is_flag=True, help="Alter table to add any missing columns") +@click.option( + "--batch-size", type=int, default=100, help="Send rows in batches of this size" +) +@click.option( + "--interval", type=float, default=10, help="Send batch at least every X seconds" +) +@click.option("--token", "-t", help="API token") +@click.option("--silent", is_flag=True, help="Don't output progress") +@click.option( + "-v", + "--verbose", + is_flag=True, + help="Verbose output: show HTTP request and response", +) +def upsert( + url_or_alias, + table, + filepath, + format_csv, + format_tsv, + format_json, + format_nl, + encoding, + no_detect_types, + alter, + batch_size, + interval, + token, + silent, + verbose, +): + """ + Upsert data into a remote Datasette table + + Rows with matching primary keys will be updated; others will be inserted. + Each row must include the primary key column(s). + + Example usage: + + \b + dclient upsert \\ + https://private.datasette.cloud/data \\ + mytable data.csv + """ + url = _resolve_url(url_or_alias) + token = _resolve_token(url, token) + + format = None + if format_csv: + format = Format.CSV + elif format_tsv: + format = Format.TSV + elif format_json: + format = Format.JSON + elif format_nl: + format = Format.NL + if format is None and filepath == "-": + raise click.ClickException( + "An explicit format is required - e.g. --csv " + "- when reading from standard input" + ) + + if filepath != "-": + file_size = pathlib.Path(filepath).stat().st_size + fp = open(filepath, "rb") + else: + fp = sys.stdin.buffer + file_size = None + + try: + rows, format = rows_from_file(fp, format=format, encoding=encoding) + except Exception as ex: + raise click.ClickException(str(ex)) + + if format in (Format.JSON, Format.NL): + file_size = None + no_detect_types = True + + first = True + + with progressbar( + length=file_size, + label="Upserting rows", + silent=silent or (file_size is None), + show_percent=True, + ) as bar: + bytes_so_far = 0 + for batch in _batches(rows, batch_size, interval=interval): + if file_size is not None: + try: + bytes_consumed_so_far = fp.tell() + new_bytes = bytes_consumed_so_far - bytes_so_far + bar.update(new_bytes) + bytes_so_far += new_bytes + except ValueError: + pass + if first and not no_detect_types: + tracker = TypeTracker() + list(tracker.wrap(batch)) + types = tracker.types + for row in batch: + for key, value in row.items(): + if value is None: + continue + if types[key] == "integer": + if not value: + row[key] = None + else: + row[key] = int(value) + elif types[key] == "float": + if not value: + row[key] = None + else: + row[key] = float(value) + first = False + data = {"rows": batch} + if alter: + data["alter"] = True + upsert_url = "{}/{}/-/upsert".format(url, table) + _api_post(upsert_url, data, token=token, verbose=verbose) + + +# --- update command --- + + +@cli.command() +@click.argument("url_or_alias") +@click.argument("pk_values") +@click.argument("updates", nargs=-1) +@click.option("--token", help="API token") +@click.option("--alter", is_flag=True, help="Alter table to add any missing columns") +@click.option("-v", "--verbose", is_flag=True, help="Verbose output") +def update(url_or_alias, pk_values, updates, token, alter, verbose): + """ + Update a row by primary key + + Pass key=value pairs to set column values. + + Example usage: + + \b + dclient update https://example.com/db/table 42 name=Alice age=30 + """ + url = _resolve_url(url_or_alias) + token = _resolve_token(url, token) + + if not updates: + raise click.ClickException("Provide at least one key=value pair") + + update_dict = {} + for pair in updates: + if "=" not in pair: + raise click.ClickException("Invalid key=value pair: {}".format(pair)) + key, _, value = pair.partition("=") + # Try to parse as JSON for numbers, booleans, null + try: + value = json.loads(value) + except (json.JSONDecodeError, ValueError): + pass + update_dict[key] = value + + data = {"update": update_dict} + if alter: + data["alter"] = True + + api_url = "{}/{}/-/update".format(url.rstrip("/"), pk_values) + result = _api_post(api_url, data, token=token, verbose=verbose) + click.echo(json.dumps(result, indent=2)) + + +# --- delete command --- + + +@cli.command() +@click.argument("url_or_alias") +@click.argument("pk_values") +@click.option("--token", help="API token") +@click.option("--yes", is_flag=True, help="Skip confirmation") +@click.option("-v", "--verbose", is_flag=True, help="Verbose output") +def delete(url_or_alias, pk_values, token, yes, verbose): + """ + Delete a row by primary key + + Example usage: + + \b + dclient delete https://example.com/db/table 42 + dclient delete https://example.com/db/table 42 --yes + """ + url = _resolve_url(url_or_alias) + token = _resolve_token(url, token) + + if not yes: + click.confirm("Delete row {}?".format(pk_values), abort=True) + + api_url = "{}/{}/-/delete".format(url.rstrip("/"), pk_values) + result = _api_post(api_url, {}, token=token, verbose=verbose) + click.echo(json.dumps(result, indent=2)) + + +# --- drop command --- + + @cli.command() @click.argument("url_or_alias") @click.option("--token", help="API token") -def actor(url_or_alias, token): +@click.option("--yes", is_flag=True, help="Skip confirmation") +@click.option("-v", "--verbose", is_flag=True, help="Verbose output") +def drop(url_or_alias, token, yes, verbose): + """ + Drop a table + + Example usage: + + \b + dclient drop https://example.com/db/table + dclient drop https://example.com/db/table --yes + """ + url = _resolve_url(url_or_alias) + token = _resolve_token(url, token) + + api_url = url.rstrip("/") + "/-/drop" + + if not yes: + # First request to get row count + info = _api_post(api_url, {}, token=token, verbose=verbose) + row_count = info.get("row_count", "unknown") + table_name = info.get("table", url.rstrip("/").split("/")[-1]) + click.confirm("Drop table {} ({} rows)?".format(table_name, row_count), abort=True) + + result = _api_post(api_url, {"confirm": True}, token=token, verbose=verbose) + click.echo(json.dumps(result, indent=2)) + + +# --- create-table command --- + + +@cli.command(name="create-table") +@click.argument("url_or_alias") +@click.argument("table_name") +@click.option( + "--column", "column_defs", multiple=True, nargs=2, + help="Column definition: name type (e.g. --column id integer --column name text)" +) +@click.option( + "pks", "--pk", multiple=True, + help="Column(s) to use as primary key" +) +@click.option("--token", help="API token") +@click.option("-v", "--verbose", is_flag=True, help="Verbose output") +def create_table(url_or_alias, table_name, column_defs, pks, token, verbose): + """ + Create a new empty table with explicit schema + + Example usage: + + \b + dclient create-table https://example.com/db mytable \\ + --column id integer --column name text --pk id + """ + url = _resolve_url(url_or_alias) + token = _resolve_token(url, token) + + if not column_defs: + raise click.ClickException("Provide at least one --column definition") + + columns = [{"name": name, "type": typ} for name, typ in column_defs] + data = {"table": table_name, "columns": columns} + if pks: + if len(pks) == 1: + data["pk"] = pks[0] + else: + data["pks"] = list(pks) + + api_url = url.rstrip("/") + "/-/create" + result = _api_post(api_url, data, token=token, verbose=verbose) + click.echo(json.dumps(result, indent=2)) + + +# --- actor command --- + + +@cli.command() +@click.argument("url_or_alias", required=False) +@click.option("--token", help="API token") +@click.pass_context +def actor(ctx, url_or_alias, token): """ Show the actor represented by an API token @@ -276,18 +1009,12 @@ def actor(url_or_alias, token): \b dclient actor https://latest.datasette.io/fixtures """ - aliases_file = get_config_dir() / "aliases.json" - aliases = _load_aliases(aliases_file) - if url_or_alias in aliases: - url = aliases[url_or_alias] - else: - url = url_or_alias + url = _resolve_url_with_env(url_or_alias) if not (url.startswith("http://") or url.startswith("https://")): raise click.ClickException("Invalid URL: " + url) - if token is None: - token = token_for_url(url, _load_auths(get_config_dir() / "auth.json")) + token = _resolve_token(url, token) url_bits = url.split("/") url_bits[-1] = "-/actor.json" @@ -299,6 +1026,9 @@ def actor(url_or_alias, token): click.echo(json.dumps(response.json(), indent=4)) +# --- alias commands --- + + @cli.group() def alias(): "Manage aliases for different instances" @@ -369,6 +1099,9 @@ def alias_remove(name): raise click.ClickException("No such alias") +# --- auth commands --- + + @cli.group() def auth(): "Manage authentication for different instances" @@ -441,20 +1174,7 @@ def auth_remove(alias_or_url): raise click.ClickException("No such URL or alias") -def _load_aliases(aliases_file): - if aliases_file.exists(): - aliases = json.loads(aliases_file.read_text()) - else: - aliases = {} - return aliases - - -def _load_auths(auth_file): - if auth_file.exists(): - auths = json.loads(auth_file.read_text()) - else: - auths = {} - return auths +# --- Internal helpers --- def _batches(iterable, size, interval=None): @@ -521,7 +1241,6 @@ def _insert_batch( if verbose: click.echo(str(response), err=True) if str(response.status_code)[0] != "2": - # Is there an error we can show? if "/json" in response.headers["content-type"]: data = response.json() if "errors" in data: diff --git a/demos/exploring-writing.md b/demos/exploring-writing.md new file mode 100644 index 0000000..8a5999d --- /dev/null +++ b/demos/exploring-writing.md @@ -0,0 +1,322 @@ +# Exploring and Writing Data with dclient + +*2026-02-12T01:33:47Z* + +This demo walks through dclient's data exploration and write commands against a live Datasette instance running on localhost with a mutable in-memory database called `demo`. + +## Creating a table + +`create-table` creates an empty table with an explicit schema. We define columns and a primary key up front. + +```bash +uv run dclient create-table http://127.0.0.1:8042/demo dogs --column id integer --column name text --column age integer --column breed text --pk id --token dstok_eyJhIjoicm9vdCIsInQiOjE3NzA4NTk5ODJ9.1KDQjswLBzm5-R-MLmopOIHRZls +``` + +```output +{ + "ok": true, + "database": "demo", + "table": "dogs", + "table_url": "http://127.0.0.1:8042/demo/dogs", + "table_api_url": "http://127.0.0.1:8042/demo/dogs.json", + "schema": "CREATE TABLE [dogs] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] INTEGER,\n [breed] TEXT\n)" +} +``` + +## Inserting rows from CSV + +`insert` reads CSV (or TSV, JSON, newline-delimited JSON) from a file or stdin. Here we pipe a CSV file in. + +```bash +cat /tmp/dogs.csv | uv run dclient insert http://127.0.0.1:8042/demo dogs - --csv --token dstok_eyJhIjoicm9vdCIsInQiOjE3NzA4NTk5ODJ9.1KDQjswLBzm5-R-MLmopOIHRZls +``` + +```output +``` + +## Listing databases + +`databases` lists every database on the instance. The `--table` flag formats the output as a text table. + +```bash +uv run dclient databases http://127.0.0.1:8042 --table --token dstok_eyJhIjoicm9vdCIsInQiOjE3NzA4NTk5ODJ9.1KDQjswLBzm5-R-MLmopOIHRZls +``` + +```output +name path is_mutable +------- ---- ---------- +_memory False +demo True +``` + +## Listing tables + +`tables` shows all tables in a database with row counts and column names. + +```bash +uv run dclient tables http://127.0.0.1:8042/demo --table --token dstok_eyJhIjoicm9vdCIsInQiOjE3NzA4NTk5ODJ9.1KDQjswLBzm5-R-MLmopOIHRZls +``` + +```output +name rows columns +---- ---- -------------------- +dogs 5 id, name, age, breed +``` + +## Viewing the schema + +`schema` prints the CREATE TABLE SQL for every table in the database. + +```bash +uv run dclient schema http://127.0.0.1:8042/demo --token dstok_eyJhIjoicm9vdCIsInQiOjE3NzA4NTk5ODJ9.1KDQjswLBzm5-R-MLmopOIHRZls +``` + +```output +CREATE TABLE [dogs] ( + [id] INTEGER PRIMARY KEY, + [name] TEXT, + [age] INTEGER, + [breed] TEXT +); +``` + +## Browsing rows + +`rows` lets you explore table data without writing SQL. The default output is JSON; `--table` renders a text table. + +```bash +uv run dclient rows http://127.0.0.1:8042/demo/dogs --table --token dstok_eyJhIjoicm9vdCIsInQiOjE3NzA4NTk5ODJ9.1KDQjswLBzm5-R-MLmopOIHRZls +``` + +```output +id name age breed +-- -------- --- ---------------- +1 Cleo 5 Golden Retriever +2 Pancakes 3 Poodle +3 Fido 7 Labrador +4 Muffin 2 Corgi +5 Rex 4 German Shepherd +``` + +### Filtering + +Use `-w` to filter using Datasette's column operators (`__gt`, `__contains`, `__exact`, etc.). + +```bash +uv run dclient rows http://127.0.0.1:8042/demo/dogs --table -w age__gt=4 --token dstok_eyJhIjoicm9vdCIsInQiOjE3NzA4NTk5ODJ9.1KDQjswLBzm5-R-MLmopOIHRZls +``` + +```output +id name age breed +-- ---- --- ---------------- +1 Cleo 5 Golden Retriever +3 Fido 7 Labrador +``` + +### Sorting + +`--sort` and `--sort-desc` control row order. + +```bash +uv run dclient rows http://127.0.0.1:8042/demo/dogs --table --sort-desc age --token dstok_eyJhIjoicm9vdCIsInQiOjE3NzA4NTk5ODJ9.1KDQjswLBzm5-R-MLmopOIHRZls +``` + +```output +id name age breed +-- -------- --- ---------------- +3 Fido 7 Labrador +1 Cleo 5 Golden Retriever +5 Rex 4 German Shepherd +2 Pancakes 3 Poodle +4 Muffin 2 Corgi +``` + +### Selecting columns + +`--col` picks specific columns. Here we also output as CSV, which is handy for piping into other tools. + +```bash +uv run dclient rows http://127.0.0.1:8042/demo/dogs --csv --col name --col breed --token dstok_eyJhIjoicm9vdCIsInQiOjE3NzA4NTk5ODJ9.1KDQjswLBzm5-R-MLmopOIHRZls +``` + +```output +id,name,breed +1,Cleo,Golden Retriever +2,Pancakes,Poodle +3,Fido,Labrador +4,Muffin,Corgi +5,Rex,German Shepherd +``` + +### Limiting results + +`--limit` caps the total number of rows returned. + +```bash +uv run dclient rows http://127.0.0.1:8042/demo/dogs --table --limit 2 --sort age --token dstok_eyJhIjoicm9vdCIsInQiOjE3NzA4NTk5ODJ9.1KDQjswLBzm5-R-MLmopOIHRZls +``` + +```output +id name age breed +-- -------- --- ------ +4 Muffin 2 Corgi +2 Pancakes 3 Poodle +``` + +## Fetching a single row + +`get` retrieves one row by its primary key. + +```bash +uv run dclient get http://127.0.0.1:8042/demo/dogs 3 --token dstok_eyJhIjoicm9vdCIsInQiOjE3NzA4NTk5ODJ9.1KDQjswLBzm5-R-MLmopOIHRZls +``` + +```output +{ + "id": 3, + "name": "Fido", + "age": 7, + "breed": "Labrador" +} +``` + +## Running SQL queries + +`query` executes arbitrary SQL and returns results. Here we use `--table` for a readable summary. + +```bash +uv run dclient query http://127.0.0.1:8042/demo 'select breed, count(*) as count from dogs group by breed order by count desc' --table --token dstok_eyJhIjoicm9vdCIsInQiOjE3NzA4NTk5ODJ9.1KDQjswLBzm5-R-MLmopOIHRZls +``` + +```output +breed count +---------------- ----- +Poodle 1 +Labrador 1 +Golden Retriever 1 +German Shepherd 1 +Corgi 1 +``` + +The same query as newline-delimited JSON, useful for streaming into `jq` or other line-oriented tools. + +```bash +uv run dclient query http://127.0.0.1:8042/demo 'select name, age from dogs order by age desc limit 3' --nl --token dstok_eyJhIjoicm9vdCIsInQiOjE3NzA4NTk5ODJ9.1KDQjswLBzm5-R-MLmopOIHRZls +``` + +```output +{"name": "Fido", "age": 7} +{"name": "Cleo", "age": 5} +{"name": "Rex", "age": 4} +``` + +## Updating a row + +`update` modifies a single row by primary key. Pass column=value pairs as arguments. Numeric values are auto-detected. + +```bash +uv run dclient update http://127.0.0.1:8042/demo/dogs 3 name=Buddy age=8 --token dstok_eyJhIjoicm9vdCIsInQiOjE3NzA4NTk5ODJ9.1KDQjswLBzm5-R-MLmopOIHRZls +``` + +```output +{ + "ok": true +} +``` + +Verify the update: + +```bash +uv run dclient get http://127.0.0.1:8042/demo/dogs 3 --token dstok_eyJhIjoicm9vdCIsInQiOjE3NzA4NTk5ODJ9.1KDQjswLBzm5-R-MLmopOIHRZls +``` + +```output +{ + "id": 3, + "name": "Buddy", + "age": 8, + "breed": "Labrador" +} +``` + +## Upserting rows + +`upsert` inserts new rows and updates existing ones that match by primary key. Here we update Cleo's age to 6 and add a new dog Luna, all in one call. + +```bash +cat /tmp/upsert_dogs.json | uv run dclient upsert http://127.0.0.1:8042/demo dogs - --json --token dstok_eyJhIjoicm9vdCIsInQiOjE3NzA4NTk5ODJ9.1KDQjswLBzm5-R-MLmopOIHRZls +``` + +```output +``` + +The table now shows Cleo at age 6 and the new row for Luna: + +```bash +uv run dclient rows http://127.0.0.1:8042/demo/dogs --table --sort id --token dstok_eyJhIjoicm9vdCIsInQiOjE3NzA4NTk5ODJ9.1KDQjswLBzm5-R-MLmopOIHRZls +``` + +```output +id name age breed +-- -------- --- ---------------- +1 Cleo 6 Golden Retriever +2 Pancakes 3 Poodle +3 Buddy 8 Labrador +4 Muffin 2 Corgi +5 Rex 4 German Shepherd +6 Luna 1 Husky +``` + +## Deleting a row + +`delete` removes a row by primary key. Use `--yes` to skip the confirmation prompt. + +```bash +uv run dclient delete http://127.0.0.1:8042/demo/dogs 4 --yes --token dstok_eyJhIjoicm9vdCIsInQiOjE3NzA4NTk5ODJ9.1KDQjswLBzm5-R-MLmopOIHRZls +``` + +```output +{ + "ok": true +} +``` + +Muffin (id=4) is gone: + +```bash +uv run dclient rows http://127.0.0.1:8042/demo/dogs --table --sort id --token dstok_eyJhIjoicm9vdCIsInQiOjE3NzA4NTk5ODJ9.1KDQjswLBzm5-R-MLmopOIHRZls +``` + +```output +id name age breed +-- -------- --- ---------------- +1 Cleo 6 Golden Retriever +2 Pancakes 3 Poodle +3 Buddy 8 Labrador +5 Rex 4 German Shepherd +6 Luna 1 Husky +``` + +## Dropping a table + +`drop` removes an entire table. Without `--yes` it shows the row count and asks for confirmation. + +```bash +uv run dclient drop http://127.0.0.1:8042/demo/dogs --yes --token dstok_eyJhIjoicm9vdCIsInQiOjE3NzA4NTk5ODJ9.1KDQjswLBzm5-R-MLmopOIHRZls +``` + +```output +{ + "ok": true +} +``` + +The table is gone: + +```bash +uv run dclient tables http://127.0.0.1:8042/demo --json --token dstok_eyJhIjoicm9vdCIsInQiOjE3NzA4NTk5ODJ9.1KDQjswLBzm5-R-MLmopOIHRZls +``` + +```output +[] +``` diff --git a/docs/authentication.md b/docs/authentication.md index 4e5f22f..caadba7 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -161,7 +161,7 @@ cog.out( ) ]]] --> ``` -Usage: dclient actor [OPTIONS] URL_OR_ALIAS +Usage: dclient actor [OPTIONS] [URL_OR_ALIAS] Show the actor represented by an API token diff --git a/docs/exploring.md b/docs/exploring.md new file mode 100644 index 0000000..9392ab9 --- /dev/null +++ b/docs/exploring.md @@ -0,0 +1,347 @@ +# Exploring data + +dclient provides several commands for exploring Datasette instances without writing SQL. + +## Listing databases + +Use `dclient databases` to list all databases on a Datasette instance: + +```bash +dclient databases https://latest.datasette.io +``` + +Use `--table` for a formatted text table: +```bash +dclient databases https://latest.datasette.io --table +``` +Output: +``` +name path is_mutable +-------- -------- ---------- +fixtures fixtures False +``` + +## Listing tables + +Use `dclient tables` to list tables in a specific database: + +```bash +dclient tables https://latest.datasette.io/fixtures +``` + +Use `--table` for formatted output: +```bash +dclient tables https://latest.datasette.io/fixtures --table +``` + +Use `--schema` to include the CREATE TABLE SQL for each table: +```bash +dclient tables https://latest.datasette.io/fixtures --schema +``` + +Use `--views` to include views in the output (hidden by default). + +## Viewing schemas + +Use `dclient schema` to show the SQL schema for a database or an individual table: + +```bash +# Schema for all tables in a database +dclient schema https://latest.datasette.io/fixtures + +# Schema for a specific table +dclient schema https://latest.datasette.io/fixtures/facetable +``` + +## Browsing rows + +Use `dclient rows` to browse table data with filtering, sorting, and pagination — no SQL required: + +```bash +dclient rows https://latest.datasette.io/fixtures/facetable --table +``` + +### Filtering + +Use `-w` / `--where` to filter rows. Supports [Datasette filter operators](https://docs.datasette.io/en/stable/json_api.html#table-arguments) like `__gt`, `__contains`, `__exact`, etc.: + +```bash +# Exact match +dclient rows https://example.com/db/dogs -w breed=Poodle + +# Greater than +dclient rows https://example.com/db/dogs -w age__gt=4 + +# Multiple filters (AND) +dclient rows https://example.com/db/dogs -w age__gte=3 -w breed=Labrador +``` + +For raw SQL WHERE clauses, use `--where-sql`: +```bash +dclient rows https://example.com/db/dogs --where-sql "age > 3 and breed != 'Poodle'" +``` + +### Sorting + +```bash +# Sort ascending +dclient rows https://example.com/db/dogs --sort name + +# Sort descending +dclient rows https://example.com/db/dogs --sort-desc age +``` + +### Selecting columns + +```bash +# Include only specific columns +dclient rows https://example.com/db/dogs --col name --col breed + +# Exclude specific columns +dclient rows https://example.com/db/dogs --nocol id +``` + +### Full-text search + +```bash +dclient rows https://example.com/db/articles --search "python tutorial" +``` + +### Faceting + +```bash +dclient rows https://example.com/db/dogs --facet breed +``` +Facet results are printed to stderr, with the rows on stdout. + +### Pagination + +By default, only the first page of results is returned. Use `--all` to fetch every page: + +```bash +dclient rows https://example.com/db/dogs --all +``` + +Control page size with `--size` and limit total rows with `--limit`: + +```bash +# 10 rows per page, max 50 rows total +dclient rows https://example.com/db/dogs --size 10 --limit 50 +``` + +## Fetching a single row + +Use `dclient get` to fetch a single row by its primary key: + +```bash +dclient get https://latest.datasette.io/fixtures/facetable 1 +``` +Output: +```json +{ + "pk": 1, + "created": "2019-01-14 08:00:00", + "planet_int": 1, + "on_earth": 1, + "state": "CA", + ... +} +``` + +For compound primary keys, separate values with a comma: +```bash +dclient get https://example.com/db/compound_pk a,b +``` + +## Output formats + +The `databases`, `tables`, `rows`, and `query` commands all support multiple output formats: + +- `--json` — JSON (default) +- `--csv` — CSV +- `--tsv` — TSV +- `--nl` — Newline-delimited JSON +- `--table` — Formatted text table + +```bash +dclient rows https://example.com/db/dogs --csv > dogs.csv +dclient rows https://example.com/db/dogs --table +``` + +## dclient databases --help + +``` +Usage: dclient databases [OPTIONS] [URL_OR_ALIAS] + + List databases on a Datasette instance + + Example usage: + + dclient databases https://latest.datasette.io + +Options: + --token TEXT API token + -v, --verbose Verbose output + --csv Output as CSV + --tsv Output as TSV + --json Output as JSON (default) + --nl Output as newline-delimited JSON + --table Output as text table + --help Show this message and exit. + +``` + + +## dclient tables --help + +``` +Usage: dclient tables [OPTIONS] [URL_OR_ALIAS] + + List tables in a Datasette database + + Example usage: + + dclient tables https://latest.datasette.io/fixtures + +Options: + --token TEXT API token + --views Include views + --schema Show CREATE TABLE SQL + -v, --verbose Verbose output + --csv Output as CSV + --tsv Output as TSV + --json Output as JSON (default) + --nl Output as newline-delimited JSON + --table Output as text table + --help Show this message and exit. + +``` + + +## dclient schema --help + +``` +Usage: dclient schema [OPTIONS] [URL_OR_ALIAS] + + Show the SQL schema for a database or table + + Example usage: + + dclient schema https://latest.datasette.io/fixtures + dclient schema https://latest.datasette.io/fixtures/facetable + +Options: + --token TEXT API token + -v, --verbose Verbose output + --help Show this message and exit. + +``` + + +## dclient rows --help + +``` +Usage: dclient rows [OPTIONS] [URL_OR_ALIAS] + + Browse rows in a Datasette table with filtering and sorting + + Example usage: + + dclient rows https://latest.datasette.io/fixtures/facetable + dclient rows https://latest.datasette.io/fixtures/facetable -w state=CA + dclient rows https://latest.datasette.io/fixtures/facetable --sort city + dclient rows https://latest.datasette.io/fixtures/facetable --search text + +Options: + --token TEXT API token + -w, --where TEXT Filter: column__op=value (e.g. age__gt=30) + --where-sql TEXT Raw SQL WHERE clause + --search TEXT Full-text search query + --sort TEXT Sort by column (ascending) + --sort-desc TEXT Sort by column (descending) + --col TEXT Include only these columns + --nocol TEXT Exclude these columns + --facet TEXT Facet by column + --size INTEGER Number of rows per page + --all Fetch all pages + --limit INTEGER Maximum total rows to return + -v, --verbose Verbose output + --csv Output as CSV + --tsv Output as TSV + --json Output as JSON (default) + --nl Output as newline-delimited JSON + --table Output as text table + --help Show this message and exit. + +``` + + +## dclient get --help + +``` +Usage: dclient get [OPTIONS] [URL_OR_ALIAS] PK_VALUES + + Fetch a single row by primary key + + Example usage: + + dclient get https://latest.datasette.io/fixtures/facetable 1 + dclient get https://latest.datasette.io/fixtures/compound_pk a,b + +Options: + --token TEXT API token + -v, --verbose Verbose output + --help Show this message and exit. + +``` + diff --git a/docs/index.md b/docs/index.md index 00897bd..6564798 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,7 +9,7 @@ A client CLI utility for [Datasette](https://datasette.io/) instances ## Installation -Install `dclient` using `pip` (or [pipx](https://pipxproject.github.io/pipx/): +Install `dclient` using `pip` (or [pipx](https://pipxproject.github.io/pipx/)): ```bash pip install dclient @@ -21,16 +21,11 @@ If you also have Datasette installed in the same environment it will register it ```bash datasette install dclient ``` -This means you can run any of these commands using `datasette dt` instead, like this: +This means you can run any of these commands using `datasette dc` instead, like this: ```bash datasette dc --help datasette dc query https://latest.datasette.io/fixtures "select * from facetable limit 1" ``` -You can install it into Datasette this way using: - -```bash -datasette install dclient -``` ## Contents @@ -38,8 +33,10 @@ datasette install dclient --- maxdepth: 3 --- +exploring queries +inserting +writing aliases authentication -inserting ``` diff --git a/docs/inserting.md b/docs/inserting.md index 316a3c0..df0c6a5 100644 --- a/docs/inserting.md +++ b/docs/inserting.md @@ -4,6 +4,8 @@ The `dclient insert` command can be used to insert data from a local file direct First you'll need to {ref}`authenticate ` with the instance. +See also {doc}`writing` for related commands: `upsert`, `update`, `delete`, `drop`, and `create-table`. + To insert data from a `data.csv` file into a table called `my_table`, creating that table if it does not exist: ```bash @@ -138,3 +140,7 @@ Options: ``` + +:::{note} +To update existing rows rather than inserting new ones, see {doc}`writing` for the `upsert` and `update` commands. +::: diff --git a/docs/queries.md b/docs/queries.md index 3bd671a..1e15e4c 100644 --- a/docs/queries.md +++ b/docs/queries.md @@ -24,6 +24,34 @@ Output: ] ``` +## Output formats + +Query results default to JSON. Use these flags for other formats: + +- `--csv` — CSV +- `--tsv` — TSV +- `--nl` — Newline-delimited JSON (one JSON object per line) +- `--table` — Formatted text table + +```bash +dclient query https://latest.datasette.io/fixtures \ + "select state, count(*) as n from facetable group by state" --table +``` +Output: +``` +state n +----- -- +CA 10 +MI 4 +MO 1 +``` + +Export as CSV: +```bash +dclient query https://latest.datasette.io/fixtures \ + "select * from facetable limit 5" --csv > results.csv +``` + ## dclient query --help +``` +Usage: dclient create-table [OPTIONS] URL_OR_ALIAS TABLE_NAME + + Create a new empty table with explicit schema + + Example usage: + + dclient create-table https://example.com/db mytable \ + --column id integer --column name text --pk id + +Options: + --column TEXT... Column definition: name type (e.g. --column id integer + --column name text) + --pk TEXT Column(s) to use as primary key + --token TEXT API token + -v, --verbose Verbose output + --help Show this message and exit. + +``` + + +## dclient upsert --help + +``` +Usage: dclient upsert [OPTIONS] URL_OR_ALIAS TABLE FILEPATH + + Upsert data into a remote Datasette table + + Rows with matching primary keys will be updated; others will be inserted. Each + row must include the primary key column(s). + + Example usage: + + dclient upsert \ + https://private.datasette.cloud/data \ + mytable data.csv + +Options: + --csv Input is CSV + --tsv Input is TSV + --json Input is JSON + --nl Input is newline-delimited JSON + --encoding TEXT Character encoding for CSV/TSV + --no-detect-types Don't detect column types for CSV/TSV + --alter Alter table to add any missing columns + --batch-size INTEGER Send rows in batches of this size + --interval FLOAT Send batch at least every X seconds + -t, --token TEXT API token + --silent Don't output progress + -v, --verbose Verbose output: show HTTP request and response + --help Show this message and exit. + +``` + + +## dclient update --help + +``` +Usage: dclient update [OPTIONS] URL_OR_ALIAS PK_VALUES [UPDATES]... + + Update a row by primary key + + Pass key=value pairs to set column values. + + Example usage: + + dclient update https://example.com/db/table 42 name=Alice age=30 + +Options: + --token TEXT API token + --alter Alter table to add any missing columns + -v, --verbose Verbose output + --help Show this message and exit. + +``` + + +## dclient delete --help + +``` +Usage: dclient delete [OPTIONS] URL_OR_ALIAS PK_VALUES + + Delete a row by primary key + + Example usage: + + dclient delete https://example.com/db/table 42 + dclient delete https://example.com/db/table 42 --yes + +Options: + --token TEXT API token + --yes Skip confirmation + -v, --verbose Verbose output + --help Show this message and exit. + +``` + + +## dclient drop --help + +``` +Usage: dclient drop [OPTIONS] URL_OR_ALIAS + + Drop a table + + Example usage: + + dclient drop https://example.com/db/table + dclient drop https://example.com/db/table --yes + +Options: + --token TEXT API token + --yes Skip confirmation + -v, --verbose Verbose output + --help Show this message and exit. + +``` + diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..975916b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,42 @@ +[project] +name = "dclient" +version = "0.4" +description = "A client CLI utility for Datasette instances" +readme = "README.md" +requires-python = ">=3.8" +license = "Apache-2.0" +authors = [{name = "Simon Willison"}] +dependencies = [ + "click", + "httpx", + "sqlite-utils", +] + +[project.urls] +Homepage = "https://github.com/simonw/dclient" +Issues = "https://github.com/simonw/dclient/issues" +CI = "https://github.com/simonw/dclient/actions" +Changelog = "https://github.com/simonw/dclient/releases" + +[project.scripts] +dclient = "dclient.cli:cli" + +[project.entry-points.datasette] +client = "dclient.plugin" + +[dependency-groups] +dev = [ + "pytest", + "pytest-asyncio", + "pytest-httpx", + "cogapp", + "pytest-mock", + "datasette>=1.0a2", +] + +[build-system] +requires = ["uv_build>=0.9.18,<0.10.0"] +build-backend = "uv_build" + +[tool.uv.build-backend] +module-root = "" diff --git a/setup.py b/setup.py deleted file mode 100644 index e50775e..0000000 --- a/setup.py +++ /dev/null @@ -1,46 +0,0 @@ -from setuptools import setup -import os - -VERSION = "0.4" - - -def get_long_description(): - with open( - os.path.join(os.path.dirname(os.path.abspath(__file__)), "README.md"), - encoding="utf8", - ) as fp: - return fp.read() - - -setup( - name="dclient", - description="A client CLI utility for Datasette instances", - long_description=get_long_description(), - long_description_content_type="text/markdown", - author="Simon Willison", - url="https://github.com/simonw/dclient", - project_urls={ - "Issues": "https://github.com/simonw/dclient/issues", - "CI": "https://github.com/simonw/dclient/actions", - "Changelog": "https://github.com/simonw/dclient/releases", - }, - license="Apache License, Version 2.0", - version=VERSION, - packages=["dclient"], - entry_points={ - "datasette": ["client = dclient.plugin"], - "console_scripts": ["dclient = dclient.cli:cli"], - }, - install_requires=["click", "httpx", "sqlite-utils"], - extras_require={ - "test": [ - "pytest", - "pytest-asyncio", - "pytest-httpx", - "cogapp", - "pytest-mock", - "datasette>=1.0a2", - ] - }, - python_requires=">=3.8", -) diff --git a/tests/test_insert.py b/tests/test_insert.py index c849a67..3ce64b1 100644 --- a/tests/test_insert.py +++ b/tests/test_insert.py @@ -120,7 +120,7 @@ def make_format_test(content, arg): input_data=SIMPLE_CSV, cmd_args=[], table_exists=False, - expected_output="Inserting rows\nError: Table not found: table1\n", + expected_output="Inserting rows\nError: Table not found\n", should_error=True, expected_table_json=None, ), @@ -296,7 +296,7 @@ def test_insert_against_datasette( return loop.run_until_complete(run()) - httpx_mock.add_callback(custom_response) + httpx_mock.add_callback(custom_response, is_optional=should_error) path = pathlib.Path(tmpdir) / "data.txt" if isinstance(input_data, str): diff --git a/tests/test_new_commands.py b/tests/test_new_commands.py new file mode 100644 index 0000000..318eb5b --- /dev/null +++ b/tests/test_new_commands.py @@ -0,0 +1,506 @@ +"""Tests for new T1 and T2 commands: databases, tables, schema, rows, get, +upsert, update, delete, drop, create-table.""" + +import asyncio +import json +import pathlib + +import pytest +from click.testing import CliRunner +from datasette.app import Datasette + +from dclient.cli import cli + + +@pytest.fixture +def ds(): + """Create a Datasette instance with a memory database and sample data.""" + datasette = Datasette( + config={ + "permissions": { + "create-table": {"id": "*"}, + "insert-row": {"id": "*"}, + "update-row": {"id": "*"}, + "delete-row": {"id": "*"}, + "drop-table": {"id": "*"}, + "alter-table": {"id": "*"}, + } + } + ) + db = datasette.add_memory_database("data") + loop = asyncio.get_event_loop() + + async def setup(): + # Drop all tables first + for table in await db.table_names(): + await db.execute_write("drop table [{}]".format(table)) + await db.execute_write( + "create table dogs (id integer primary key, name text, age integer)" + ) + await db.execute_write( + "insert into dogs (id, name, age) values (1, 'Cleo', 5), (2, 'Pancakes', 3), (3, 'Fido', 7)" + ) + + loop.run_until_complete(setup()) + return datasette, db, loop + + +@pytest.fixture +def token(ds): + datasette, _, _ = ds + return datasette.create_token("actor") + + +def _mock_datasette(httpx_mock, ds): + """Set up httpx_mock to route requests through the Datasette instance.""" + datasette, _, loop = ds + + def custom_response(request): + async def run(): + method = request.method + url_path = request.url.raw_path.decode() + if method == "GET": + response = await datasette.client.get( + url_path, headers=request.headers + ) + else: + content = request.read() + try: + body = json.loads(content) if content else {} + except json.JSONDecodeError: + body = {} + response = await datasette.client.request( + method, url_path, json=body, headers=request.headers + ) + import httpx as httpx_lib + + return httpx_lib.Response( + status_code=response.status_code, + headers=response.headers, + content=response.content, + ) + + return loop.run_until_complete(run()) + + httpx_mock.add_callback(custom_response, is_optional=True) + + +# --- databases command --- + + +def test_databases(httpx_mock, ds, token): + _mock_datasette(httpx_mock, ds) + runner = CliRunner() + result = runner.invoke(cli, ["databases", "http://localhost/", "--token", token]) + assert result.exit_code == 0 + data = json.loads(result.output) + names = [d["name"] for d in data] + assert "data" in names + + +def test_databases_table_format(httpx_mock, ds, token): + _mock_datasette(httpx_mock, ds) + runner = CliRunner() + result = runner.invoke( + cli, ["databases", "http://localhost/", "--token", token, "--table"] + ) + assert result.exit_code == 0 + assert "name" in result.output + assert "data" in result.output + + +# --- tables command --- + + +def test_tables(httpx_mock, ds, token): + _mock_datasette(httpx_mock, ds) + runner = CliRunner() + result = runner.invoke(cli, ["tables", "http://localhost/data", "--token", token]) + assert result.exit_code == 0 + data = json.loads(result.output) + table_names = [t["name"] for t in data] + assert "dogs" in table_names + + +def test_tables_table_format(httpx_mock, ds, token): + _mock_datasette(httpx_mock, ds) + runner = CliRunner() + result = runner.invoke( + cli, ["tables", "http://localhost/data", "--token", token, "--table"] + ) + assert result.exit_code == 0 + assert "dogs" in result.output + + +# --- schema command --- + + +def test_schema(httpx_mock, ds, token): + _mock_datasette(httpx_mock, ds) + runner = CliRunner() + result = runner.invoke(cli, ["schema", "http://localhost/data", "--token", token]) + assert result.exit_code == 0 + assert "CREATE TABLE" in result.output + assert "dogs" in result.output + + +# --- rows command --- + + +def test_rows_basic(httpx_mock, ds, token): + _mock_datasette(httpx_mock, ds) + runner = CliRunner() + result = runner.invoke( + cli, ["rows", "http://localhost/data/dogs", "--token", token] + ) + assert result.exit_code == 0 + data = json.loads(result.output) + assert len(data) == 3 + names = [r["name"] for r in data] + assert "Cleo" in names + + +def test_rows_with_filter(httpx_mock, ds, token): + _mock_datasette(httpx_mock, ds) + runner = CliRunner() + result = runner.invoke( + cli, + ["rows", "http://localhost/data/dogs", "--token", token, "-w", "age__gt=4"], + ) + assert result.exit_code == 0 + data = json.loads(result.output) + for row in data: + assert row["age"] > 4 + + +def test_rows_with_sort(httpx_mock, ds, token): + _mock_datasette(httpx_mock, ds) + runner = CliRunner() + result = runner.invoke( + cli, + ["rows", "http://localhost/data/dogs", "--token", token, "--sort", "name"], + ) + assert result.exit_code == 0 + data = json.loads(result.output) + names = [r["name"] for r in data] + assert names == sorted(names) + + +def test_rows_with_sort_desc(httpx_mock, ds, token): + _mock_datasette(httpx_mock, ds) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "rows", + "http://localhost/data/dogs", + "--token", + token, + "--sort-desc", + "age", + ], + ) + assert result.exit_code == 0 + data = json.loads(result.output) + ages = [r["age"] for r in data] + assert ages == sorted(ages, reverse=True) + + +def test_rows_csv(httpx_mock, ds, token): + _mock_datasette(httpx_mock, ds) + runner = CliRunner() + result = runner.invoke( + cli, ["rows", "http://localhost/data/dogs", "--token", token, "--csv"] + ) + assert result.exit_code == 0 + assert "id,name,age" in result.output + assert "Cleo" in result.output + + +def test_rows_nl(httpx_mock, ds, token): + _mock_datasette(httpx_mock, ds) + runner = CliRunner() + result = runner.invoke( + cli, ["rows", "http://localhost/data/dogs", "--token", token, "--nl"] + ) + assert result.exit_code == 0 + lines = [l for l in result.output.strip().split("\n") if l] + for line in lines: + parsed = json.loads(line) + assert "name" in parsed + + +def test_rows_table_format(httpx_mock, ds, token): + _mock_datasette(httpx_mock, ds) + runner = CliRunner() + result = runner.invoke( + cli, ["rows", "http://localhost/data/dogs", "--token", token, "--table"] + ) + assert result.exit_code == 0 + assert "id" in result.output + assert "name" in result.output + assert "Cleo" in result.output + # Should have header separator + assert "---" in result.output + + +def test_rows_with_limit(httpx_mock, ds, token): + _mock_datasette(httpx_mock, ds) + runner = CliRunner() + result = runner.invoke( + cli, + ["rows", "http://localhost/data/dogs", "--token", token, "--limit", "2"], + ) + assert result.exit_code == 0 + data = json.loads(result.output) + assert len(data) == 2 + + +def test_rows_with_columns(httpx_mock, ds, token): + _mock_datasette(httpx_mock, ds) + runner = CliRunner() + result = runner.invoke( + cli, + ["rows", "http://localhost/data/dogs", "--token", token, "--col", "name"], + ) + assert result.exit_code == 0 + data = json.loads(result.output) + for row in data: + assert "name" in row + + +# --- get command --- + + +def test_get(httpx_mock, ds, token): + _mock_datasette(httpx_mock, ds) + runner = CliRunner() + result = runner.invoke( + cli, ["get", "http://localhost/data/dogs", "1", "--token", token] + ) + assert result.exit_code == 0 + data = json.loads(result.output) + assert data["name"] == "Cleo" + assert data["id"] == 1 + + +# --- upsert command --- + + +def test_upsert(httpx_mock, ds, token, tmpdir): + _mock_datasette(httpx_mock, ds) + # Upsert: update Cleo's age and add a new dog + data = json.dumps([ + {"id": 1, "name": "Cleo", "age": 6}, + {"id": 4, "name": "Muffin", "age": 2}, + ]) + path = pathlib.Path(tmpdir) / "upsert.json" + path.write_text(data) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "upsert", + "http://localhost/data", + "dogs", + str(path), + "--json", + "--token", + token, + ], + ) + assert result.exit_code == 0, result.output + + # Verify + datasette, _, loop = ds + + async def check(): + r = await datasette.client.get("/data/dogs.json?_shape=array") + return r.json() + + rows = loop.run_until_complete(check()) + names = {r["name"]: r["age"] for r in rows} + assert names["Cleo"] == 6 # updated + assert names["Muffin"] == 2 # inserted + + +# --- update command --- + + +def test_update(httpx_mock, ds, token): + _mock_datasette(httpx_mock, ds) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "update", + "http://localhost/data/dogs", + "1", + "name=Rex", + "age=10", + "--token", + token, + ], + ) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert data.get("ok") is True + + # Verify the update + datasette, _, loop = ds + + async def check(): + r = await datasette.client.get("/data/dogs/1.json?_shape=objects") + return r.json() + + row_data = loop.run_until_complete(check()) + row = row_data["rows"][0] + assert row["name"] == "Rex" + assert row["age"] == 10 + + +# --- delete command --- + + +def test_delete(httpx_mock, ds, token): + _mock_datasette(httpx_mock, ds) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "delete", + "http://localhost/data/dogs", + "3", + "--yes", + "--token", + token, + ], + ) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert data.get("ok") is True + + # Verify deletion + datasette, _, loop = ds + + async def check(): + r = await datasette.client.get("/data/dogs.json?_shape=array") + return r.json() + + rows = loop.run_until_complete(check()) + ids = [r["id"] for r in rows] + assert 3 not in ids + assert len(rows) == 2 + + +# --- drop command --- + + +def test_drop(httpx_mock, ds, token): + _mock_datasette(httpx_mock, ds) + runner = CliRunner() + result = runner.invoke( + cli, + ["drop", "http://localhost/data/dogs", "--yes", "--token", token], + ) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert data.get("ok") is True + + # Verify table is gone + datasette, db, loop = ds + + async def check(): + return await db.table_names() + + tables = loop.run_until_complete(check()) + assert "dogs" not in tables + + +# --- create-table command --- + + +def test_create_table(httpx_mock, ds, token): + _mock_datasette(httpx_mock, ds) + runner = CliRunner() + result = runner.invoke( + cli, + [ + "create-table", + "http://localhost/data", + "cats", + "--column", + "id", + "integer", + "--column", + "name", + "text", + "--pk", + "id", + "--token", + token, + ], + ) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + assert data.get("ok") is True + + # Verify table exists + datasette, db, loop = ds + + async def check(): + return await db.table_names() + + tables = loop.run_until_complete(check()) + assert "cats" in tables + + +# --- output format tests for query --- + + +SAMPLE_QUERY_RESPONSE = { + "ok": True, + "database": "data", + "rows": [ + {"id": 1, "name": "Cleo", "age": 5}, + {"id": 2, "name": "Pancakes", "age": 3}, + {"id": 3, "name": "Fido", "age": 7}, + ], + "truncated": False, + "columns": ["id", "name", "age"], +} + + +def test_query_csv(httpx_mock): + httpx_mock.add_response(json=SAMPLE_QUERY_RESPONSE, status_code=200) + runner = CliRunner() + result = runner.invoke( + cli, ["query", "https://example.com/data", "select * from dogs", "--csv"] + ) + assert result.exit_code == 0 + assert "id,name,age" in result.output + assert "Cleo" in result.output + + +def test_query_nl(httpx_mock): + httpx_mock.add_response(json=SAMPLE_QUERY_RESPONSE, status_code=200) + runner = CliRunner() + result = runner.invoke( + cli, ["query", "https://example.com/data", "select * from dogs", "--nl"] + ) + assert result.exit_code == 0 + lines = [l for l in result.output.strip().split("\n") if l] + assert len(lines) == 3 + first = json.loads(lines[0]) + assert first["name"] == "Cleo" + + +def test_query_table_format(httpx_mock): + httpx_mock.add_response(json=SAMPLE_QUERY_RESPONSE, status_code=200) + runner = CliRunner() + result = runner.invoke( + cli, ["query", "https://example.com/data", "select * from dogs", "--table"] + ) + assert result.exit_code == 0 + assert "Cleo" in result.output + assert "---" in result.output