diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 6ca0fac8..00000000 --- a/.coveragerc +++ /dev/null @@ -1,2 +0,0 @@ -[run] -omit = datasette/_version.py, datasette/utils/shutil_backport.py diff --git a/.dockerignore b/.dockerignore index 5078bf47..2c34db66 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,13 +1,13 @@ .DS_Store .cache .eggs +.git .gitignore .ipynb_checkpoints +.travis.yml build *.spec *.egg-info dist scratchpad venv -*.db -*.sqlite diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs deleted file mode 100644 index 84e574fd..00000000 --- a/.git-blame-ignore-revs +++ /dev/null @@ -1,4 +0,0 @@ -# Applying Black -35d6ee2790e41e96f243c1ff58be0c9c0519a8ce -368638555160fb9ac78f462d0f79b1394163fa30 -2b344f6a34d2adaa305996a1a580ece06397f6e4 diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index 744258eb..00000000 --- a/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -datasette/static/codemirror-* linguist-vendored diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index f0bcdbe0..00000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1 +0,0 @@ -github: [simonw] diff --git a/.github/actions/setup-sqlite-version/action.yml b/.github/actions/setup-sqlite-version/action.yml deleted file mode 100644 index fdbc71c9..00000000 --- a/.github/actions/setup-sqlite-version/action.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: "Setup SQLite version" -description: "Build and activate a specific SQLite version from its amalgamation archive" -inputs: - version: - description: "The SQLite version to install" - required: true - cflags: - description: "CFLAGS to use when compiling SQLite" - required: false - default: "" - skip-activate: - description: "Set to true to skip modifying the library path" - required: false - default: "false" - fallback-urls: - description: "Whitespace-separated fallback download URLs to try after sqlite.org" - required: false - default: "" -outputs: - sqlite-location: - description: "Directory containing the compiled SQLite library" - value: ${{ steps.build.outputs.sqlite-location }} -runs: - using: "composite" - steps: - - shell: bash - run: mkdir -p "$RUNNER_TEMP/sqlite-versions/downloads" - - uses: actions/cache@v6 - with: - path: ${{ runner.temp }}/sqlite-versions/downloads - key: setup-sqlite-version-${{ inputs.version }}-amalgamation-v1 - - id: build - shell: bash - run: bash "$GITHUB_ACTION_PATH/setup-sqlite-version.sh" - env: - SQLITE_VERSION: ${{ inputs.version }} - SQLITE_CFLAGS: ${{ inputs.cflags }} - SQLITE_SKIP_ACTIVATE: ${{ inputs.skip-activate }} - SQLITE_EXTRA_FALLBACK_URLS: ${{ inputs.fallback-urls }} diff --git a/.github/actions/setup-sqlite-version/setup-sqlite-version.sh b/.github/actions/setup-sqlite-version/setup-sqlite-version.sh deleted file mode 100644 index 03d6a68f..00000000 --- a/.github/actions/setup-sqlite-version/setup-sqlite-version.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -version_spec="${SQLITE_VERSION:?SQLITE_VERSION is required}" -cflags="${SQLITE_CFLAGS:-}" -skip_activate="${SQLITE_SKIP_ACTIVATE:-false}" -extra_fallback_urls="${SQLITE_EXTRA_FALLBACK_URLS:-}" - -case "$version_spec" in - 3.46 | 3.46.0) - sqlite_version="3.46.0" - sqlite_year="2024" - amalgamation_id="3460000" - builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3460000.zip" - ;; - 3.25 | 3.25.0) - sqlite_version="3.25.0" - sqlite_year="2018" - amalgamation_id="3250000" - builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3250000.zip?v=1" - ;; - *) - echo "::error::Unsupported SQLite version '$version_spec'. Add its release year and amalgamation id to $GITHUB_ACTION_PATH/setup-sqlite-version.sh." - exit 1 - ;; -esac - -case "$(uname -s)" in - Linux) - library_name="libsqlite3.so.0" - library_path_var="LD_LIBRARY_PATH" - ;; - Darwin) - library_name="libsqlite3.dylib" - library_path_var="DYLD_LIBRARY_PATH" - ;; - *) - echo "::error::Unsupported platform $(uname -s)" - exit 1 - ;; -esac - -runner_temp="${RUNNER_TEMP:-}" -if [ -z "$runner_temp" ]; then - runner_temp="$(mktemp -d)" -fi - -filename="sqlite-amalgamation-${amalgamation_id}" -official_url="https://www.sqlite.org/${sqlite_year}/${filename}.zip" -download_dir="${runner_temp}/sqlite-versions/downloads" -source_root="${runner_temp}/sqlite-versions/source" -source_dir="${source_root}/${filename}" -build_dir="${runner_temp}/sqlite-versions/build/${sqlite_version}" -archive_path="${download_dir}/${filename}.zip" - -mkdir -p "$download_dir" "$source_root" "$build_dir" - -download_archive() { - local url - local candidate_path="${archive_path}.tmp" - local urls=("$official_url") - - for url in $builtin_fallback_urls $extra_fallback_urls; do - urls+=("$url") - done - - rm -f "$candidate_path" - for url in "${urls[@]}"; do - echo "Downloading SQLite ${sqlite_version} amalgamation from ${url}" - if curl \ - --fail \ - --location \ - --show-error \ - --retry 5 \ - --retry-delay 2 \ - --retry-max-time 180 \ - --retry-all-errors \ - --connect-timeout 20 \ - --max-time 240 \ - --output "$candidate_path" \ - "$url"; then - mv "$candidate_path" "$archive_path" - return 0 - fi - - echo "::warning::Download failed from ${url}" - rm -f "$candidate_path" - done - - echo "::error::Could not download SQLite ${sqlite_version} amalgamation" - return 1 -} - -if [ ! -f "${source_dir}/sqlite3.c" ]; then - if [ ! -f "$archive_path" ]; then - download_archive - fi - - rm -rf "$source_dir" - unzip -q "$archive_path" -d "$source_root" -fi - -if [ ! -f "${source_dir}/sqlite3.c" ]; then - echo "::error::Expected ${source_dir}/sqlite3.c after extracting ${archive_path}" - exit 1 -fi - -read -r -a cflag_args <<< "$cflags" - -echo "Compiling SQLite ${sqlite_version} to ${build_dir}/${library_name}" -gcc \ - -fPIC \ - -shared \ - "${cflag_args[@]}" \ - "${source_dir}/sqlite3.c" \ - "-I${source_dir}" \ - -o "${build_dir}/${library_name}" - -if [ "$library_name" = "libsqlite3.so.0" ]; then - ln -sf "$library_name" "${build_dir}/libsqlite3.so" -fi - -if [ -n "${GITHUB_OUTPUT:-}" ]; then - echo "sqlite-location=${build_dir}" >> "$GITHUB_OUTPUT" -else - echo "sqlite-location=${build_dir}" -fi - -case "$(printf '%s' "$skip_activate" | tr '[:upper:]' '[:lower:]')" in - true | 1 | yes) - echo "Skipping ${library_path_var} activation" - ;; - *) - existing_value="${!library_path_var:-}" - if [ -n "${GITHUB_ENV:-}" ]; then - if [ -n "$existing_value" ]; then - echo "${library_path_var}=${build_dir}:${existing_value}" >> "$GITHUB_ENV" - else - echo "${library_path_var}=${build_dir}" >> "$GITHUB_ENV" - fi - fi - echo "Added ${build_dir} to ${library_path_var}" - ;; -esac diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 88bb03b1..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: 2 -updates: -- package-ecosystem: pip - directory: "/" - schedule: - interval: daily - time: "13:00" - groups: - python-packages: - patterns: - - "*" diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml deleted file mode 100644 index 3fc83438..00000000 --- a/.github/workflows/deploy-latest.yml +++ /dev/null @@ -1,132 +0,0 @@ -name: Deploy latest.datasette.io - -on: - workflow_dispatch: - push: - branches: - - main - # - 1.0-dev - -permissions: - contents: read - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - name: Check out datasette - uses: actions/checkout@v7 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: "3.13" - cache: pip - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - python -m pip install . --group dev - python -m pip install sphinx-to-sqlite==0.1a1 - - name: Run tests - if: ${{ github.ref == 'refs/heads/main' }} - run: | - pytest -n auto -m "not serial" - pytest -m "serial" - - name: Build fixtures.db and other files needed to deploy the demo - run: |- - python tests/fixtures.py \ - fixtures.db \ - fixtures-config.json \ - fixtures-metadata.json \ - plugins \ - --extra-db-filename extra_database.db - - name: Build docs.db - if: ${{ github.ref == 'refs/heads/main' }} - run: |- - cd docs - DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build - sphinx-to-sqlite ../docs.db _build - cd .. - - name: Set up the alternate-route demo - run: | - echo ' - from datasette import hookimpl - - @hookimpl - def startup(datasette): - db = datasette.get_database("fixtures2") - db.route = "alternative-route" - ' > plugins/alternative_route.py - cp fixtures.db fixtures2.db - - name: And the counters writable stored query demo - run: | - cat > plugins/counters.py < metadata.json - # cat metadata.json - - id: auth - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v3 - with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v3 - - name: Deploy to Cloud Run - env: - LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }} - run: |- - gcloud config set run/region us-central1 - gcloud config set project datasette-222320 - export SUFFIX="-${GITHUB_REF#refs/heads/}" - export SUFFIX=${SUFFIX#-main} - # Replace 1.0 with one-dot-zero in SUFFIX - export SUFFIX=${SUFFIX//1.0/one-dot-zero} - datasette publish cloudrun fixtures.db fixtures2.db extra_database.db \ - -m fixtures-metadata.json \ - --plugins-dir=plugins \ - --branch=$GITHUB_SHA \ - --version-note=$GITHUB_SHA \ - --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb --root" \ - --install 'datasette-ephemeral-tables>=0.2.2' \ - --service "datasette-latest$SUFFIX" \ - --secret $LATEST_DATASETTE_SECRET - - name: Deploy to docs as well (only for main) - if: ${{ github.ref == 'refs/heads/main' }} - run: |- - # Deploy docs.db to a different service - datasette publish cloudrun docs.db \ - --branch=$GITHUB_SHA \ - --version-note=$GITHUB_SHA \ - --extra-options="--setting template_debug 1" \ - --service=datasette-docs-latest diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml deleted file mode 100644 index f5b8dbf6..00000000 --- a/.github/workflows/playwright.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Playwright - -on: - push: - pull_request: - workflow_dispatch: - -permissions: - contents: read - -jobs: - test: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - browser: [chromium, firefox, webkit] - steps: - - uses: actions/checkout@v7 - - name: Set up Python 3.14 - uses: actions/setup-python@v6 - with: - python-version: "3.14" - allow-prereleases: true - cache: pip - cache-dependency-path: pyproject.toml - - name: Cache uv - uses: actions/cache@v6 - with: - path: ~/.cache/uv - key: ${{ runner.os }}-py3.14-uv-${{ hashFiles('pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-py3.14-uv- - - name: Cache Playwright browsers - uses: actions/cache@v6 - with: - path: ~/.cache/ms-playwright/ - key: ${{ runner.os }}-playwright-${{ matrix.browser }}-${{ hashFiles('pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-playwright-${{ matrix.browser }}- - - name: Install uv - run: python -m pip install uv - - name: Install dependencies - run: uv sync --group dev --group playwright - - name: Install ${{ matrix.browser }} - run: uv run --group dev --group playwright playwright install --with-deps ${{ matrix.browser }} - - name: Run Playwright tests - run: uv run --group dev --group playwright pytest tests/test_playwright.py --playwright --browser ${{ matrix.browser }} diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml deleted file mode 100644 index d92ab82b..00000000 --- a/.github/workflows/prettier.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Check JavaScript for conformance with Prettier - -on: [push] - -permissions: - contents: read - -jobs: - prettier: - runs-on: ubuntu-latest - steps: - - name: Check out repo - uses: actions/checkout@v7 - - uses: actions/cache@v6 - name: Configure npm caching - with: - path: ~/.npm - key: ${{ runner.OS }}-npm-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.OS }}-npm- - - name: Install dependencies - run: npm ci - - name: Run prettier - run: |- - npm run prettier -- --check diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 21ed4c12..00000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,109 +0,0 @@ -name: Publish Python Package - -on: - release: - types: [created] - -permissions: - contents: read - -jobs: - test: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] - steps: - - uses: actions/checkout@v7 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - cache: pip - cache-dependency-path: pyproject.toml - - name: Install dependencies - run: | - pip install . --group dev - - name: Run tests - run: | - pytest - - deploy: - runs-on: ubuntu-latest - needs: [test] - environment: release - permissions: - id-token: write - steps: - - uses: actions/checkout@v7 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.13' - cache: pip - cache-dependency-path: pyproject.toml - - name: Install dependencies - run: | - pip install setuptools wheel build - - name: Build - run: | - python -m build - - name: Publish - uses: pypa/gh-action-pypi-publish@release/v1 - - deploy_static_docs: - runs-on: ubuntu-latest - needs: [deploy] - if: "!github.event.release.prerelease" - steps: - - uses: actions/checkout@v7 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.10' - cache: pip - cache-dependency-path: pyproject.toml - - name: Install dependencies - run: | - python -m pip install . --group dev - python -m pip install sphinx-to-sqlite==0.1a1 - - name: Build docs.db - run: |- - cd docs - DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build - sphinx-to-sqlite ../docs.db _build - cd .. - - id: auth - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v3 - - name: Deploy stable-docs.datasette.io to Cloud Run - run: |- - gcloud config set run/region us-central1 - gcloud config set project datasette-222320 - datasette publish cloudrun docs.db \ - --service=datasette-docs-stable - - deploy_docker: - runs-on: ubuntu-latest - needs: [deploy] - if: "!github.event.release.prerelease" - steps: - - uses: actions/checkout@v7 - - name: Build and push to Docker Hub - env: - DOCKER_USER: ${{ secrets.DOCKER_USER }} - DOCKER_PASS: ${{ secrets.DOCKER_PASS }} - run: |- - sleep 60 # Give PyPI time to make the new release available - docker login -u $DOCKER_USER -p $DOCKER_PASS - export REPO=datasetteproject/datasette - docker build -f Dockerfile \ - -t $REPO:${GITHUB_REF#refs/tags/} \ - --build-arg VERSION=${GITHUB_REF#refs/tags/} . - docker tag $REPO:${GITHUB_REF#refs/tags/} $REPO:latest - docker push $REPO:${GITHUB_REF#refs/tags/} - docker push $REPO:latest diff --git a/.github/workflows/push_docker_tag.yml b/.github/workflows/push_docker_tag.yml deleted file mode 100644 index c5a4f0db..00000000 --- a/.github/workflows/push_docker_tag.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Push specific Docker tag - -on: - workflow_dispatch: - inputs: - version_tag: - description: Tag to build and push - -permissions: - contents: read - -jobs: - deploy_docker: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - name: Build and push to Docker Hub - env: - DOCKER_USER: ${{ secrets.DOCKER_USER }} - DOCKER_PASS: ${{ secrets.DOCKER_PASS }} - VERSION_TAG: ${{ github.event.inputs.version_tag }} - run: |- - docker login -u $DOCKER_USER -p $DOCKER_PASS - export REPO=datasetteproject/datasette - docker build -f Dockerfile \ - -t $REPO:${VERSION_TAG} \ - --build-arg VERSION=${VERSION_TAG} . - docker push $REPO:${VERSION_TAG} diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml deleted file mode 100644 index 58635025..00000000 --- a/.github/workflows/spellcheck.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Check spelling in documentation - -on: [push, pull_request] - -permissions: - contents: read - -jobs: - spellcheck: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.11' - cache: 'pip' - cache-dependency-path: '**/pyproject.toml' - - name: Install dependencies - run: | - pip install . --group dev - - name: Check spelling - run: | - codespell README.md --ignore-words docs/codespell-ignore-words.txt - codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt - codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt - codespell tests --ignore-words docs/codespell-ignore-words.txt diff --git a/.github/workflows/stable-docs.yml b/.github/workflows/stable-docs.yml deleted file mode 100644 index ecde5940..00000000 --- a/.github/workflows/stable-docs.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: Update Stable Docs - -on: - release: - types: [published] - push: - branches: - - main - -permissions: - contents: write - -jobs: - update_stable_docs: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v7 - with: - fetch-depth: 0 # We need all commits to find docs/ changes - - name: Set up Git user - run: | - git config user.name "Automated" - git config user.email "actions@users.noreply.github.com" - - name: Create stable branch if it does not yet exist - run: | - if ! git ls-remote --heads origin stable | grep -qE '\bstable\b'; then - # Make sure we have all tags locally - git fetch --tags --quiet - - # Latest tag that is just numbers and dots (optionally prefixed with 'v') - # e.g., 0.65.2 or v0.65.2 — excludes 1.0a20, 1.0-rc1, etc. - LATEST_RELEASE=$( - git tag -l --sort=-v:refname \ - | grep -E '^v?[0-9]+(\.[0-9]+){1,3}$' \ - | head -n1 - ) - - git checkout -b stable - - # If there are any stable releases, copy docs/ from the most recent - if [ -n "$LATEST_RELEASE" ]; then - rm -rf docs/ - git checkout "$LATEST_RELEASE" -- docs/ || true - fi - - git commit -m "Populate docs/ from $LATEST_RELEASE" || echo "No changes" - git push -u origin stable - fi - - name: Handle Release - if: github.event_name == 'release' && !github.event.release.prerelease - run: | - git fetch --all - git checkout stable - git reset --hard ${GITHUB_REF#refs/tags/} - git push origin stable --force - - name: Handle Commit to Main - if: contains(github.event.head_commit.message, '!stable-docs') - run: | - git fetch origin - git checkout -b stable origin/stable - # Get the list of modified files in docs/ from the current commit - FILES=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} -- docs/) - # Check if the list of files is non-empty - if [[ -n "$FILES" ]]; then - # Checkout those files to the stable branch to over-write with their contents - for FILE in $FILES; do - git checkout ${{ github.sha }} -- $FILE - done - git add docs/ - git commit -m "Doc changes from ${{ github.sha }}" - git push origin stable - else - echo "No changes to docs/ in this commit." - exit 0 - fi diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml deleted file mode 100644 index e9bd4bab..00000000 --- a/.github/workflows/test-coverage.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Calculate test coverage - -on: - push: - branches: - - main - pull_request: - branches: - - main -permissions: - contents: read - -jobs: - test: - runs-on: ubuntu-latest - steps: - - name: Check out datasette - uses: actions/checkout@v7 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.12' - cache: 'pip' - cache-dependency-path: '**/pyproject.toml' - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - python -m pip install . --group dev - python -m pip install pytest-cov - - name: Run tests - run: |- - ls -lah - cat .coveragerc - pytest -m "not serial" --cov=datasette --cov-config=.coveragerc --cov-report xml:coverage.xml --cov-report term -x - ls -lah - - name: Upload coverage report - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: coverage.xml diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml deleted file mode 100644 index 5e81ed82..00000000 --- a/.github/workflows/test-pyodide.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Test in Pyodide with shot-scraper - -on: - push: - pull_request: - workflow_dispatch: - -permissions: - contents: read - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - name: Set up Python 3.10 - uses: actions/setup-python@v6 - with: - python-version: "3.10" - cache: 'pip' - cache-dependency-path: '**/pyproject.toml' - - name: Cache Playwright browsers - uses: actions/cache@v6 - with: - path: ~/.cache/ms-playwright/ - key: ${{ runner.os }}-browsers - - name: Install Playwright dependencies - run: | - pip install shot-scraper build - shot-scraper install - - name: Run test - run: | - ./test-in-pyodide-with-shot-scraper.sh diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml deleted file mode 100644 index 2fdb3a40..00000000 --- a/.github/workflows/test-sqlite-support.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Test SQLite versions - -on: [push, pull_request] - -permissions: - contents: read - -jobs: - test: - runs-on: ${{ matrix.platform }} - continue-on-error: true - strategy: - matrix: - platform: [ubuntu-latest] - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] - sqlite-version: [ - #"3", # latest version - "3.46", - #"3.45", - #"3.27", - #"3.26", - "3.25", - #"3.25.3", # 2018-09-25, window functions breaks test_upsert for some reason on 3.10, skip for now - #"3.24", # 2018-06-04, added UPSERT support - #"3.23.1" # 2018-04-10, before UPSERT - ] - steps: - - uses: actions/checkout@v7 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - allow-prereleases: true - cache: pip - cache-dependency-path: pyproject.toml - - name: Set up SQLite ${{ matrix.sqlite-version }} - uses: ./.github/actions/setup-sqlite-version - with: - version: ${{ matrix.sqlite-version }} - cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1" - - run: python3 -c "import sqlite3; print(sqlite3.sqlite_version)" - - run: echo $LD_LIBRARY_PATH - - name: Build extension for --load-extension test - run: |- - (cd tests && gcc ext.c -fPIC -shared -o ext.so) - - name: Install dependencies - run: | - pip install . --group dev - pip freeze - - name: Run tests - run: | - pytest -n auto -m "not serial" - pytest -m "serial" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 751eedfd..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Test - -on: [push, pull_request] - -permissions: - contents: read - -jobs: - test: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] - steps: - - uses: actions/checkout@v7 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - allow-prereleases: true - cache: pip - cache-dependency-path: pyproject.toml - - name: Build extension for --load-extension test - run: |- - (cd tests && gcc ext.c -fPIC -shared -o ext.so) - - name: Install dependencies - run: | - pip install . --group dev - pip freeze - - name: Run tests - run: | - pytest -n auto -m "not serial" - pytest -m "serial" - # And the test that exceeds a localhost HTTPS server - tests/test_datasette_https_server.sh - - name: Black - run: | - black --version - black --check . - - name: Ruff - run: ruff check datasette tests - - name: Check if cog needs to be run - run: | - cog --check docs/*.rst - - name: Check if blacken-docs needs to be run - run: | - # This fails on syntax errors, or a diff was applied - blacken-docs -l 60 docs/*.rst - - name: Test DATASETTE_LOAD_PLUGINS - run: | - pip install datasette-init datasette-json-html - tests/test-datasette-load-plugins.sh diff --git a/.github/workflows/tmate-mac.yml b/.github/workflows/tmate-mac.yml deleted file mode 100644 index f2c074a6..00000000 --- a/.github/workflows/tmate-mac.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: tmate session mac - -on: - workflow_dispatch: - -permissions: - contents: read - -jobs: - build: - runs-on: macos-latest - steps: - - uses: actions/checkout@v7 - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/tmate.yml b/.github/workflows/tmate.yml deleted file mode 100644 index 5b8818c3..00000000 --- a/.github/workflows/tmate.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: tmate session - -on: - workflow_dispatch: - -permissions: - contents: read - models: read - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 2a7f6620..cf8578ee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,26 +1,11 @@ build-metadata.json datasets.json -.playwright-mcp - scratchpad -ignored/ - -.vscode - -uv.lock -data.db - -# test databases +# SQLite databases *.db - -# We don't use Pipfile, so ignore them -Pipfile -Pipfile.lock - -fixtures.db -*test.db +*.sqlite # Byte-compiled / optimized / DLL files __pycache__/ @@ -126,13 +111,3 @@ ENV/ # macOS files .DS_Store -node_modules -.*.swp - -# In case someone compiled tests/ext.c for test_load_extensions, don't -# include it in source control. -tests/*.dylib -tests/*.so -tests/*.dll - -.idea diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index 0cece53b..00000000 --- a/.isort.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[settings] -multi_line_output=3 - diff --git a/.prettierrc b/.prettierrc deleted file mode 100644 index 222861c3..00000000 --- a/.prettierrc +++ /dev/null @@ -1,4 +0,0 @@ -{ - "tabWidth": 2, - "useTabs": false -} diff --git a/.readthedocs.yaml b/.readthedocs.yaml deleted file mode 100644 index 8b3e54aa..00000000 --- a/.readthedocs.yaml +++ /dev/null @@ -1,17 +0,0 @@ -version: 2 - -sphinx: - configuration: docs/conf.py - -build: - os: ubuntu-24.04 - tools: - python: "3.13" - jobs: - install: - - pip install --upgrade pip - - pip install . --group dev - -formats: -- pdf -- epub diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..fc01da26 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,8 @@ +language: python + +python: + - 3.5 + - 3.6 + +script: + - python setup.py test diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md deleted file mode 100644 index 14d4c567..00000000 --- a/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,128 +0,0 @@ -# Contributor Covenant Code of Conduct - -## Our Pledge - -We as members, contributors, and leaders pledge to make participation in our -community a harassment-free experience for everyone, regardless of age, body -size, visible or invisible disability, ethnicity, sex characteristics, gender -identity and expression, level of experience, education, socio-economic status, -nationality, personal appearance, race, religion, or sexual identity -and orientation. - -We pledge to act and interact in ways that contribute to an open, welcoming, -diverse, inclusive, and healthy community. - -## Our Standards - -Examples of behavior that contributes to a positive environment for our -community include: - -* Demonstrating empathy and kindness toward other people -* Being respectful of differing opinions, viewpoints, and experiences -* Giving and gracefully accepting constructive feedback -* Accepting responsibility and apologizing to those affected by our mistakes, - and learning from the experience -* Focusing on what is best not just for us as individuals, but for the - overall community - -Examples of unacceptable behavior include: - -* The use of sexualized language or imagery, and sexual attention or - advances of any kind -* Trolling, insulting or derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or email - address, without their explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Enforcement Responsibilities - -Community leaders are responsible for clarifying and enforcing our standards of -acceptable behavior and will take appropriate and fair corrective action in -response to any behavior that they deem inappropriate, threatening, offensive, -or harmful. - -Community leaders have the right and responsibility to remove, edit, or reject -comments, commits, code, wiki edits, issues, and other contributions that are -not aligned to this Code of Conduct, and will communicate reasons for moderation -decisions when appropriate. - -## Scope - -This Code of Conduct applies within all community spaces, and also applies when -an individual is officially representing the community in public spaces. -Examples of representing our community include using an official e-mail address, -posting via an official social media account, or acting as an appointed -representative at an online or offline event. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported to the community leaders responsible for enforcement at -`swillison+datasette-code-of-conduct@gmail.com`. -All complaints will be reviewed and investigated promptly and fairly. - -All community leaders are obligated to respect the privacy and security of the -reporter of any incident. - -## Enforcement Guidelines - -Community leaders will follow these Community Impact Guidelines in determining -the consequences for any action they deem in violation of this Code of Conduct: - -### 1. Correction - -**Community Impact**: Use of inappropriate language or other behavior deemed -unprofessional or unwelcome in the community. - -**Consequence**: A private, written warning from community leaders, providing -clarity around the nature of the violation and an explanation of why the -behavior was inappropriate. A public apology may be requested. - -### 2. Warning - -**Community Impact**: A violation through a single incident or series -of actions. - -**Consequence**: A warning with consequences for continued behavior. No -interaction with the people involved, including unsolicited interaction with -those enforcing the Code of Conduct, for a specified period of time. This -includes avoiding interactions in community spaces as well as external channels -like social media. Violating these terms may lead to a temporary or -permanent ban. - -### 3. Temporary Ban - -**Community Impact**: A serious violation of community standards, including -sustained inappropriate behavior. - -**Consequence**: A temporary ban from any sort of interaction or public -communication with the community for a specified period of time. No public or -private interaction with the people involved, including unsolicited interaction -with those enforcing the Code of Conduct, is allowed during this period. -Violating these terms may lead to a permanent ban. - -### 4. Permanent Ban - -**Community Impact**: Demonstrating a pattern of violation of community -standards, including sustained inappropriate behavior, harassment of an -individual, or aggression toward or disparagement of classes of individuals. - -**Consequence**: A permanent ban from any sort of public interaction within -the community. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], -version 2.0, available at -https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. - -Community Impact Guidelines were inspired by [Mozilla's code of conduct -enforcement ladder](https://github.com/mozilla/diversity). - -[homepage]: https://www.contributor-covenant.org - -For answers to common questions about this code of conduct, see the FAQ at -https://www.contributor-covenant.org/faq. Translations are available at -https://www.contributor-covenant.org/translations. diff --git a/Dockerfile b/Dockerfile index 9a8f06cf..04c08d2c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,21 @@ -FROM python:3.11.0-slim-bullseye as build +FROM python:3.6-slim-stretch as build -# Version of Datasette to install, e.g. 0.55 -# docker build . -t datasette --build-arg VERSION=0.55 -ARG VERSION +# Setup build dependencies +RUN apt update +RUN apt install -y python3-dev gcc libsqlite3-mod-spatialite +# Add local code to the image instead of fetching from pypi. +ADD . /datasette -RUN apt-get update && \ - apt-get install -y --no-install-recommends libsqlite3-mod-spatialite && \ - apt clean && \ - rm -rf /var/lib/apt && \ - rm -rf /var/lib/dpkg/info/* +RUN pip install /datasette -RUN pip install https://github.com/simonw/datasette/archive/refs/tags/${VERSION}.zip && \ - find /usr/local/lib -name '__pycache__' | xargs rm -r && \ - rm -rf /root/.cache/pip +FROM python:3.6-slim-stretch + +# Copy python dependencies +COPY --from=build /usr/local/lib/python3.6/site-packages /usr/local/lib/python3.6/site-packages +# Copy executables +COPY --from=build /usr/local/bin /usr/local/bin +# Copy spatial extensions +COPY --from=build /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu EXPOSE 8001 CMD ["datasette"] diff --git a/Justfile b/Justfile deleted file mode 100644 index 6ffff870..00000000 --- a/Justfile +++ /dev/null @@ -1,77 +0,0 @@ -export DATASETTE_SECRET := "not_a_secret" - -# Run tests and linters -@default: test lint - -# Setup project -@init: - uv sync - -# Run pytest with supplied options -@test *options: init - uv run pytest -n auto {{options}} - -# Install Playwright browser support, Chromium by default -@playwright-install browser="chromium": - uv run --group playwright playwright install {{browser}} - -# Install all Playwright browsers used by the test suite -@playwright-install-all: - uv run --group playwright playwright install chromium firefox webkit - -# Run Playwright tests, Chromium by default -@playwright browser="chromium" *options: - uv run --group playwright pytest tests/test_playwright.py --playwright --browser {{browser}} {{options}} - -# Run Playwright tests against all supported browsers -@playwright-all *options: - uv run --group playwright pytest tests/test_playwright.py --playwright --browser chromium --browser firefox --browser webkit {{options}} - -@codespell: - uv run codespell README.md --ignore-words docs/codespell-ignore-words.txt - uv run codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt - uv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt - uv run codespell tests --ignore-words docs/codespell-ignore-words.txt - -# Run linters: black, ruff, prettier, cog -@lint: codespell - uv run black datasette tests --check - uv run ruff check datasette tests - npm run prettier -- --check - uv run cog --check README.md docs/*.rst - -# Apply ruff fixes -@fix: - uv run ruff check --fix datasette tests - -# Rebuild docs with cog -@cog: - uv run cog -r README.md docs/*.rst - -# Serve live docs on localhost:8000 -@docs: cog blacken-docs - uv run make -C docs livehtml - -# Build docs as static HTML -@docs-build: cog blacken-docs - rm -rf docs/_build && cd docs && uv run make html - -# Apply Black -@black: - uv run black datasette tests - -# Apply blacken-docs -@blacken-docs: - uv run blacken-docs -l 60 docs/*.rst - -# Apply prettier -@prettier: - npm run fix - -# Format code with both black and prettier -@format: black prettier blacken-docs - -@serve *options: - uv run sqlite-utils create-database data.db - uv run sqlite-utils create-table data.db docs id integer title text --pk id --ignore - uv run python -m datasette data.db --root --reload {{options}} diff --git a/MANIFEST.in b/MANIFEST.in index 8c5e3ee6..696c1146 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1 @@ recursive-include datasette/static * -recursive-include datasette/templates * -include versioneer.py -include datasette/_version.py -include LICENSE diff --git a/README.md b/README.md index 393e8e5c..f5d1d074 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,34 @@ -Datasette +# Datasette -[![PyPI](https://img.shields.io/pypi/v/datasette.svg)](https://pypi.org/project/datasette/) -[![Changelog](https://img.shields.io/github/v/release/simonw/datasette?label=changelog)](https://docs.datasette.io/en/latest/changelog.html) -[![Python 3.x](https://img.shields.io/pypi/pyversions/datasette.svg?logo=python&logoColor=white)](https://pypi.org/project/datasette/) -[![Tests](https://github.com/simonw/datasette/workflows/Test/badge.svg)](https://github.com/simonw/datasette/actions?query=workflow%3ATest) -[![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](https://docs.datasette.io/en/latest/?badge=latest) -[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette/blob/main/LICENSE) -[![docker: datasette](https://img.shields.io/badge/docker-datasette-blue)](https://hub.docker.com/r/datasetteproject/datasette) -[![discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://datasette.io/discord) +[![PyPI](https://img.shields.io/pypi/v/datasette.svg)](https://pypi.python.org/pypi/datasette) +[![Travis CI](https://travis-ci.org/simonw/datasette.svg?branch=master)](https://travis-ci.org/simonw/datasette) +[![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](http://datasette.readthedocs.io/en/latest/?badge=latest) -*An open source multi-tool for exploring and publishing data* +*An instant JSON API for your SQLite databases* -Datasette is a tool for exploring and publishing data. It helps people take data of any shape or size and publish that as an interactive, explorable website and accompanying API. +Datasette provides an instant, read-only JSON API for any SQLite database. It also provides tools for packaging the database up as a Docker container and deploying that container to hosting providers such as [Zeit Now](https://zeit.co/now). -Datasette is aimed at data journalists, museum curators, archivists, local governments, scientists, researchers and anyone else who has data that they wish to share with the world. +Got CSV data? Use [csvs-to-sqlite](https://github.com/simonw/csvs-to-sqlite) to convert them to SOLite, then publish them with Datasette. Or try [Datasette Publish](https://publish.datasettes.com), a web app that lets you upload CSV data and deploy it using Datasette without needing to install any software. -[Explore a demo](https://datasette.io/global-power-plants/global-power-plants), watch [a video about the project](https://simonwillison.net/2021/Feb/7/video/) or try it out [on GitHub Codespaces](https://github.com/datasette/datasette-studio). +Some examples: https://github.com/simonw/datasette/wiki/Datasettes -* [datasette.io](https://datasette.io/) is the official project website -* Latest [Datasette News](https://datasette.io/news) -* Comprehensive documentation: https://docs.datasette.io/ -* Examples: https://datasette.io/examples -* Live demo of current `main` branch: https://latest.datasette.io/ -* Questions, feedback or want to talk about the project? Join our [Discord](https://datasette.io/discord) +## News -Want to stay up-to-date with the project? Subscribe to the [Datasette newsletter](https://datasette.substack.com/) for tips, tricks and news on what's new in the Datasette ecosystem. +* 9th April 2018: [Datasette 0.15: sort by column](https://github.com/simonw/datasette/releases/tag/0.15) +* 28th March 2018: [Baltimore Sun Public Salary Records](https://simonwillison.net/2018/Mar/28/datasette-in-the-wild/) - a data journalism project from the Baltimore Sun powered by Datasette - source code [is available here](https://github.com/baltimore-sun-data/salaries-datasette) +* 27th March 2018: [Cloud-first: Rapid webapp deployment using containers](https://wwwf.imperial.ac.uk/blog/research-software-engineering/2018/03/27/cloud-first-rapid-webapp-deployment-using-containers/) - a tutorial covering deploying Datasette using Microsoft Azure by the Research Software Engineering team at Imperial College London +* 28th January 2018: [Analyzing my Twitter followers with Datasette](https://simonwillison.net/2018/Jan/28/analyzing-my-twitter-followers/) - a tutorial on using Datasette to analyze follower data pulled from the Twitter API +* 17th January 2018: [Datasette Publish: a web app for publishing CSV files as an online database](https://simonwillison.net/2018/Jan/17/datasette-publish/) +* 12th December 2017: [Building a location to time zone API with SpatiaLite, OpenStreetMap and Datasette](https://simonwillison.net/2017/Dec/12/building-a-location-time-zone-api/) +* 9th December 2017: [Datasette 0.14: customization edition](https://github.com/simonw/datasette/releases/tag/0.14) +* 25th November 2017: [New in Datasette: filters, foreign keys and search](https://simonwillison.net/2017/Nov/25/new-in-datasette/) +* 13th November 2017: [Datasette: instantly create and publish an API for your SQLite databases](https://simonwillison.net/2017/Nov/13/datasette/) ## Installation -If you are on a Mac, [Homebrew](https://brew.sh/) is the easiest way to install Datasette: + pip3 install datasette - brew install datasette - -You can also install it using `pip` or `pipx`: - - pip install datasette - -Datasette requires Python 3.8 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker. +Datasette requires Python 3.5 or higher. ## Basic usage @@ -48,12 +40,86 @@ This will start a web server on port 8001 - visit http://localhost:8001/ to acce Use Chrome on OS X? You can run datasette against your browser history like so: - datasette ~/Library/Application\ Support/Google/Chrome/Default/History --nolock + datasette ~/Library/Application\ Support/Google/Chrome/Default/History Now visiting http://localhost:8001/History/downloads will show you a web interface to browse your downloads data: ![Downloads table rendered by datasette](https://static.simonwillison.net/static/2017/datasette-downloads.png) +http://localhost:8001/History/downloads.json will return that data as JSON: + + { + "database": "History", + "columns": [ + "id", + "current_path", + "target_path", + "start_time", + "received_bytes", + "total_bytes", + ... + ], + "table_rows_count": 576, + "rows": [ + [ + 1, + "/Users/simonw/Downloads/DropboxInstaller.dmg", + "/Users/simonw/Downloads/DropboxInstaller.dmg", + 13097290269022132, + 626688, + 0, + ... + ] + ] + } + + +http://localhost:8001/History/downloads.json?_shape=objects will return that data as JSON in a more convenient but less efficient format: + + { + ... + "rows": [ + { + "start_time": 13097290269022132, + "interrupt_reason": 0, + "hash": "", + "id": 1, + "site_url": "", + "referrer": "https://www.dropbox.com/downloading?src=index", + ... + } + ] + } + +## datasette serve options + + $ datasette serve --help + Usage: datasette serve [OPTIONS] [FILES]... + + Serve up specified SQLite database files with a web UI + + Options: + -h, --host TEXT host for server, defaults to 127.0.0.1 + -p, --port INTEGER port for server, defaults to 8001 + --debug Enable debug mode - useful for development + --reload Automatically reload if code change detected - + useful for development + --cors Enable CORS by serving Access-Control-Allow- + Origin: * + --page_size INTEGER Page size - default is 100 + --max_returned_rows INTEGER Max allowed rows to return at once - default is + 1000. Set to 0 to disable check entirely. + --sql_time_limit_ms INTEGER Max time allowed for SQL queries in ms + --load-extension PATH Path to a SQLite extension to load + --inspect-file TEXT Path to JSON file created using "datasette + inspect" + -m, --metadata FILENAME Path to JSON file containing license/source + metadata + --template-dir DIRECTORY Path to directory containing custom templates + --static STATIC MOUNT mountpoint:path-to-directory for serving static + files + --help Show this message and exit. + ## metadata.json If you want to include licensing and source information in the generated datasette website you can do so using a JSON file that looks something like this: @@ -66,26 +132,108 @@ If you want to include licensing and source information in the generated dataset "source_url": "https://github.com/fivethirtyeight/data" } -Save this in `metadata.json` and run Datasette like so: - - datasette serve fivethirtyeight.db -m metadata.json - The license and source information will be displayed on the index page and in the footer. They will also be included in the JSON produced by the API. ## datasette publish -If you have [Heroku](https://heroku.com/) or [Google Cloud Run](https://cloud.google.com/run/) configured, Datasette can deploy one or more SQLite databases to the internet with a single command: +If you have [Zeit Now](https://zeit.co/now) or [Heroku](https://heroku.com/) configured, datasette can deploy one or more SQLite databases to the internet with a single command: - datasette publish heroku database.db + datasette publish now database.db Or: - datasette publish cloudrun database.db + datasette publish heroku database.db -This will create a docker image containing both the datasette application and the specified SQLite database files. It will then deploy that image to Heroku or Cloud Run and give you a URL to access the resulting website and API. +This will create a docker image containing both the datasette application and the specified SQLite database files. It will then deploy that image to Zeit Now or Heroku and give you a URL to access the API. -See [Publishing data](https://docs.datasette.io/en/stable/publish.html) in the documentation for more details. + $ datasette publish --help + Usage: datasette publish [OPTIONS] PUBLISHER [FILES]... -## Datasette Lite + Publish specified SQLite database files to the internet along with a + datasette API. -[Datasette Lite](https://lite.datasette.io/) is Datasette packaged using WebAssembly so that it runs entirely in your browser, no Python web application server required. Read more about that in the [Datasette Lite documentation](https://github.com/simonw/datasette-lite/blob/main/README.md). + Options for PUBLISHER: * 'now' - You must have Zeit Now installed: + https://zeit.co/now * 'heroku' - You must have Heroku installed: + https://cli.heroku.com/ + + Example usage: datasette publish now my-database.db + + Options: + -n, --name TEXT Application name to use when deploying to Now + (ignored for Heroku) + -m, --metadata FILENAME Path to JSON file containing metadata to publish + --extra-options TEXT Extra options to pass to datasette serve + --force Pass --force option to now + --branch TEXT Install datasette from a GitHub branch e.g. master + --template-dir DIRECTORY Path to directory containing custom templates + --static STATIC MOUNT mountpoint:path-to-directory for serving static + files + --title TEXT Title for metadata + --license TEXT License label for metadata + --license_url TEXT License URL for metadata + --source TEXT Source label for metadata + --source_url TEXT Source URL for metadata + --help Show this message and exit. + +## datasette package + +If you have docker installed you can use `datasette package` to create a new Docker image in your local repository containing the datasette app and selected SQLite databases: + + $ datasette package --help + Usage: datasette package [OPTIONS] FILES... + + Package specified SQLite files into a new datasette Docker container + + Options: + -t, --tag TEXT Name for the resulting Docker container, can + optionally use name:tag format + -m, --metadata FILENAME Path to JSON file containing metadata to publish + --extra-options TEXT Extra options to pass to datasette serve + --branch TEXT Install datasette from a GitHub branch e.g. master + --template-dir DIRECTORY Path to directory containing custom templates + --static STATIC MOUNT mountpoint:path-to-directory for serving static + files + --title TEXT Title for metadata + --license TEXT License label for metadata + --license_url TEXT License URL for metadata + --source TEXT Source label for metadata + --source_url TEXT Source URL for metadata + --help Show this message and exit. + +Both publish and package accept an `extra_options` argument option, which will affect how the resulting application is executed. For example, say you want to increase the SQL time limit for a particular container: + + datasette package parlgov.db --extra-options="--sql_time_limit_ms=2500 --page_size=10" + +The resulting container will run the application with those options. + +Here's example output for the package command: + + $ datasette package parlgov.db --extra-options="--sql_time_limit_ms=2500 --page_size=10" + Sending build context to Docker daemon 4.459MB + Step 1/7 : FROM python:3 + ---> 79e1dc9af1c1 + Step 2/7 : COPY . /app + ---> Using cache + ---> cd4ec67de656 + Step 3/7 : WORKDIR /app + ---> Using cache + ---> 139699e91621 + Step 4/7 : RUN pip install datasette + ---> Using cache + ---> 340efa82bfd7 + Step 5/7 : RUN datasette inspect parlgov.db --inspect-file inspect-data.json + ---> Using cache + ---> 5fddbe990314 + Step 6/7 : EXPOSE 8001 + ---> Using cache + ---> 8e83844b0fed + Step 7/7 : CMD datasette serve parlgov.db --port 8001 --inspect-file inspect-data.json --sql_time_limit_ms=2500 --page_size=10 + ---> Using cache + ---> 1bd380ea8af3 + Successfully built 1bd380ea8af3 + +You can now run the resulting container like so: + + docker run -p 8081:8001 1bd380ea8af3 + +This exposes port 8001 inside the container as port 8081 on your host machine, so you can access the application at http://localhost:8081/ diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index bfdc9877..00000000 --- a/codecov.yml +++ /dev/null @@ -1,8 +0,0 @@ -coverage: - status: - project: - default: - informational: true - patch: - default: - informational: true diff --git a/datasette/__init__.py b/datasette/__init__.py index e0022178..668a8c82 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -1,15 +1 @@ -from datasette.permissions import Permission # noqa from datasette.version import __version_info__, __version__ # noqa -from datasette.events import Event # noqa -from datasette.tokens import TokenHandler, TokenInvalid, TokenRestrictions # noqa -from datasette.utils.asgi import ( # noqa - Forbidden, - NotFound, - PayloadTooLarge, - Request, - Response, -) -from datasette.utils import actor_matches_allow # noqa -from datasette.views import Context # noqa -from .hookspecs import hookimpl # noqa -from .hookspecs import hookspec # noqa diff --git a/datasette/__main__.py b/datasette/__main__.py deleted file mode 100644 index 4adef844..00000000 --- a/datasette/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -from datasette.cli import cli - -if __name__ == "__main__": - cli() diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py deleted file mode 100644 index 103c616d..00000000 --- a/datasette/_pytest_plugin.py +++ /dev/null @@ -1,123 +0,0 @@ -""" -Pytest plugin that automatically closes any Datasette instances constructed -during a pytest test — both in the test body and in function-scoped -fixtures. Instances constructed by session-, module-, class- or package- -scoped fixtures are left alone, because other tests in the session will -still want to use them. - -Registered as a pytest11 entry point in pyproject.toml so that downstream -projects using Datasette get the same FD-safety net for their own tests. - -Opt out by setting ``datasette_autoclose = false`` in pytest.ini (or the -equivalent ini file). -""" - -from __future__ import annotations - -import contextvars -import weakref - -import pytest - -_active_instances: contextvars.ContextVar[list | None] = contextvars.ContextVar( - "datasette_active_instances", default=None -) - -_original_init = None - - -def _install_tracking(): - # datasette.app is imported lazily here rather than at module level: - # as a pytest11 entry point this module is imported during pytest - # startup, before pytest-cov starts measuring, so a module-level - # import would drag in all of datasette and make every import-time - # line in the package invisible to coverage - global _original_init - if _original_init is not None: - return - from datasette.app import Datasette - - _original_init = Datasette.__init__ - - def _tracking_init(self, *args, **kwargs): - _original_init(self, *args, **kwargs) - instances = _active_instances.get() - if instances is not None: - instances.append(weakref.ref(self)) - - Datasette.__init__ = _tracking_init - - -def pytest_configure(config): - if _enabled(config): - _install_tracking() - - -def pytest_addoption(parser): - parser.addini( - "datasette_autoclose", - help=( - "Automatically close Datasette instances created inside test " - "bodies and function-scoped fixtures (default: true)." - ), - default="true", - ) - - -def _enabled(config) -> bool: - value = config.getini("datasette_autoclose") - if isinstance(value, bool): - return value - return str(value).strip().lower() not in ("false", "0", "no", "off") - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_protocol(item, nextitem): - """Track Datasette instances across setup, call and teardown; close at end.""" - if not _enabled(item.config): - yield - return - refs: list[weakref.ref] = [] - token = _active_instances.set(refs) - try: - yield - finally: - _active_instances.reset(token) - for ref in reversed(refs): - ds = ref() - if ds is None: - continue - try: - ds.close() - except Exception as e: - item.warn( - pytest.PytestUnraisableExceptionWarning( - f"Error closing Datasette instance: {e!r}" - ) - ) - - -@pytest.hookimpl(hookwrapper=True) -def pytest_fixture_setup(fixturedef, request): - """Exempt instances created by non-function-scoped fixtures. - - Session-, module-, class- and package-scoped fixtures produce Datasette - instances that must survive beyond the current test — other tests in - the session will still use them. When such a fixture creates one or - more Datasette instances during its setup, we snapshot the tracking - list before the fixture runs and subtract off any instances that were - added during its setup, so they don't get closed at test teardown. - """ - refs = _active_instances.get() - if refs is None: - yield - return - before_ids = {id(ref) for ref in refs} - yield - if fixturedef.scope != "function": - new_refs = [ref for ref in refs if id(ref) not in before_ids] - for new_ref in new_refs: - try: - refs.remove(new_ref) - except ValueError: - pass diff --git a/datasette/actor_auth_cookie.py b/datasette/actor_auth_cookie.py deleted file mode 100644 index 368213af..00000000 --- a/datasette/actor_auth_cookie.py +++ /dev/null @@ -1,23 +0,0 @@ -from datasette import hookimpl -from itsdangerous import BadSignature -from datasette.utils import baseconv -import time - - -@hookimpl -def actor_from_request(datasette, request): - if "ds_actor" not in request.cookies: - return None - try: - decoded = datasette.unsign(request.cookies["ds_actor"], "actor") - # If it has "e" and "a" keys process the "e" expiry - if not isinstance(decoded, dict) or "a" not in decoded: - return None - expires_at = decoded.get("e") - if expires_at: - timestamp = int(baseconv.base62.decode(expires_at)) - if time.time() > timestamp: - return None - return decoded["a"] - except BadSignature: - return None diff --git a/datasette/app.py b/datasette/app.py index 4ba5d20f..4f2980ef 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1,3326 +1,1286 @@ -from __future__ import annotations - -import asyncio -import contextvars -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Sequence - -if TYPE_CHECKING: - from datasette.permissions import Resource - from datasette.tokens import TokenRestrictions -import collections -import dataclasses -import datetime -import functools -import glob -import httpx -import importlib.metadata -import inspect -from itsdangerous import BadSignature -import json -import os +from sanic import Sanic +from sanic import response +from sanic.exceptions import NotFound +from sanic.views import HTTPMethodView +from sanic.request import RequestParameters +from jinja2 import Environment, FileSystemLoader, ChoiceLoader, PrefixLoader import re -import secrets -import sys -import threading -import time -import types -import urllib.parse -from concurrent import futures +import sqlite3 from pathlib import Path - -from markupsafe import Markup, escape -from itsdangerous import URLSafeSerializer -from jinja2 import ( - ChoiceLoader, - Environment, - FileSystemLoader, - pass_context, - PrefixLoader, -) -from jinja2.environment import Template -from jinja2.exceptions import TemplateNotFound - -from .events import Event -from .column_types import SQLiteType -from . import stored_queries, write_sql -from .views import Context -from .views.database import ( - database_download, - DatabaseView, - QueryView, -) -from .views.table_create_alter import ( - DatabaseForeignKeyTargetsView, - TableAlterView, - TableCreateView, - TableForeignKeySuggestionsView, -) -from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView -from .views.stored_queries import ( - QueryCreateAnalyzeView, - QueryDeleteView, - QueryDefinitionView, - QueryEditView, - GlobalQueryListView, - QueryListView, - QueryParametersView, - QueryStoreView, - QueryUpdateView, -) -from .views.index import IndexView -from .views.special import ( - JsonDataView, - PatternPortfolioView, - AutocompleteDebugView, - AuthTokenView, - ApiExplorerView, - CreateTokenView, - LogoutView, - AllowDebugView, - PermissionsDebugView, - MessagesDebugView, - AllowedResourcesView, - PermissionRulesView, - PermissionCheckView, - JumpView, - InstanceSchemaView, - DatabaseSchemaView, - TableSchemaView, -) -from .views.table import ( - TableAutocompleteView, - TableInsertView, - TableUpsertView, - TableSetColumnTypeView, - TableDropView, - TableFragmentView, - table_view, -) -from .views.row import RowView, RowDeleteView, RowUpdateView -from .renderer import json_renderer -from .url_builder import Urls -from .database import Database, QueryInterrupted - +from concurrent import futures +import asyncio +import os +import threading +import urllib.parse +import json +import jinja2 +import hashlib +import time +import pint +import traceback from .utils import ( - PaginatedResources, - PrefixedUrlString, - SPATIALITE_FUNCTIONS, - StartupError, - async_call_with_supported_arguments, - await_me_maybe, - baseconv, - call_with_supported_arguments, - detect_json1, - add_cors_headers, - display_actor, + Filters, + CustomJSONEncoder, + compound_keys_after_sql, + detect_fts_sql, + detect_spatialite, escape_css_string, escape_sqlite, - find_spatialite, - format_bytes, - module_from_path, - move_plugins_and_allow, - move_table_config, - parse_metadata, - resolve_env_secrets, - resolve_routes, - sha256_file, - tilde_decode, - tilde_encode, + filters_should_redirect, + get_all_foreign_keys, + is_url, + InvalidSql, + path_from_row_pks, + path_with_added_args, + path_with_ext, + sqlite_timelimit, to_css_class, urlsafe_components, - redact_keys, - row_sql_params_pks, + validate_sql_select, ) -from .tokens import TokenInvalid -from .utils.asgi import ( - AsgiLifespan, - BadRequest, - Forbidden, - NotFound, - DatabaseNotFound, - TableNotFound, - RowNotFound, - Request, - Response, - AsgiRunOnFirstRequest, - asgi_static, - asgi_send, - asgi_send_file, - asgi_send_redirect, -) -from .csrf import CrossOriginProtectionMiddleware -from .utils.internal_db import init_internal_db, populate_schema_tables -from .utils.sqlite import ( - sqlite3, - using_pysqlite3, -) -from .tracer import AsgiTracer -from .plugins import pm, DEFAULT_PLUGINS, get_plugins from .version import __version__ -from .resources import DatabaseResource, TableResource - app_root = Path(__file__).parent.parent +HASH_BLOCK_SIZE = 1024 * 1024 +HASH_LENGTH = 7 -# Context variable to track when code is executing within a datasette.client request -_in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False) +connections = threading.local() +ureg = pint.UnitRegistry() -class _DatasetteClientContext: - """Context manager to mark code as executing within a datasette.client request.""" - - def __enter__(self): - self.token = _in_datasette_client.set(True) - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - _in_datasette_client.reset(self.token) - return False +class DatasetteError(Exception): + def __init__(self, message, title=None, error_dict=None, status=500, template=None): + self.message = message + self.title = title + self.error_dict = error_dict or {} + self.status = status -@dataclasses.dataclass -class PermissionCheck: - """Represents a logged permission check for debugging purposes.""" - - when: str - actor: Dict[str, Any] | None - action: str - parent: str | None - child: str | None - result: bool +class RenderMixin(HTTPMethodView): + def render(self, templates, **context): + template = self.jinja_env.select_template(templates) + select_templates = ['{}{}'.format( + '*' if template_name == template.name else '', + template_name + ) for template_name in templates] + return response.html( + template.render({ + **context, **{ + 'app_css_hash': self.ds.app_css_hash(), + 'select_templates': select_templates, + } + }) + ) -# https://github.com/simonw/datasette/issues/283#issuecomment-781591015 -SQLITE_LIMIT_ATTACHED = 10 +class BaseView(RenderMixin): + re_named_parameter = re.compile(':([a-zA-Z0-9_]+)') -INTERNAL_DB_NAME = "__INTERNAL__" + def __init__(self, datasette): + self.ds = datasette + self.files = datasette.files + self.jinja_env = datasette.jinja_env + self.executor = datasette.executor + self.page_size = datasette.page_size + self.max_returned_rows = datasette.max_returned_rows -Setting = collections.namedtuple("Setting", ("name", "default", "help")) -SETTINGS = ( - Setting("default_page_size", 100, "Default page size for the table view"), - Setting( - "max_returned_rows", - 1000, - "Maximum rows that can be returned from a table or custom query", - ), - Setting( - "max_insert_rows", - 100, - "Maximum rows that can be inserted at a time using the bulk insert API", - ), - Setting( - "max_post_body_bytes", - 2 * 1024 * 1024, - "Maximum size in bytes for a POST body read into memory, e.g. JSON API requests - set 0 to disable this limit", - ), - Setting( - "num_sql_threads", - 3, - "Number of threads in the thread pool for executing SQLite queries", - ), - Setting("sql_time_limit_ms", 1000, "Time limit for a SQL query in milliseconds"), - Setting( - "default_facet_size", 30, "Number of values to return for requested facets" - ), - Setting("facet_time_limit_ms", 200, "Time limit for calculating a requested facet"), - Setting( - "facet_suggest_time_limit_ms", - 50, - "Time limit for calculating a suggested facet", - ), - Setting( - "allow_facet", - True, - "Allow users to specify columns to facet using ?_facet= parameter", - ), - Setting( - "allow_download", - True, - "Allow users to download the original SQLite database files", - ), - Setting( - "allow_signed_tokens", - True, - "Allow users to create and use signed API tokens", - ), - Setting( - "default_allow_sql", - True, - "Allow anyone to run arbitrary SQL queries", - ), - Setting( - "max_signed_tokens_ttl", - 0, - "Maximum allowed expiry time for signed API tokens", - ), - Setting("suggest_facets", True, "Calculate and display suggested facets"), - Setting( - "default_cache_ttl", - 5, - "Default HTTP cache TTL (used in Cache-Control: max-age= header)", - ), - Setting("cache_size_kb", 0, "SQLite cache size in KB (0 == use SQLite default)"), - Setting( - "allow_csv_stream", - True, - "Allow .csv?_stream=1 to download all rows (ignoring max_returned_rows)", - ), - Setting( - "max_csv_mb", - 100, - "Maximum size allowed for CSV export in MB - set 0 to disable this limit", - ), - Setting( - "truncate_cells_html", - 2048, - "Truncate cells longer than this in HTML table view - set 0 to disable", - ), - Setting( - "force_https_urls", - False, - "Force URLs in API output to always use https:// protocol", - ), - Setting( - "template_debug", - False, - "Allow display of template debug information with ?_context=1", - ), - Setting( - "trace_debug", - False, - "Allow display of SQL trace debug information with ?_trace=1", - ), - Setting("base_url", "/", "Datasette URLs should use this base path"), -) -_HASH_URLS_REMOVED = "The hash_urls setting has been removed, try the datasette-hashed-urls plugin instead" -OBSOLETE_SETTINGS = { - "hash_urls": _HASH_URLS_REMOVED, - "default_cache_ttl_hashed": _HASH_URLS_REMOVED, -} -DEFAULT_SETTINGS = {option.name: option.default for option in SETTINGS} + def table_metadata(self, database, table): + "Fetch table-specific metadata." + return self.ds.metadata.get( + 'databases', {} + ).get(database, {}).get('tables', {}).get(table, {}) -FAVICON_PATH = app_root / "datasette" / "static" / "favicon.png" + def options(self, request, *args, **kwargs): + r = response.text('ok') + if self.ds.cors: + r.headers['Access-Control-Allow-Origin'] = '*' + return r -DEFAULT_NOT_SET = object() + def redirect(self, request, path, forward_querystring=True): + if request.query_string and '?' not in path and forward_querystring: + path = '{}?{}'.format( + path, request.query_string + ) + r = response.redirect(path) + r.headers['Link'] = '<{}>; rel=preload'.format(path) + if self.ds.cors: + r.headers['Access-Control-Allow-Origin'] = '*' + return r + def resolve_db_name(self, db_name, **kwargs): + databases = self.ds.inspect() + hash = None + name = None + if '-' in db_name: + # Might be name-and-hash, or might just be + # a name with a hyphen in it + name, hash = db_name.rsplit('-', 1) + if name not in databases: + # Try the whole name + name = db_name + hash = None + else: + name = db_name + # Verify the hash + try: + info = databases[name] + except KeyError: + raise NotFound('Database not found: {}'.format(name)) + expected = info['hash'][:HASH_LENGTH] + if expected != hash: + should_redirect = '/{}-{}'.format( + name, expected, + ) + if 'table' in kwargs: + should_redirect += '/' + kwargs['table'] + if 'pk_path' in kwargs: + should_redirect += '/' + kwargs['pk_path'] + if 'as_json' in kwargs: + should_redirect += kwargs['as_json'] + if 'as_db' in kwargs: + should_redirect += kwargs['as_db'] + return name, expected, should_redirect + return name, expected, None -ResourcesSQL = collections.namedtuple("ResourcesSQL", ("sql", "params")) + async def execute(self, db_name, sql, params=None, truncate=False, custom_time_limit=None): + """Executes sql against db_name in a thread""" + def sql_operation_in_thread(): + conn = getattr(connections, db_name, None) + if not conn: + info = self.ds.inspect()[db_name] + conn = sqlite3.connect( + 'file:{}?immutable=1'.format(info['file']), + uri=True, + check_same_thread=False, + ) + self.ds.prepare_connection(conn) + setattr(connections, db_name, conn) + time_limit_ms = self.ds.sql_time_limit_ms + if custom_time_limit and custom_time_limit < self.ds.sql_time_limit_ms: + time_limit_ms = custom_time_limit -def _permission_cache_key(actor, action, parent, child): - # Key on the full serialized actor so actors differing in any field - # (e.g. token restrictions) never share cache entries - actor_key = ( - json.dumps(actor, sort_keys=True, default=repr) if actor is not None else None - ) - return (actor_key, action, parent, child) + with sqlite_timelimit(conn, time_limit_ms): + try: + cursor = conn.cursor() + cursor.execute(sql, params or {}) + if self.max_returned_rows and truncate: + rows = cursor.fetchmany(self.max_returned_rows + 1) + truncated = len(rows) > self.max_returned_rows + rows = rows[:self.max_returned_rows] + else: + rows = cursor.fetchall() + truncated = False + except Exception as e: + print('ERROR: conn={}, sql = {}, params = {}: {}'.format( + conn, repr(sql), params, e + )) + raise + if truncate: + return rows, truncated, cursor.description + else: + return rows + return await asyncio.get_event_loop().run_in_executor( + self.executor, sql_operation_in_thread + ) -async def favicon(request, send): - await asgi_send_file( - send, - str(FAVICON_PATH), - content_type="image/png", - headers={"Cache-Control": "max-age=3600, public"}, - ) + def get_templates(self, database, table=None): + assert NotImplemented + async def get(self, request, db_name, **kwargs): + name, hash, should_redirect = self.resolve_db_name(db_name, **kwargs) + if should_redirect: + return self.redirect(request, should_redirect) + return await self.view_get(request, name, hash, **kwargs) -ResolvedTable = collections.namedtuple("ResolvedTable", ("db", "table", "is_view")) -ResolvedRow = collections.namedtuple( - "ResolvedRow", ("db", "table", "sql", "params", "pks", "pk_values", "row") -) + async def view_get(self, request, name, hash, **kwargs): + try: + as_json = kwargs.pop('as_json') + except KeyError: + as_json = False + extra_template_data = {} + start = time.time() + status_code = 200 + templates = [] + try: + response_or_template_contexts = await self.data( + request, name, hash, **kwargs + ) + if isinstance(response_or_template_contexts, response.HTTPResponse): + return response_or_template_contexts + else: + data, extra_template_data, templates = response_or_template_contexts + except (sqlite3.OperationalError, InvalidSql, DatasetteError) as e: + raise DatasetteError(str(e), title='Invalid SQL', status=400) + except (sqlite3.OperationalError) as e: + raise DatasetteError(str(e)) + except DatasetteError: + raise + end = time.time() + data['query_ms'] = (end - start) * 1000 + for key in ('source', 'source_url', 'license', 'license_url'): + value = self.ds.metadata.get(key) + if value: + data[key] = value + if as_json: + # Special case for .jsono extension - redirect to _shape=objects + if as_json == '.jsono': + return self.redirect( + request, + path_with_added_args( + request, + {'_shape': 'objects'}, + path=request.path.rsplit('.jsono', 1)[0] + '.json' + ), + forward_querystring=False + ) + # Deal with the _shape option + shape = request.args.get('_shape', 'lists') + if shape in ('objects', 'object'): + columns = data.get('columns') + rows = data.get('rows') + if rows and columns: + data['rows'] = [ + dict(zip(columns, row)) + for row in rows + ] + if shape == 'object': + error = None + if 'primary_keys' not in data: + error = '_shape=object is only available on tables' + else: + pks = data['primary_keys'] + if not pks: + error = '_shape=object not available for tables with no primary keys' + else: + object_rows = {} + for row in data['rows']: + pk_string = path_from_row_pks(row, pks, not pks) + object_rows[pk_string] = row + data['rows'] = object_rows + if error: + data = { + 'ok': False, + 'error': error, + 'database': name, + 'database_hash': hash, + } + headers = {} + if self.ds.cors: + headers['Access-Control-Allow-Origin'] = '*' + r = response.HTTPResponse( + json.dumps( + data, cls=CustomJSONEncoder + ), + status=status_code, + content_type='application/json', + headers=headers, + ) + else: + extras = {} + if callable(extra_template_data): + extras = extra_template_data() + if asyncio.iscoroutine(extras): + extras = await extras + else: + extras = extra_template_data + context = { + **data, + **extras, + **{ + 'url_json': path_with_ext(request, '.json'), + 'url_jsono': path_with_ext(request, '.jsono'), + 'extra_css_urls': self.ds.extra_css_urls(), + 'extra_js_urls': self.ds.extra_js_urls(), + 'datasette_version': __version__, + } + } + if 'metadata' not in context: + context['metadata'] = self.ds.metadata + r = self.render( + templates, + **context, + ) + r.status = status_code + # Set far-future cache expiry + if self.ds.cache_headers: + r.headers['Cache-Control'] = 'max-age={}'.format( + 365 * 24 * 60 * 60 + ) + return r -def _to_string(value): - if isinstance(value, str): - return value - else: - return json.dumps(value, default=str) - - -def _template_context_json_default(value): - if dataclasses.is_dataclass(value) and not isinstance(value, type): - return { - field.name: getattr(value, field.name) - for field in dataclasses.fields(value) + async def custom_sql(self, request, name, hash, sql, editable=True, canned_query=None): + params = request.raw_args + if 'sql' in params: + params.pop('sql') + if '_shape' in params: + params.pop('_shape') + # Extract any :named parameters + named_parameters = self.re_named_parameter.findall(sql) + named_parameter_values = { + named_parameter: params.get(named_parameter) or '' + for named_parameter in named_parameters } - return repr(value) + + # Set to blank string if missing from params + for named_parameter in named_parameters: + if named_parameter not in params: + params[named_parameter] = '' + + extra_args = {} + if params.get('_sql_time_limit_ms'): + extra_args['custom_time_limit'] = int(params['_sql_time_limit_ms']) + rows, truncated, description = await self.execute( + name, sql, params, truncate=True, **extra_args + ) + columns = [r[0] for r in description] + + templates = ['query-{}.html'.format(to_css_class(name)), 'query.html'] + if canned_query: + templates.insert(0, 'query-{}-{}.html'.format( + to_css_class(name), to_css_class(canned_query) + )) + + return { + 'database': name, + 'rows': rows, + 'truncated': truncated, + 'columns': columns, + 'query': { + 'sql': sql, + 'params': params, + } + }, { + 'database_hash': hash, + 'custom_sql': True, + 'named_parameter_values': named_parameter_values, + 'editable': editable, + 'canned_query': canned_query, + }, templates -@pass_context -def _legacy_template_csrftoken(context): - request = context.get("request") - if request and "csrftoken" in request.scope: - return request.scope["csrftoken"]() - return "" +class IndexView(RenderMixin): + def __init__(self, datasette): + self.ds = datasette + self.files = datasette.files + self.jinja_env = datasette.jinja_env + self.executor = datasette.executor + + async def get(self, request, as_json): + databases = [] + for key, info in sorted(self.ds.inspect().items()): + tables = [t for t in info['tables'].values() if not t['hidden']] + hidden_tables = [t for t in info['tables'].values() if t['hidden']] + database = { + 'name': key, + 'hash': info['hash'], + 'path': '{}-{}'.format(key, info['hash'][:HASH_LENGTH]), + 'tables_truncated': sorted( + tables, + key=lambda t: t['count'], + reverse=True + )[:5], + 'tables_count': len(tables), + 'tables_more': len(tables) > 5, + 'table_rows_sum': sum(t['count'] for t in tables), + 'hidden_table_rows_sum': sum(t['count'] for t in hidden_tables), + 'hidden_tables_count': len(hidden_tables), + 'views_count': len(info['views']), + } + databases.append(database) + if as_json: + return response.HTTPResponse( + json.dumps( + {db['name']: db for db in databases}, + cls=CustomJSONEncoder + ), + content_type='application/json', + headers={ + 'Access-Control-Allow-Origin': '*' + } + ) + else: + return self.render( + ['index.html'], + databases=databases, + metadata=self.ds.metadata, + datasette_version=__version__, + extra_css_urls=self.ds.extra_css_urls(), + extra_js_urls=self.ds.extra_js_urls(), + ) -def _resolve_static_asset_path(root_path, path): - root = Path(root_path).resolve() - full_path = (root / path).resolve() - try: - full_path.relative_to(root) - except ValueError: - raise ValueError("Static asset path cannot escape static root") from None - return full_path +async def favicon(request): + return response.text('') -# Documentation for the variables Datasette.render_template() adds to the -# context for every page. This is part of the documented template contract: -# keys added in render_template() must be documented here - the contract -# tests in tests/test_template_context.py enforce this, and the docs in -# docs/template_context.rst are generated from it. -TEMPLATE_BASE_CONTEXT = { - "request": "The current :ref:`Request object `, or None. Common properties include ``request.path``, ``request.args``, ``request.actor``, ``request.url_vars`` and ``request.host``.", - "crumb_items": 'Async function returning breadcrumb navigation items for the current page. Call it with ``request=request`` plus optional ``database=`` and ``table=`` arguments; it returns a list of ``{"href": url, "label": label}`` dictionaries.', - "urls": "Object with methods for constructing URLs within Datasette. Common methods include ``urls.instance()``, ``urls.database(database)``, ``urls.table(database, table)``, ``urls.query(database, query)``, ``urls.row(database, table, row_path)`` and ``urls.static(path)`` - see :ref:`internals_datasette_urls`.", - "actor": "The currently authenticated actor dictionary, or None. Actors usually include an ``id`` key and may include any other keys supplied by authentication plugins.", - "menu_links": "Async function returning links for the Datasette application menu, including links added by plugins. Each item is a link dictionary with ``href`` and ``label`` keys. See :ref:`plugin_hook_menu_links`; for page action menus that can also include JavaScript-backed buttons, see :ref:`plugin_actions`.", - "display_actor": "Function that accepts an actor dictionary and returns the display string used in the navigation menu.", - "show_logout": "True if the logout link should be shown in the navigation menu", - "zip": "Python's ``zip()`` builtin, made available to template logic", - "body_scripts": 'List of JavaScript snippets contributed by plugins using :ref:`plugin_hook_extra_body_script`. Each item is a dictionary with ``script`` containing JavaScript source and ``module`` indicating whether Datasette will wrap it in `` diff --git a/datasette/templates/_codemirror.html b/datasette/templates/_codemirror.html index 75c16168..237d6907 100644 --- a/datasette/templates/_codemirror.html +++ b/datasette/templates/_codemirror.html @@ -1,16 +1,7 @@ - - + + + diff --git a/datasette/templates/_codemirror_foot.html b/datasette/templates/_codemirror_foot.html index a624c8a4..1e07fc72 100644 --- a/datasette/templates/_codemirror_foot.html +++ b/datasette/templates/_codemirror_foot.html @@ -1,42 +1,13 @@ diff --git a/datasette/templates/_crumbs.html b/datasette/templates/_crumbs.html deleted file mode 100644 index bd1ff0da..00000000 --- a/datasette/templates/_crumbs.html +++ /dev/null @@ -1,15 +0,0 @@ -{% macro nav(request, database=None, table=None) -%} -{% if crumb_items is defined %} - {% set items=crumb_items(request=request, database=database, table=table) %} - {% if items %} -

- {% for item in items %} - {{ item.label }} - {% if not loop.last %} - / - {% endif %} - {% endfor %} -

- {% endif %} -{% endif %} -{%- endmacro %} diff --git a/datasette/templates/_debug_common_functions.html b/datasette/templates/_debug_common_functions.html deleted file mode 100644 index d988a2f3..00000000 --- a/datasette/templates/_debug_common_functions.html +++ /dev/null @@ -1,50 +0,0 @@ - diff --git a/datasette/templates/_description_source_license.html b/datasette/templates/_description_source_license.html index f852268f..eba4eb1a 100644 --- a/datasette/templates/_description_source_license.html +++ b/datasette/templates/_description_source_license.html @@ -1,6 +1,6 @@ -{% if metadata.get("description_html") or metadata.get("description") %} +{% if metadata.description_html or metadata.description %}