diff --git a/.dockerignore b/.dockerignore index 5078bf47..490f509e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,5 +9,3 @@ build dist scratchpad venv -*.db -*.sqlite diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs deleted file mode 100644 index 84e574fd..00000000 --- a/.git-blame-ignore-revs +++ /dev/null @@ -1,4 +0,0 @@ -# Applying Black -35d6ee2790e41e96f243c1ff58be0c9c0519a8ce -368638555160fb9ac78f462d0f79b1394163fa30 -2b344f6a34d2adaa305996a1a580ece06397f6e4 diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index f0bcdbe0..00000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1 +0,0 @@ -github: [simonw] diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 88bb03b1..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: 2 -updates: -- package-ecosystem: pip - directory: "/" - schedule: - interval: daily - time: "13:00" - groups: - python-packages: - patterns: - - "*" diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index b0640ae8..2de0a8b6 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -1,129 +1,61 @@ name: Deploy latest.datasette.io on: - workflow_dispatch: push: branches: - - main - # - 1.0-dev - -permissions: - contents: read + - main jobs: deploy: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v6 + uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v2 with: - python-version: "3.13" - cache: pip + python-version: 3.9 + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- - name: Install Python dependencies run: | python -m pip install --upgrade pip - python -m pip install . --group dev + python -m pip install -e .[test] + python -m pip install -e .[docs] python -m pip install sphinx-to-sqlite==0.1a1 - name: Run tests - if: ${{ github.ref == 'refs/heads/main' }} - run: | - pytest -n auto -m "not serial" - pytest -m "serial" - - name: Build fixtures.db and other files needed to deploy the demo - run: |- - python tests/fixtures.py \ - fixtures.db \ - fixtures-config.json \ - fixtures-metadata.json \ - plugins \ - --extra-db-filename extra_database.db + run: pytest + - name: Build fixtures.db + run: python tests/fixtures.py fixtures.db fixtures.json plugins - name: Build docs.db - if: ${{ github.ref == 'refs/heads/main' }} run: |- cd docs - DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build + sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - - name: Set up the alternate-route demo - run: | - echo ' - from datasette import hookimpl - - @hookimpl - def startup(datasette): - db = datasette.get_database("fixtures2") - db.route = "alternative-route" - ' > plugins/alternative_route.py - cp fixtures.db fixtures2.db - - name: And the counters writable stored query demo - run: | - cat > plugins/counters.py < metadata.json - # cat metadata.json - - id: auth - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v3 + - name: Set up Cloud Run + uses: google-github-actions/setup-gcloud@master with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v3 + version: '275.0.0' + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} - name: Deploy to Cloud Run - env: - LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }} run: |- gcloud config set run/region us-central1 gcloud config set project datasette-222320 - export SUFFIX="-${GITHUB_REF#refs/heads/}" - export SUFFIX=${SUFFIX#-main} - # Replace 1.0 with one-dot-zero in SUFFIX - export SUFFIX=${SUFFIX//1.0/one-dot-zero} - datasette publish cloudrun fixtures.db fixtures2.db extra_database.db \ - -m fixtures-metadata.json \ + datasette publish cloudrun fixtures.db \ + -m fixtures.json \ --plugins-dir=plugins \ --branch=$GITHUB_SHA \ --version-note=$GITHUB_SHA \ - --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb --root" \ - --install 'datasette-ephemeral-tables>=0.2.2' \ - --service "datasette-latest$SUFFIX" \ - --secret $LATEST_DATASETTE_SECRET - - name: Deploy to docs as well (only for main) - if: ${{ github.ref == 'refs/heads/main' }} - run: |- + --extra-options="--setting template_debug 1" \ + --install=pysqlite3-binary \ + --service=datasette-latest # Deploy docs.db to a different service datasette publish cloudrun docs.db \ --branch=$GITHUB_SHA \ diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml deleted file mode 100644 index b8fb8aaa..00000000 --- a/.github/workflows/documentation-links.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Read the Docs Pull Request Preview -on: - pull_request: - types: - - opened - -permissions: - pull-requests: write - -jobs: - documentation-links: - runs-on: ubuntu-latest - steps: - - uses: readthedocs/actions/preview@v1 - with: - project-slug: "datasette" diff --git a/.github/workflows/mirror-master-and-main.yml b/.github/workflows/mirror-master-and-main.yml new file mode 100644 index 00000000..8418df40 --- /dev/null +++ b/.github/workflows/mirror-master-and-main.yml @@ -0,0 +1,21 @@ +name: Mirror "master" and "main" branches +on: + push: + branches: + - master + - main + +jobs: + mirror: + runs-on: ubuntu-latest + steps: + - name: Mirror to "master" + uses: zofrex/mirror-branch@ea152f124954fa4eb26eea3fe0dbe313a3a08d94 + with: + target-branch: master + force: false + - name: Mirror to "main" + uses: zofrex/mirror-branch@ea152f124954fa4eb26eea3fe0dbe313a3a08d94 + with: + target-branch: main + force: false diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml deleted file mode 100644 index 5275ddef..00000000 --- a/.github/workflows/playwright.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Playwright - -on: - push: - pull_request: - workflow_dispatch: - -permissions: - contents: read - -jobs: - test: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - browser: [chromium, firefox, webkit] - steps: - - uses: actions/checkout@v6 - - name: Set up Python 3.14 - uses: actions/setup-python@v6 - with: - python-version: "3.14" - allow-prereleases: true - cache: pip - cache-dependency-path: pyproject.toml - - name: Cache uv - uses: actions/cache@v5 - with: - path: ~/.cache/uv - key: ${{ runner.os }}-py3.14-uv-${{ hashFiles('pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-py3.14-uv- - - name: Cache Playwright browsers - uses: actions/cache@v5 - with: - path: ~/.cache/ms-playwright/ - key: ${{ runner.os }}-playwright-${{ matrix.browser }}-${{ hashFiles('pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-playwright-${{ matrix.browser }}- - - name: Install uv - run: python -m pip install uv - - name: Install dependencies - run: uv sync --group dev --group playwright - - name: Install ${{ matrix.browser }} - run: uv run --group dev --group playwright playwright install --with-deps ${{ matrix.browser }} - - name: Run Playwright tests - run: uv run --group dev --group playwright pytest tests/test_playwright.py --playwright --browser ${{ matrix.browser }} diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index 735e14e9..d846cca7 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -2,16 +2,13 @@ name: Check JavaScript for conformance with Prettier on: [push] -permissions: - contents: read - jobs: prettier: runs-on: ubuntu-latest steps: - name: Check out repo - uses: actions/checkout@v6 - - uses: actions/cache@v5 + uses: actions/checkout@v2 + - uses: actions/cache@v2 name: Configure npm caching with: path: ~/.npm @@ -22,4 +19,4 @@ jobs: run: npm ci - name: Run prettier run: |- - npm run prettier -- --check + npx --no-install prettier --check 'datasette/static/*[!.min].js' diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 87300593..c1909bbe 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,106 +4,70 @@ on: release: types: [created] -permissions: - contents: read - jobs: test: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - cache: pip - cache-dependency-path: pyproject.toml + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- - name: Install dependencies run: | - pip install . --group dev + pip install -e '.[test]' - name: Run tests run: | pytest - deploy: runs-on: ubuntu-latest needs: [test] - environment: release - permissions: - id-token: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v2 with: - python-version: '3.13' - cache: pip - cache-dependency-path: pyproject.toml + python-version: '3.9' + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-publish-pip- - name: Install dependencies run: | - pip install setuptools wheel build - - name: Build - run: | - python -m build + pip install setuptools wheel twine - name: Publish - uses: pypa/gh-action-pypi-publish@release/v1 - - deploy_static_docs: - runs-on: ubuntu-latest - needs: [deploy] - if: "!github.event.release.prerelease" - steps: - - uses: actions/checkout@v6 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.10' - cache: pip - cache-dependency-path: pyproject.toml - - name: Install dependencies + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} run: | - python -m pip install . --group dev - python -m pip install sphinx-to-sqlite==0.1a1 - - name: Build docs.db - run: |- - cd docs - DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build - sphinx-to-sqlite ../docs.db _build - cd .. - - id: auth - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v3 - - name: Deploy stable-docs.datasette.io to Cloud Run - run: |- - gcloud config set run/region us-central1 - gcloud config set project datasette-222320 - datasette publish cloudrun docs.db \ - --service=datasette-docs-stable - + python setup.py sdist bdist_wheel + twine upload dist/* deploy_docker: runs-on: ubuntu-latest needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v2 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} DOCKER_PASS: ${{ secrets.DOCKER_PASS }} run: |- - sleep 60 # Give PyPI time to make the new release available - docker login -u $DOCKER_USER -p $DOCKER_PASS - export REPO=datasetteproject/datasette - docker build -f Dockerfile \ - -t $REPO:${GITHUB_REF#refs/tags/} \ - --build-arg VERSION=${GITHUB_REF#refs/tags/} . - docker tag $REPO:${GITHUB_REF#refs/tags/} $REPO:latest - docker push $REPO:${GITHUB_REF#refs/tags/} - docker push $REPO:latest + docker login -u $DOCKER_USER -p $DOCKER_PASS + export REPO=datasetteproject/datasette + docker build -f Dockerfile -t $REPO:${GITHUB_REF#refs/tags/} . + docker tag $REPO:${GITHUB_REF#refs/tags/} $REPO:latest + docker push $REPO diff --git a/.github/workflows/push_docker_tag.yml b/.github/workflows/push_docker_tag.yml deleted file mode 100644 index e622ef4c..00000000 --- a/.github/workflows/push_docker_tag.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Push specific Docker tag - -on: - workflow_dispatch: - inputs: - version_tag: - description: Tag to build and push - -permissions: - contents: read - -jobs: - deploy_docker: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - name: Build and push to Docker Hub - env: - DOCKER_USER: ${{ secrets.DOCKER_USER }} - DOCKER_PASS: ${{ secrets.DOCKER_PASS }} - VERSION_TAG: ${{ github.event.inputs.version_tag }} - run: |- - docker login -u $DOCKER_USER -p $DOCKER_PASS - export REPO=datasetteproject/datasette - docker build -f Dockerfile \ - -t $REPO:${VERSION_TAG} \ - --build-arg VERSION=${VERSION_TAG} . - docker push $REPO:${VERSION_TAG} diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml deleted file mode 100644 index 9a808194..00000000 --- a/.github/workflows/spellcheck.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Check spelling in documentation - -on: [push, pull_request] - -permissions: - contents: read - -jobs: - spellcheck: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.11' - cache: 'pip' - cache-dependency-path: '**/pyproject.toml' - - name: Install dependencies - run: | - pip install . --group dev - - name: Check spelling - run: | - codespell README.md --ignore-words docs/codespell-ignore-words.txt - codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt - codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt - codespell tests --ignore-words docs/codespell-ignore-words.txt diff --git a/.github/workflows/stable-docs.yml b/.github/workflows/stable-docs.yml deleted file mode 100644 index 59b5fbc0..00000000 --- a/.github/workflows/stable-docs.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: Update Stable Docs - -on: - release: - types: [published] - push: - branches: - - main - -permissions: - contents: write - -jobs: - update_stable_docs: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - fetch-depth: 0 # We need all commits to find docs/ changes - - name: Set up Git user - run: | - git config user.name "Automated" - git config user.email "actions@users.noreply.github.com" - - name: Create stable branch if it does not yet exist - run: | - if ! git ls-remote --heads origin stable | grep -qE '\bstable\b'; then - # Make sure we have all tags locally - git fetch --tags --quiet - - # Latest tag that is just numbers and dots (optionally prefixed with 'v') - # e.g., 0.65.2 or v0.65.2 — excludes 1.0a20, 1.0-rc1, etc. - LATEST_RELEASE=$( - git tag -l --sort=-v:refname \ - | grep -E '^v?[0-9]+(\.[0-9]+){1,3}$' \ - | head -n1 - ) - - git checkout -b stable - - # If there are any stable releases, copy docs/ from the most recent - if [ -n "$LATEST_RELEASE" ]; then - rm -rf docs/ - git checkout "$LATEST_RELEASE" -- docs/ || true - fi - - git commit -m "Populate docs/ from $LATEST_RELEASE" || echo "No changes" - git push -u origin stable - fi - - name: Handle Release - if: github.event_name == 'release' && !github.event.release.prerelease - run: | - git fetch --all - git checkout stable - git reset --hard ${GITHUB_REF#refs/tags/} - git push origin stable --force - - name: Handle Commit to Main - if: contains(github.event.head_commit.message, '!stable-docs') - run: | - git fetch origin - git checkout -b stable origin/stable - # Get the list of modified files in docs/ from the current commit - FILES=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} -- docs/) - # Check if the list of files is non-empty - if [[ -n "$FILES" ]]; then - # Checkout those files to the stable branch to over-write with their contents - for FILE in $FILES; do - git checkout ${{ github.sha }} -- $FILE - done - git add docs/ - git commit -m "Doc changes from ${{ github.sha }}" - git push origin stable - else - echo "No changes to docs/ in this commit." - exit 0 - fi diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index c514048e..1d1cf332 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -7,31 +7,33 @@ on: pull_request: branches: - main -permissions: - contents: read - jobs: test: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v6 + uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v2 with: - python-version: '3.12' - cache: 'pip' - cache-dependency-path: '**/pyproject.toml' + python-version: 3.9 + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- - name: Install Python dependencies run: | python -m pip install --upgrade pip - python -m pip install . --group dev + python -m pip install -e .[test] python -m pip install pytest-cov - name: Run tests run: |- ls -lah cat .coveragerc - pytest -m "not serial" --cov=datasette --cov-config=.coveragerc --cov-report xml:coverage.xml --cov-report term -x + pytest --cov=datasette --cov-config=.coveragerc --cov-report xml:coverage.xml --cov-report term ls -lah - name: Upload coverage report uses: codecov/codecov-action@v1 diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml deleted file mode 100644 index 5162c47a..00000000 --- a/.github/workflows/test-pyodide.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Test in Pyodide with shot-scraper - -on: - push: - pull_request: - workflow_dispatch: - -permissions: - contents: read - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - name: Set up Python 3.10 - uses: actions/setup-python@v6 - with: - python-version: "3.10" - cache: 'pip' - cache-dependency-path: '**/pyproject.toml' - - name: Cache Playwright browsers - uses: actions/cache@v5 - with: - path: ~/.cache/ms-playwright/ - key: ${{ runner.os }}-browsers - - name: Install Playwright dependencies - run: | - pip install shot-scraper build - shot-scraper install - - name: Run test - run: | - ./test-in-pyodide-with-shot-scraper.sh diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml deleted file mode 100644 index 23fce459..00000000 --- a/.github/workflows/test-sqlite-support.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Test SQLite versions - -on: [push, pull_request] - -permissions: - contents: read - -jobs: - test: - runs-on: ${{ matrix.platform }} - continue-on-error: true - strategy: - matrix: - platform: [ubuntu-latest] - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] - sqlite-version: [ - #"3", # latest version - "3.46", - #"3.45", - #"3.27", - #"3.26", - "3.25", - #"3.25.3", # 2018-09-25, window functions breaks test_upsert for some reason on 3.10, skip for now - #"3.24", # 2018-06-04, added UPSERT support - #"3.23.1" # 2018-04-10, before UPSERT - ] - steps: - - uses: actions/checkout@v6 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - allow-prereleases: true - cache: pip - cache-dependency-path: pyproject.toml - - name: Set up SQLite ${{ matrix.sqlite-version }} - uses: asg017/sqlite-versions@71ea0de37ae739c33e447af91ba71dda8fcf22e6 - with: - version: ${{ matrix.sqlite-version }} - cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1" - - run: python3 -c "import sqlite3; print(sqlite3.sqlite_version)" - - run: echo $LD_LIBRARY_PATH - - name: Build extension for --load-extension test - run: |- - (cd tests && gcc ext.c -fPIC -shared -o ext.so) - - name: Install dependencies - run: | - pip install . --group dev - pip freeze - - name: Run tests - run: | - pytest -n auto -m "not serial" - pytest -m "serial" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9e47db6f..a1774213 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,53 +1,29 @@ name: Test -on: [push, pull_request] - -permissions: - contents: read +on: [push] jobs: test: runs-on: ubuntu-latest strategy: - fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - allow-prereleases: true - cache: pip - cache-dependency-path: pyproject.toml - - name: Build extension for --load-extension test - run: |- - (cd tests && gcc ext.c -fPIC -shared -o ext.so) + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- - name: Install dependencies run: | - pip install . --group dev - pip freeze + pip install -e '.[test]' - name: Run tests run: | - pytest -n auto -m "not serial" - pytest -m "serial" - # And the test that exceeds a localhost HTTPS server - tests/test_datasette_https_server.sh - - name: Black - run: | - black --version - black --check . - - name: Ruff - run: ruff check datasette tests - - name: Check if cog needs to be run - run: | - cog --check docs/*.rst - - name: Check if blacken-docs needs to be run - run: | - # This fails on syntax errors, or a diff was applied - blacken-docs -l 60 docs/*.rst - - name: Test DATASETTE_LOAD_PLUGINS - run: | - pip install datasette-init datasette-json-html - tests/test-datasette-load-plugins.sh + pytest diff --git a/.github/workflows/tmate-mac.yml b/.github/workflows/tmate-mac.yml index a033cd92..46be117e 100644 --- a/.github/workflows/tmate-mac.yml +++ b/.github/workflows/tmate-mac.yml @@ -3,13 +3,10 @@ name: tmate session mac on: workflow_dispatch: -permissions: - contents: read - jobs: build: runs-on: macos-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v2 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/tmate.yml b/.github/workflows/tmate.yml index 72af1eec..02e7bd33 100644 --- a/.github/workflows/tmate.yml +++ b/.github/workflows/tmate.yml @@ -3,16 +3,10 @@ name: tmate session on: workflow_dispatch: -permissions: - contents: read - models: read - jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v2 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 8c058692..29ac176f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,18 +1,10 @@ build-metadata.json datasets.json -.playwright-mcp - scratchpad .vscode -uv.lock -data.db - -# test databases -*.db - # We don't use Pipfile, so ignore them Pipfile Pipfile.lock @@ -125,12 +117,3 @@ ENV/ # macOS files .DS_Store node_modules -.*.swp - -# In case someone compiled tests/ext.c for test_load_extensions, don't -# include it in source control. -tests/*.dylib -tests/*.so -tests/*.dll - -.idea diff --git a/.readthedocs.yaml b/.readthedocs.yaml deleted file mode 100644 index 8b3e54aa..00000000 --- a/.readthedocs.yaml +++ /dev/null @@ -1,17 +0,0 @@ -version: 2 - -sphinx: - configuration: docs/conf.py - -build: - os: ubuntu-24.04 - tools: - python: "3.13" - jobs: - install: - - pip install --upgrade pip - - pip install . --group dev - -formats: -- pdf -- epub diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md deleted file mode 100644 index 14d4c567..00000000 --- a/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,128 +0,0 @@ -# Contributor Covenant Code of Conduct - -## Our Pledge - -We as members, contributors, and leaders pledge to make participation in our -community a harassment-free experience for everyone, regardless of age, body -size, visible or invisible disability, ethnicity, sex characteristics, gender -identity and expression, level of experience, education, socio-economic status, -nationality, personal appearance, race, religion, or sexual identity -and orientation. - -We pledge to act and interact in ways that contribute to an open, welcoming, -diverse, inclusive, and healthy community. - -## Our Standards - -Examples of behavior that contributes to a positive environment for our -community include: - -* Demonstrating empathy and kindness toward other people -* Being respectful of differing opinions, viewpoints, and experiences -* Giving and gracefully accepting constructive feedback -* Accepting responsibility and apologizing to those affected by our mistakes, - and learning from the experience -* Focusing on what is best not just for us as individuals, but for the - overall community - -Examples of unacceptable behavior include: - -* The use of sexualized language or imagery, and sexual attention or - advances of any kind -* Trolling, insulting or derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or email - address, without their explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Enforcement Responsibilities - -Community leaders are responsible for clarifying and enforcing our standards of -acceptable behavior and will take appropriate and fair corrective action in -response to any behavior that they deem inappropriate, threatening, offensive, -or harmful. - -Community leaders have the right and responsibility to remove, edit, or reject -comments, commits, code, wiki edits, issues, and other contributions that are -not aligned to this Code of Conduct, and will communicate reasons for moderation -decisions when appropriate. - -## Scope - -This Code of Conduct applies within all community spaces, and also applies when -an individual is officially representing the community in public spaces. -Examples of representing our community include using an official e-mail address, -posting via an official social media account, or acting as an appointed -representative at an online or offline event. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported to the community leaders responsible for enforcement at -`swillison+datasette-code-of-conduct@gmail.com`. -All complaints will be reviewed and investigated promptly and fairly. - -All community leaders are obligated to respect the privacy and security of the -reporter of any incident. - -## Enforcement Guidelines - -Community leaders will follow these Community Impact Guidelines in determining -the consequences for any action they deem in violation of this Code of Conduct: - -### 1. Correction - -**Community Impact**: Use of inappropriate language or other behavior deemed -unprofessional or unwelcome in the community. - -**Consequence**: A private, written warning from community leaders, providing -clarity around the nature of the violation and an explanation of why the -behavior was inappropriate. A public apology may be requested. - -### 2. Warning - -**Community Impact**: A violation through a single incident or series -of actions. - -**Consequence**: A warning with consequences for continued behavior. No -interaction with the people involved, including unsolicited interaction with -those enforcing the Code of Conduct, for a specified period of time. This -includes avoiding interactions in community spaces as well as external channels -like social media. Violating these terms may lead to a temporary or -permanent ban. - -### 3. Temporary Ban - -**Community Impact**: A serious violation of community standards, including -sustained inappropriate behavior. - -**Consequence**: A temporary ban from any sort of interaction or public -communication with the community for a specified period of time. No public or -private interaction with the people involved, including unsolicited interaction -with those enforcing the Code of Conduct, is allowed during this period. -Violating these terms may lead to a permanent ban. - -### 4. Permanent Ban - -**Community Impact**: Demonstrating a pattern of violation of community -standards, including sustained inappropriate behavior, harassment of an -individual, or aggression toward or disparagement of classes of individuals. - -**Consequence**: A permanent ban from any sort of public interaction within -the community. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], -version 2.0, available at -https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. - -Community Impact Guidelines were inspired by [Mozilla's code of conduct -enforcement ladder](https://github.com/mozilla/diversity). - -[homepage]: https://www.contributor-covenant.org - -For answers to common questions about this code of conduct, see the FAQ at -https://www.contributor-covenant.org/faq. Translations are available at -https://www.contributor-covenant.org/translations. diff --git a/Dockerfile b/Dockerfile index 9a8f06cf..f008ff69 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,42 @@ -FROM python:3.11.0-slim-bullseye as build +FROM python:3.7.2-slim-stretch as build -# Version of Datasette to install, e.g. 0.55 -# docker build . -t datasette --build-arg VERSION=0.55 -ARG VERSION +# Setup build dependencies +RUN apt update \ +&& apt install -y python3-dev build-essential wget libxml2-dev libproj-dev libgeos-dev libsqlite3-dev zlib1g-dev pkg-config git \ + && apt clean -RUN apt-get update && \ - apt-get install -y --no-install-recommends libsqlite3-mod-spatialite && \ - apt clean && \ - rm -rf /var/lib/apt && \ - rm -rf /var/lib/dpkg/info/* -RUN pip install https://github.com/simonw/datasette/archive/refs/tags/${VERSION}.zip && \ - find /usr/local/lib -name '__pycache__' | xargs rm -r && \ - rm -rf /root/.cache/pip +RUN wget "https://www.sqlite.org/2020/sqlite-autoconf-3310100.tar.gz" && tar xzf sqlite-autoconf-3310100.tar.gz \ + && cd sqlite-autoconf-3310100 && ./configure --disable-static --enable-fts5 --enable-json1 CFLAGS="-g -O2 -DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_JSON1" \ + && make && make install + +RUN wget "http://www.gaia-gis.it/gaia-sins/freexl-sources/freexl-1.0.5.tar.gz" && tar zxf freexl-1.0.5.tar.gz \ + && cd freexl-1.0.5 && ./configure && make && make install + +RUN wget "http://www.gaia-gis.it/gaia-sins/libspatialite-sources/libspatialite-4.4.0-RC0.tar.gz" && tar zxf libspatialite-4.4.0-RC0.tar.gz \ + && cd libspatialite-4.4.0-RC0 && ./configure && make && make install + +RUN wget "http://www.gaia-gis.it/gaia-sins/readosm-sources/readosm-1.1.0.tar.gz" && tar zxf readosm-1.1.0.tar.gz && cd readosm-1.1.0 && ./configure && make && make install + +RUN wget "http://www.gaia-gis.it/gaia-sins/spatialite-tools-sources/spatialite-tools-4.4.0-RC0.tar.gz" && tar zxf spatialite-tools-4.4.0-RC0.tar.gz \ + && cd spatialite-tools-4.4.0-RC0 && ./configure && make && make install + + +# Add local code to the image instead of fetching from pypi. +COPY . /datasette + +RUN pip install /datasette + +FROM python:3.7.2-slim-stretch + +# Copy python dependencies and spatialite libraries +COPY --from=build /usr/local/lib/ /usr/local/lib/ +# Copy executables +COPY --from=build /usr/local/bin /usr/local/bin +# Copy spatial extensions +COPY --from=build /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu + +ENV LD_LIBRARY_PATH=/usr/local/lib EXPOSE 8001 CMD ["datasette"] diff --git a/Justfile b/Justfile deleted file mode 100644 index 5fcd9afd..00000000 --- a/Justfile +++ /dev/null @@ -1,76 +0,0 @@ -export DATASETTE_SECRET := "not_a_secret" - -# Run tests and linters -@default: test lint - -# Setup project -@init: - uv sync - -# Run pytest with supplied options -@test *options: init - uv run pytest -n auto {{options}} - -# Install Playwright browser support, Chromium by default -@playwright-install browser="chromium": - uv run --group playwright playwright install {{browser}} - -# Install all Playwright browsers used by the test suite -@playwright-install-all: - uv run --group playwright playwright install chromium firefox webkit - -# Run Playwright tests, Chromium by default -@playwright browser="chromium" *options: - uv run --group playwright pytest tests/test_playwright.py --playwright --browser {{browser}} {{options}} - -# Run Playwright tests against all supported browsers -@playwright-all *options: - uv run --group playwright pytest tests/test_playwright.py --playwright --browser chromium --browser firefox --browser webkit {{options}} - -@codespell: - uv run codespell README.md --ignore-words docs/codespell-ignore-words.txt - uv run codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt - uv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt - uv run codespell tests --ignore-words docs/codespell-ignore-words.txt - -# Run linters: black, ruff, cog -@lint: codespell - uv run black datasette tests --check - uv run ruff check datasette tests - uv run cog --check README.md docs/*.rst - -# Apply ruff fixes -@fix: - uv run ruff check --fix datasette tests - -# Rebuild docs with cog -@cog: - uv run cog -r README.md docs/*.rst - -# Serve live docs on localhost:8000 -@docs: cog blacken-docs - uv run make -C docs livehtml - -# Build docs as static HTML -@docs-build: cog blacken-docs - rm -rf docs/_build && cd docs && uv run make html - -# Apply Black -@black: - uv run black datasette tests - -# Apply blacken-docs -@blacken-docs: - uv run blacken-docs -l 60 docs/*.rst - -# Apply prettier -@prettier: - npm run fix - -# Format code with both black and prettier -@format: black prettier blacken-docs - -@serve *options: - uv run sqlite-utils create-database data.db - uv run sqlite-utils create-table data.db docs id integer title text --pk id --ignore - uv run python -m datasette data.db --root --reload {{options}} diff --git a/README.md b/README.md index 393e8e5c..16fc8f0e 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,28 @@ -Datasette +# Datasette [![PyPI](https://img.shields.io/pypi/v/datasette.svg)](https://pypi.org/project/datasette/) -[![Changelog](https://img.shields.io/github/v/release/simonw/datasette?label=changelog)](https://docs.datasette.io/en/latest/changelog.html) +[![Changelog](https://img.shields.io/github/v/release/simonw/datasette?label=changelog)](https://docs.datasette.io/en/stable/changelog.html) [![Python 3.x](https://img.shields.io/pypi/pyversions/datasette.svg?logo=python&logoColor=white)](https://pypi.org/project/datasette/) [![Tests](https://github.com/simonw/datasette/workflows/Test/badge.svg)](https://github.com/simonw/datasette/actions?query=workflow%3ATest) [![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](https://docs.datasette.io/en/latest/?badge=latest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette/blob/main/LICENSE) [![docker: datasette](https://img.shields.io/badge/docker-datasette-blue)](https://hub.docker.com/r/datasetteproject/datasette) -[![discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://datasette.io/discord) *An open source multi-tool for exploring and publishing data* Datasette is a tool for exploring and publishing data. It helps people take data of any shape or size and publish that as an interactive, explorable website and accompanying API. -Datasette is aimed at data journalists, museum curators, archivists, local governments, scientists, researchers and anyone else who has data that they wish to share with the world. +Datasette is aimed at data journalists, museum curators, archivists, local governments and anyone else who has data that they wish to share with the world. -[Explore a demo](https://datasette.io/global-power-plants/global-power-plants), watch [a video about the project](https://simonwillison.net/2021/Feb/7/video/) or try it out [on GitHub Codespaces](https://github.com/datasette/datasette-studio). +[Explore a demo](https://fivethirtyeight.datasettes.com/fivethirtyeight), watch [a video about the project](https://www.youtube.com/watch?v=pTr1uLQTJNE) or try it out by [uploading and publishing your own CSV data](https://simonwillison.net/2019/Apr/23/datasette-glitch/). -* [datasette.io](https://datasette.io/) is the official project website * Latest [Datasette News](https://datasette.io/news) * Comprehensive documentation: https://docs.datasette.io/ * Examples: https://datasette.io/examples -* Live demo of current `main` branch: https://latest.datasette.io/ -* Questions, feedback or want to talk about the project? Join our [Discord](https://datasette.io/discord) +* Live demo of current main: https://latest.datasette.io/ +* Support questions, feedback? Join our [GitHub Discussions forum](https://github.com/simonw/datasette/discussions) -Want to stay up-to-date with the project? Subscribe to the [Datasette newsletter](https://datasette.substack.com/) for tips, tricks and news on what's new in the Datasette ecosystem. +Want to stay up-to-date with the project? Subscribe to the [Datasette Weekly newsletter](https://datasette.substack.com/) for tips, tricks and news on what's new in the Datasette ecosystem. ## Installation @@ -36,7 +34,7 @@ You can also install it using `pip` or `pipx`: pip install datasette -Datasette requires Python 3.8 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker. +Datasette requires Python 3.6 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker. ## Basic usage @@ -48,12 +46,45 @@ This will start a web server on port 8001 - visit http://localhost:8001/ to acce Use Chrome on OS X? You can run datasette against your browser history like so: - datasette ~/Library/Application\ Support/Google/Chrome/Default/History --nolock + datasette ~/Library/Application\ Support/Google/Chrome/Default/History Now visiting http://localhost:8001/History/downloads will show you a web interface to browse your downloads data: ![Downloads table rendered by datasette](https://static.simonwillison.net/static/2017/datasette-downloads.png) +## datasette serve options + + Usage: datasette serve [OPTIONS] [FILES]... + + Serve up specified SQLite database files with a web UI + + Options: + -i, --immutable PATH Database files to open in immutable mode + -h, --host TEXT Host for server. Defaults to 127.0.0.1 which means + only connections from the local machine will be + allowed. Use 0.0.0.0 to listen to all IPs and + allow access from other machines. + -p, --port INTEGER Port for server, defaults to 8001 + --reload Automatically reload if database or code change + detected - useful for development + --cors Enable CORS by serving Access-Control-Allow- + Origin: * + --load-extension PATH Path to a SQLite extension to load + --inspect-file TEXT Path to JSON file created using "datasette + inspect" + -m, --metadata FILENAME Path to JSON file containing license/source + metadata + --template-dir DIRECTORY Path to directory containing custom templates + --plugins-dir DIRECTORY Path to directory containing custom plugins + --static STATIC MOUNT mountpoint:path-to-directory for serving static + files + --memory Make :memory: database available + --config CONFIG Set config option using configname:value + docs.datasette.io/en/stable/config.html + --version-note TEXT Additional note to show on /-/versions + --help-config Show available config options + --help Show this message and exit. + ## metadata.json If you want to include licensing and source information in the generated datasette website you can do so using a JSON file that looks something like this: @@ -85,7 +116,3 @@ Or: This will create a docker image containing both the datasette application and the specified SQLite database files. It will then deploy that image to Heroku or Cloud Run and give you a URL to access the resulting website and API. See [Publishing data](https://docs.datasette.io/en/stable/publish.html) in the documentation for more details. - -## Datasette Lite - -[Datasette Lite](https://lite.datasette.io/) is Datasette packaged using WebAssembly so that it runs entirely in your browser, no Python web application server required. Read more about that in the [Datasette Lite documentation](https://github.com/simonw/datasette-lite/blob/main/README.md). diff --git a/datasette/__init__.py b/datasette/__init__.py index eb18e59e..0e59760a 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -1,9 +1,3 @@ -from datasette.permissions import Permission # noqa from datasette.version import __version_info__, __version__ # noqa -from datasette.events import Event # noqa -from datasette.tokens import TokenHandler, TokenRestrictions # noqa -from datasette.utils.asgi import Forbidden, NotFound, Request, Response # noqa -from datasette.utils import actor_matches_allow # noqa -from datasette.views import Context # noqa from .hookspecs import hookimpl # noqa from .hookspecs import hookspec # noqa diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py deleted file mode 100644 index 103c616d..00000000 --- a/datasette/_pytest_plugin.py +++ /dev/null @@ -1,123 +0,0 @@ -""" -Pytest plugin that automatically closes any Datasette instances constructed -during a pytest test — both in the test body and in function-scoped -fixtures. Instances constructed by session-, module-, class- or package- -scoped fixtures are left alone, because other tests in the session will -still want to use them. - -Registered as a pytest11 entry point in pyproject.toml so that downstream -projects using Datasette get the same FD-safety net for their own tests. - -Opt out by setting ``datasette_autoclose = false`` in pytest.ini (or the -equivalent ini file). -""" - -from __future__ import annotations - -import contextvars -import weakref - -import pytest - -_active_instances: contextvars.ContextVar[list | None] = contextvars.ContextVar( - "datasette_active_instances", default=None -) - -_original_init = None - - -def _install_tracking(): - # datasette.app is imported lazily here rather than at module level: - # as a pytest11 entry point this module is imported during pytest - # startup, before pytest-cov starts measuring, so a module-level - # import would drag in all of datasette and make every import-time - # line in the package invisible to coverage - global _original_init - if _original_init is not None: - return - from datasette.app import Datasette - - _original_init = Datasette.__init__ - - def _tracking_init(self, *args, **kwargs): - _original_init(self, *args, **kwargs) - instances = _active_instances.get() - if instances is not None: - instances.append(weakref.ref(self)) - - Datasette.__init__ = _tracking_init - - -def pytest_configure(config): - if _enabled(config): - _install_tracking() - - -def pytest_addoption(parser): - parser.addini( - "datasette_autoclose", - help=( - "Automatically close Datasette instances created inside test " - "bodies and function-scoped fixtures (default: true)." - ), - default="true", - ) - - -def _enabled(config) -> bool: - value = config.getini("datasette_autoclose") - if isinstance(value, bool): - return value - return str(value).strip().lower() not in ("false", "0", "no", "off") - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_protocol(item, nextitem): - """Track Datasette instances across setup, call and teardown; close at end.""" - if not _enabled(item.config): - yield - return - refs: list[weakref.ref] = [] - token = _active_instances.set(refs) - try: - yield - finally: - _active_instances.reset(token) - for ref in reversed(refs): - ds = ref() - if ds is None: - continue - try: - ds.close() - except Exception as e: - item.warn( - pytest.PytestUnraisableExceptionWarning( - f"Error closing Datasette instance: {e!r}" - ) - ) - - -@pytest.hookimpl(hookwrapper=True) -def pytest_fixture_setup(fixturedef, request): - """Exempt instances created by non-function-scoped fixtures. - - Session-, module-, class- and package-scoped fixtures produce Datasette - instances that must survive beyond the current test — other tests in - the session will still use them. When such a fixture creates one or - more Datasette instances during its setup, we snapshot the tracking - list before the fixture runs and subtract off any instances that were - added during its setup, so they don't get closed at test teardown. - """ - refs = _active_instances.get() - if refs is None: - yield - return - before_ids = {id(ref) for ref in refs} - yield - if fixturedef.scope != "function": - new_refs = [ref for ref in refs if id(ref) not in before_ids] - for new_ref in new_refs: - try: - refs.remove(new_ref) - except ValueError: - pass diff --git a/datasette/actor_auth_cookie.py b/datasette/actor_auth_cookie.py index 368213af..15ecd331 100644 --- a/datasette/actor_auth_cookie.py +++ b/datasette/actor_auth_cookie.py @@ -1,6 +1,6 @@ from datasette import hookimpl from itsdangerous import BadSignature -from datasette.utils import baseconv +import baseconv import time diff --git a/datasette/app.py b/datasette/app.py index 9c9b7de4..cee5ae21 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1,151 +1,80 @@ -from __future__ import annotations - import asyncio -import contextvars -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Sequence - -if TYPE_CHECKING: - from datasette.permissions import Resource - from datasette.tokens import TokenRestrictions +import asgi_csrf import collections -import dataclasses import datetime -import functools import glob +import hashlib import httpx -import importlib.metadata import inspect from itsdangerous import BadSignature import json import os +import pkg_resources import re import secrets import sys import threading -import time -import types +import traceback import urllib.parse from concurrent import futures from pathlib import Path -from markupsafe import Markup, escape +from markupsafe import Markup from itsdangerous import URLSafeSerializer -from jinja2 import ( - ChoiceLoader, - Environment, - FileSystemLoader, - pass_context, - PrefixLoader, -) +import jinja2 +from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader, escape from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound +import uvicorn -from .events import Event -from .column_types import SQLiteType -from . import stored_queries, write_sql -from .views import Context -from .views.database import ( - database_download, - DatabaseView, - QueryView, -) -from .views.table_create_alter import ( - DatabaseForeignKeyTargetsView, - TableAlterView, - TableCreateView, - TableForeignKeySuggestionsView, -) -from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView -from .views.stored_queries import ( - QueryCreateAnalyzeView, - QueryDeleteView, - QueryDefinitionView, - QueryEditView, - GlobalQueryListView, - QueryListView, - QueryParametersView, - QueryStoreView, - QueryUpdateView, -) +from .views.base import DatasetteError, ureg +from .views.database import DatabaseDownload, DatabaseView from .views.index import IndexView from .views.special import ( JsonDataView, PatternPortfolioView, - AutocompleteDebugView, AuthTokenView, - ApiExplorerView, - CreateTokenView, LogoutView, AllowDebugView, PermissionsDebugView, MessagesDebugView, - AllowedResourcesView, - PermissionRulesView, - PermissionCheckView, - JumpView, - InstanceSchemaView, - DatabaseSchemaView, - TableSchemaView, ) -from .views.table import ( - TableAutocompleteView, - TableInsertView, - TableUpsertView, - TableSetColumnTypeView, - TableDropView, - TableFragmentView, - table_view, -) -from .views.row import RowView, RowDeleteView, RowUpdateView +from .views.table import RowView, TableView from .renderer import json_renderer from .url_builder import Urls from .database import Database, QueryInterrupted from .utils import ( - PaginatedResources, PrefixedUrlString, - SPATIALITE_FUNCTIONS, + PrependingLoader, StartupError, async_call_with_supported_arguments, await_me_maybe, - baseconv, call_with_supported_arguments, - detect_json1, display_actor, escape_css_string, escape_sqlite, find_spatialite, format_bytes, module_from_path, - move_plugins_and_allow, - move_table_config, parse_metadata, resolve_env_secrets, - resolve_routes, - sha256_file, - tilde_decode, - tilde_encode, to_css_class, - urlsafe_components, - redact_keys, - row_sql_params_pks, + HASH_LENGTH, ) from .utils.asgi import ( AsgiLifespan, + Base400, Forbidden, NotFound, - DatabaseNotFound, - TableNotFound, - RowNotFound, Request, Response, - AsgiRunOnFirstRequest, asgi_static, asgi_send, - asgi_send_file, + asgi_send_html, + asgi_send_json, asgi_send_redirect, ) -from .csrf import CrossOriginProtectionMiddleware from .utils.internal_db import init_internal_db, populate_schema_tables from .utils.sqlite import ( sqlite3, @@ -155,44 +84,8 @@ from .tracer import AsgiTracer from .plugins import pm, DEFAULT_PLUGINS, get_plugins from .version import __version__ -from .resources import DatabaseResource, TableResource - app_root = Path(__file__).parent.parent - -# Context variable to track when code is executing within a datasette.client request -_in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False) - - -class _DatasetteClientContext: - """Context manager to mark code as executing within a datasette.client request.""" - - def __enter__(self): - self.token = _in_datasette_client.set(True) - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - _in_datasette_client.reset(self.token) - return False - - -@dataclasses.dataclass -class PermissionCheck: - """Represents a logged permission check for debugging purposes.""" - - when: str - actor: Dict[str, Any] | None - action: str - parent: str | None - child: str | None - result: bool - - -# https://github.com/simonw/datasette/issues/283#issuecomment-781591015 -SQLITE_LIMIT_ATTACHED = 10 - -INTERNAL_DB_NAME = "__INTERNAL__" - Setting = collections.namedtuple("Setting", ("name", "default", "help")) SETTINGS = ( Setting("default_page_size", 100, "Default page size for the table view"), @@ -201,11 +94,6 @@ SETTINGS = ( 1000, "Maximum rows that can be returned from a table or custom query", ), - Setting( - "max_insert_rows", - 100, - "Maximum rows that can be inserted at a time using the bulk insert API", - ), Setting( "num_sql_threads", 3, @@ -221,6 +109,11 @@ SETTINGS = ( 50, "Time limit for calculating a suggested facet", ), + Setting( + "hash_urls", + False, + "Include DB file contents hash in URLs, for far-future caching", + ), Setting( "allow_facet", True, @@ -231,27 +124,17 @@ SETTINGS = ( True, "Allow users to download the original SQLite database files", ), - Setting( - "allow_signed_tokens", - True, - "Allow users to create and use signed API tokens", - ), - Setting( - "default_allow_sql", - True, - "Allow anyone to run arbitrary SQL queries", - ), - Setting( - "max_signed_tokens_ttl", - 0, - "Maximum allowed expiry time for signed API tokens", - ), Setting("suggest_facets", True, "Calculate and display suggested facets"), Setting( "default_cache_ttl", 5, "Default HTTP cache TTL (used in Cache-Control: max-age= header)", ), + Setting( + "default_cache_ttl_hashed", + 365 * 24 * 60 * 60, + "Default HTTP cache TTL for hashed URL pages", + ), Setting("cache_size_kb", 0, "SQLite cache size in KB (0 == use SQLite default)"), Setting( "allow_csv_stream", @@ -278,108 +161,14 @@ SETTINGS = ( False, "Allow display of template debug information with ?_context=1", ), - Setting( - "trace_debug", - False, - "Allow display of SQL trace debug information with ?_trace=1", - ), Setting("base_url", "/", "Datasette URLs should use this base path"), ) -_HASH_URLS_REMOVED = "The hash_urls setting has been removed, try the datasette-hashed-urls plugin instead" -OBSOLETE_SETTINGS = { - "hash_urls": _HASH_URLS_REMOVED, - "default_cache_ttl_hashed": _HASH_URLS_REMOVED, -} + DEFAULT_SETTINGS = {option.name: option.default for option in SETTINGS} -FAVICON_PATH = app_root / "datasette" / "static" / "favicon.png" - -DEFAULT_NOT_SET = object() - - -ResourcesSQL = collections.namedtuple("ResourcesSQL", ("sql", "params")) - - -def _permission_cache_key(actor, action, parent, child): - # Key on the full serialized actor so actors differing in any field - # (e.g. token restrictions) never share cache entries - actor_key = ( - json.dumps(actor, sort_keys=True, default=repr) if actor is not None else None - ) - return (actor_key, action, parent, child) - async def favicon(request, send): - await asgi_send_file( - send, - str(FAVICON_PATH), - content_type="image/png", - headers={"Cache-Control": "max-age=3600, public"}, - ) - - -ResolvedTable = collections.namedtuple("ResolvedTable", ("db", "table", "is_view")) -ResolvedRow = collections.namedtuple( - "ResolvedRow", ("db", "table", "sql", "params", "pks", "pk_values", "row") -) - - -def _to_string(value): - if isinstance(value, str): - return value - else: - return json.dumps(value, default=str) - - -def _template_context_json_default(value): - if dataclasses.is_dataclass(value) and not isinstance(value, type): - return { - field.name: getattr(value, field.name) - for field in dataclasses.fields(value) - } - return repr(value) - - -@pass_context -def _legacy_template_csrftoken(context): - request = context.get("request") - if request and "csrftoken" in request.scope: - return request.scope["csrftoken"]() - return "" - - -def _resolve_static_asset_path(root_path, path): - root = Path(root_path).resolve() - full_path = (root / path).resolve() - try: - full_path.relative_to(root) - except ValueError: - raise ValueError("Static asset path cannot escape static root") from None - return full_path - - -# Documentation for the variables Datasette.render_template() adds to the -# context for every page. This is part of the documented template contract: -# keys added in render_template() must be documented here - the contract -# tests in tests/test_template_context.py enforce this, and the docs in -# docs/template_context.rst are generated from it. -TEMPLATE_BASE_CONTEXT = { - "request": "The current :ref:`Request object `, or None. Common properties include ``request.path``, ``request.args``, ``request.actor``, ``request.url_vars`` and ``request.host``.", - "crumb_items": 'Async function returning breadcrumb navigation items for the current page. Call it with ``request=request`` plus optional ``database=`` and ``table=`` arguments; it returns a list of ``{"href": url, "label": label}`` dictionaries.', - "urls": "Object with methods for constructing URLs within Datasette. Common methods include ``urls.instance()``, ``urls.database(database)``, ``urls.table(database, table)``, ``urls.query(database, query)``, ``urls.row(database, table, row_path)`` and ``urls.static(path)`` - see :ref:`internals_datasette_urls`.", - "actor": "The currently authenticated actor dictionary, or None. Actors usually include an ``id`` key and may include any other keys supplied by authentication plugins.", - "menu_links": "Async function returning links for the Datasette application menu, including links added by plugins. Each item is a link dictionary with ``href`` and ``label`` keys. See :ref:`plugin_hook_menu_links`; for page action menus that can also include JavaScript-backed buttons, see :ref:`plugin_actions`.", - "display_actor": "Function that accepts an actor dictionary and returns the display string used in the navigation menu.", - "show_logout": "True if the logout link should be shown in the navigation menu", - "zip": "Python's ``zip()`` builtin, made available to template logic", - "body_scripts": 'List of JavaScript snippets contributed by plugins using :ref:`plugin_hook_extra_body_script`. Each item is a dictionary with ``script`` containing JavaScript source and ``module`` indicating whether Datasette will wrap it in `` diff --git a/datasette/templates/_codemirror.html b/datasette/templates/_codemirror.html index 75c16168..b31235d2 100644 --- a/datasette/templates/_codemirror.html +++ b/datasette/templates/_codemirror.html @@ -1,16 +1,8 @@ - - + + + + diff --git a/datasette/templates/_codemirror_foot.html b/datasette/templates/_codemirror_foot.html index a624c8a4..4019d448 100644 --- a/datasette/templates/_codemirror_foot.html +++ b/datasette/templates/_codemirror_foot.html @@ -1,42 +1,37 @@ diff --git a/datasette/templates/_crumbs.html b/datasette/templates/_crumbs.html deleted file mode 100644 index bd1ff0da..00000000 --- a/datasette/templates/_crumbs.html +++ /dev/null @@ -1,15 +0,0 @@ -{% macro nav(request, database=None, table=None) -%} -{% if crumb_items is defined %} - {% set items=crumb_items(request=request, database=database, table=table) %} - {% if items %} -

- {% for item in items %} - {{ item.label }} - {% if not loop.last %} - / - {% endif %} - {% endfor %} -

- {% endif %} -{% endif %} -{%- endmacro %} diff --git a/datasette/templates/_debug_common_functions.html b/datasette/templates/_debug_common_functions.html deleted file mode 100644 index d988a2f3..00000000 --- a/datasette/templates/_debug_common_functions.html +++ /dev/null @@ -1,50 +0,0 @@ - diff --git a/datasette/templates/_description_source_license.html b/datasette/templates/_description_source_license.html index f852268f..a2bc18f2 100644 --- a/datasette/templates/_description_source_license.html +++ b/datasette/templates/_description_source_license.html @@ -1,6 +1,6 @@ -{% if metadata.get("description_html") or metadata.get("description") %} +{% if metadata.description_html or metadata.description %}
- {% if metadata.get("description_html") %} + {% if metadata.description_html %} {{ metadata.description_html|safe }} {% else %} {{ metadata.description }} diff --git a/datasette/templates/_execute_write_analysis_scripts.html b/datasette/templates/_execute_write_analysis_scripts.html deleted file mode 100644 index a19bae13..00000000 --- a/datasette/templates/_execute_write_analysis_scripts.html +++ /dev/null @@ -1,111 +0,0 @@ - diff --git a/datasette/templates/_execute_write_analysis_styles.html b/datasette/templates/_execute_write_analysis_styles.html deleted file mode 100644 index 165cfe9f..00000000 --- a/datasette/templates/_execute_write_analysis_styles.html +++ /dev/null @@ -1,41 +0,0 @@ - diff --git a/datasette/templates/_facet_results.html b/datasette/templates/_facet_results.html deleted file mode 100644 index 570bb37e..00000000 --- a/datasette/templates/_facet_results.html +++ /dev/null @@ -1,28 +0,0 @@ -
- {% for facet_info in sorted_facet_results %} -
-

- {{ facet_info.name }}{% if facet_info.type != "column" %} ({{ facet_info.type }}){% endif %} - {% if facet_info.truncated %}>{% endif %}{{ facet_info.results|length }} - - {% if facet_info.hideable %} - ✖ - {% endif %} -

-
    - {% for facet_value in facet_info.results %} - {% if not facet_value.selected %} -
  • {{ (facet_value.label | string()) or "-" }} {{ "{:,}".format(facet_value.count) }}
    • - {% else %} -
  • {{ facet_value.label or "-" }} · {{ "{:,}".format(facet_value.count) }} ✖
    • - {% endif %} - {% endfor %} - {% if facet_info.truncated %} -
  • {% if request.args._facet_size != "max" -%} - …{% else -%}…{% endif %} -
    • - {% endif %} -
-
- {% endfor %} -
diff --git a/datasette/templates/_footer.html b/datasette/templates/_footer.html index 074270f1..b1380ae9 100644 --- a/datasette/templates/_footer.html +++ b/datasette/templates/_footer.html @@ -1,5 +1,5 @@ Powered by Datasette -{% if query_ms %}· Queries took {{ query_ms|round(3) }}ms{% endif %} +{% if query_ms %}· Query took {{ query_ms|round(3) }}ms{% endif %} {% if metadata %} {% if metadata.license or metadata.license_url %}· Data license: {% if metadata.license_url %} diff --git a/datasette/templates/_permission_ui_styles.html b/datasette/templates/_permission_ui_styles.html deleted file mode 100644 index 53a824f1..00000000 --- a/datasette/templates/_permission_ui_styles.html +++ /dev/null @@ -1,145 +0,0 @@ - diff --git a/datasette/templates/_permissions_debug_tabs.html b/datasette/templates/_permissions_debug_tabs.html deleted file mode 100644 index d7203c1e..00000000 --- a/datasette/templates/_permissions_debug_tabs.html +++ /dev/null @@ -1,54 +0,0 @@ -{% if has_debug_permission %} -{% set query_string = '?' + request.query_string if request.query_string else '' %} - - - - -{% endif %} diff --git a/datasette/templates/_query_form_styles.html b/datasette/templates/_query_form_styles.html deleted file mode 100644 index cf2dd42c..00000000 --- a/datasette/templates/_query_form_styles.html +++ /dev/null @@ -1,138 +0,0 @@ - diff --git a/datasette/templates/_query_results.html b/datasette/templates/_query_results.html deleted file mode 100644 index 5e1e2f72..00000000 --- a/datasette/templates/_query_results.html +++ /dev/null @@ -1,20 +0,0 @@ -{% if display_rows %} -
- - - {% for column in columns %}{% endfor %} - - - - {% for row in display_rows %} - - {% for column, td in zip(columns, row) %} - - {% endfor %} - - {% endfor %} - -
{{ column }}
{{ td }}
-{% elif show_zero_results %} -

0 results

-{% endif %} diff --git a/datasette/templates/_sql_parameter_scripts.html b/datasette/templates/_sql_parameter_scripts.html deleted file mode 100644 index 9b83889e..00000000 --- a/datasette/templates/_sql_parameter_scripts.html +++ /dev/null @@ -1,307 +0,0 @@ - diff --git a/datasette/templates/_sql_parameter_styles.html b/datasette/templates/_sql_parameter_styles.html deleted file mode 100644 index bc6838f5..00000000 --- a/datasette/templates/_sql_parameter_styles.html +++ /dev/null @@ -1,58 +0,0 @@ - diff --git a/datasette/templates/_sql_parameters.html b/datasette/templates/_sql_parameters.html deleted file mode 100644 index b5c1bde8..00000000 --- a/datasette/templates/_sql_parameters.html +++ /dev/null @@ -1,10 +0,0 @@ -{% set sql_parameter_name_prefix = sql_parameter_name_prefix|default("") %} -
- {% if parameter_names %} -

Parameters

- {% for parameter in parameter_names %} - {% set parameter_id = (sql_parameter_id_prefix|default("qp")) ~ loop.index %} -

{% if sql_parameters_allow_expand|default(false) %} {% endif %}

- {% endfor %} - {% endif %} -
diff --git a/datasette/templates/_suggested_facets.html b/datasette/templates/_suggested_facets.html deleted file mode 100644 index b80208c3..00000000 --- a/datasette/templates/_suggested_facets.html +++ /dev/null @@ -1,3 +0,0 @@ -

- Suggested facets: {% for facet in suggested_facets %}{{ facet.name }}{% if facet.get("type") %} ({{ facet.type }}){% endif %}{% if not loop.last %}, {% endif %}{% endfor %} -

diff --git a/datasette/templates/_table.html b/datasette/templates/_table.html index 171b6442..d765937e 100644 --- a/datasette/templates/_table.html +++ b/datasette/templates/_table.html @@ -1,19 +1,17 @@ - -
-{% if display_columns %} +{% if display_rows %}
{% for column in display_columns %} - @@ -22,7 +20,7 @@ {% for row in display_rows %} - + {% for cell in row %} {% endfor %} @@ -31,7 +29,6 @@
+ {% if not column.sortable %} {{ column.name }} {% else %} {% if column.name == sort %} - {{ column.name }} ▼ + {{ column.name }} ▼ {% else %} - {{ column.name }}{% if column.name == sort_desc %} ▲{% endif %} + {{ column.name }}{% if column.name == sort_desc %} ▲{% endif %} {% endif %} {% endif %}
{{ cell.value }}
-{% endif %} -{% if not display_rows %} +{% else %}

0 records

{% endif %} diff --git a/datasette/templates/allow_debug.html b/datasette/templates/allow_debug.html index 1ecc92df..0f1b30f0 100644 --- a/datasette/templates/allow_debug.html +++ b/datasette/templates/allow_debug.html @@ -33,12 +33,9 @@ p.message-warning {

Debug allow rules

-{% set current_tab = "allow_debug" %} -{% include "_permissions_debug_tabs.html" %} -

Use this tool to try out different actor and allow combinations. See Defining permissions with "allow" blocks for documentation.

-
+

diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html deleted file mode 100644 index 4927cb8d..00000000 --- a/datasette/templates/api_explorer.html +++ /dev/null @@ -1,208 +0,0 @@ -{% extends "base.html" %} - -{% block title %}API Explorer{% endblock %} - -{% block extra_head %} - -{% endblock %} - -{% block content %} - -

API Explorer{% if private %} 🔒{% endif %}

- -

Use this tool to try out the - {% if datasette_version %} - Datasette API. - {% else %} - Datasette API. - {% endif %} -

-
- GET - -
- - - -
- -
-
- POST -
-
- - -
-
- - -
-

-
-
- - - - - -{% if example_links %} -

API endpoints

-
    - {% for database in example_links %} -
  • Database: {{ database.name }}
    • -
      - {% for link in database.links %} -
    • {{ link.path }} - {{ link.label }}
      • - {% endfor %} - {% for table in database.tables %} -
    • {{ table.name }} -
        - {% for link in table.links %} -
      • {{ link.path }} - {{ link.label }}
        • - {% endfor %} -
      -
      • - {% endfor %} -
    - {% endfor %} -
-{% endif %} - -{% endblock %} diff --git a/datasette/templates/base.html b/datasette/templates/base.html index 18288439..e61edc4f 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -1,27 +1,21 @@ -{% import "_crumbs.html" as crumbs with context %} - + + {% block title %}{% endblock %} - + {% for url in extra_css_urls %} - + {% endfor %} - - {% for url in extra_js_urls %} - + {% endfor %} -{%- if alternate_url_json -%} - -{%- endif -%} -{%- block extra_head %}{% endblock -%} +{% block extra_head %}{% endblock %} -
-