From c207453e030ce2cbe593f6d6d2dc9b99ecdf3af8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 6 Jul 2019 15:17:14 -0700 Subject: [PATCH 01/10] --plugin-secret option, refs #543 --- datasette/publish/cloudrun.py | 1 + datasette/publish/common.py | 7 +++++ datasette/publish/heroku.py | 1 + datasette/publish/now.py | 36 ++++++++++++++++++------ datasette/utils/__init__.py | 10 +++++++ docs/datasette-publish-cloudrun-help.txt | 2 ++ docs/datasette-publish-heroku-help.txt | 2 ++ docs/datasette-publish-nowv1-help.txt | 2 ++ 8 files changed, 52 insertions(+), 9 deletions(-) diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index 436b5d2b..ffedef44 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -33,6 +33,7 @@ def publish_subcommand(publish): plugins_dir, static, install, + plugin_secret, version_note, title, license, diff --git a/datasette/publish/common.py b/datasette/publish/common.py index a31eef02..037daf15 100644 --- a/datasette/publish/common.py +++ b/datasette/publish/common.py @@ -41,6 +41,13 @@ def add_common_publish_arguments_and_options(subcommand): help="Additional packages (e.g. plugins) to install", multiple=True, ), + click.option( + "--plugin-secret", + nargs=3, + type=str, + multiple=True, + help="Secrets to pass to plugins, e.g. --plugin-secret datasette-auth-github client_id xxx", + ), click.option( "--version-note", help="Additional note to show on /-/versions" ), diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py index 5705500f..085e6c3a 100644 --- a/datasette/publish/heroku.py +++ b/datasette/publish/heroku.py @@ -33,6 +33,7 @@ def publish_subcommand(publish): plugins_dir, static, install, + plugin_secret, version_note, title, license, diff --git a/datasette/publish/now.py b/datasette/publish/now.py index 38add86e..e15109d9 100644 --- a/datasette/publish/now.py +++ b/datasette/publish/now.py @@ -33,6 +33,7 @@ def publish_subcommand(publish): plugins_dir, static, install, + plugin_secret, version_note, title, license, @@ -54,6 +55,30 @@ def publish_subcommand(publish): extra_options = "" extra_options += "--config force_https_urls:on" + extra_metadata = { + "title": title, + "license": license, + "license_url": license_url, + "source": source, + "source_url": source_url, + "about": about, + "about_url": about_url, + } + + environment_variables = {} + if plugin_secret: + extra_metadata["plugins"] = {} + for plugin_name, plugin_setting, setting_value in plugin_secret: + environment_variable = ( + "{}_{}".format(plugin_name, plugin_setting) + .upper() + .replace("-", "_") + ) + environment_variables[environment_variable] = setting_value + extra_metadata["plugins"].setdefault(plugin_name, {})[ + plugin_setting + ] = {"$env": environment_variable} + with temporary_docker_directory( files, name, @@ -66,15 +91,8 @@ def publish_subcommand(publish): install, spatialite, version_note, - { - "title": title, - "license": license, - "license_url": license_url, - "source": source, - "source_url": source_url, - "about": about, - "about_url": about_url, - }, + extra_metadata, + environment_variables, ): now_json = {"version": 1} open("now.json", "w").write(json.dumps(now_json, indent=4)) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 17a4d595..1c3fa16b 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -272,6 +272,7 @@ def make_dockerfile( install, spatialite, version_note, + environment_variables=None, ): cmd = ["datasette", "serve", "--host", "0.0.0.0"] for filename in files: @@ -307,11 +308,18 @@ FROM python:3.6 COPY . /app WORKDIR /app {spatialite_extras} +{environment_variables} RUN pip install -U {install_from} RUN datasette inspect {files} --inspect-file inspect-data.json ENV PORT 8001 EXPOSE 8001 CMD {cmd}""".format( + environment_variables="\n".join( + [ + "ENV {} {}".format(key, value) + for key, value in (environment_variables or {}).items() + ] + ), files=" ".join(files), cmd=cmd, install_from=" ".join(install), @@ -333,6 +341,7 @@ def temporary_docker_directory( spatialite, version_note, extra_metadata=None, + environment_variables=None, ): extra_metadata = extra_metadata or {} tmp = tempfile.TemporaryDirectory() @@ -361,6 +370,7 @@ def temporary_docker_directory( install, spatialite, version_note, + environment_variables, ) os.chdir(datasette_dir) if metadata_content: diff --git a/docs/datasette-publish-cloudrun-help.txt b/docs/datasette-publish-cloudrun-help.txt index fc7d44d5..45a8f083 100644 --- a/docs/datasette-publish-cloudrun-help.txt +++ b/docs/datasette-publish-cloudrun-help.txt @@ -10,6 +10,8 @@ Options: --plugins-dir DIRECTORY Path to directory containing custom plugins --static STATIC MOUNT mountpoint:path-to-directory for serving static files --install TEXT Additional packages (e.g. plugins) to install + --plugin-secret TEXT... Secrets to pass to plugins, e.g. --plugin-secret datasette- + auth-github client_id xxx --version-note TEXT Additional note to show on /-/versions --title TEXT Title for metadata --license TEXT License label for metadata diff --git a/docs/datasette-publish-heroku-help.txt b/docs/datasette-publish-heroku-help.txt index cd9af09b..52748429 100644 --- a/docs/datasette-publish-heroku-help.txt +++ b/docs/datasette-publish-heroku-help.txt @@ -10,6 +10,8 @@ Options: --plugins-dir DIRECTORY Path to directory containing custom plugins --static STATIC MOUNT mountpoint:path-to-directory for serving static files --install TEXT Additional packages (e.g. plugins) to install + --plugin-secret TEXT... Secrets to pass to plugins, e.g. --plugin-secret datasette- + auth-github client_id xxx --version-note TEXT Additional note to show on /-/versions --title TEXT Title for metadata --license TEXT License label for metadata diff --git a/docs/datasette-publish-nowv1-help.txt b/docs/datasette-publish-nowv1-help.txt index a5417d71..a625cd67 100644 --- a/docs/datasette-publish-nowv1-help.txt +++ b/docs/datasette-publish-nowv1-help.txt @@ -10,6 +10,8 @@ Options: --plugins-dir DIRECTORY Path to directory containing custom plugins --static STATIC MOUNT mountpoint:path-to-directory for serving static files --install TEXT Additional packages (e.g. plugins) to install + --plugin-secret TEXT... Secrets to pass to plugins, e.g. --plugin-secret datasette- + auth-github client_id xxx --version-note TEXT Additional note to show on /-/versions --title TEXT Title for metadata --license TEXT License label for metadata From 4e5f6b4d112328fb0ab53185eade4b66a1604974 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 6 Jul 2019 15:35:45 -0700 Subject: [PATCH 02/10] Fix escaping of ENV --- datasette/publish/common.py | 10 ++++- datasette/utils/__init__.py | 2 +- docs/datasette-publish-cloudrun-help.txt | 43 +++++++++++----------- docs/datasette-publish-heroku-help.txt | 39 ++++++++++---------- docs/datasette-publish-nowv1-help.txt | 47 ++++++++++++------------ 5 files changed, 76 insertions(+), 65 deletions(-) diff --git a/datasette/publish/common.py b/datasette/publish/common.py index 037daf15..5bbbf613 100644 --- a/datasette/publish/common.py +++ b/datasette/publish/common.py @@ -44,7 +44,8 @@ def add_common_publish_arguments_and_options(subcommand): click.option( "--plugin-secret", nargs=3, - type=str, + type=(str, str, str), + callback=validate_plugin_secret, multiple=True, help="Secrets to pass to plugins, e.g. --plugin-secret datasette-auth-github client_id xxx", ), @@ -83,3 +84,10 @@ def fail_if_publish_binary_not_installed(binary, publish_target, install_link): err=True, ) sys.exit(1) + + +def validate_plugin_secret(ctx, param, value): + for plugin_name, plugin_setting, setting_value in value: + if "'" in setting_value: + raise click.BadParameter("--plugin-secret cannot contain single quotes") + return value diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 1c3fa16b..d92d0cd5 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -316,7 +316,7 @@ EXPOSE 8001 CMD {cmd}""".format( environment_variables="\n".join( [ - "ENV {} {}".format(key, value) + "ENV {} '{}'".format(key, value) for key, value in (environment_variables or {}).items() ] ), diff --git a/docs/datasette-publish-cloudrun-help.txt b/docs/datasette-publish-cloudrun-help.txt index 45a8f083..19e3c799 100644 --- a/docs/datasette-publish-cloudrun-help.txt +++ b/docs/datasette-publish-cloudrun-help.txt @@ -3,24 +3,25 @@ $ datasette publish cloudrun --help Usage: datasette publish cloudrun [OPTIONS] [FILES]... Options: - -m, --metadata FILENAME Path to JSON file containing metadata to publish - --extra-options TEXT Extra options to pass to datasette serve - --branch TEXT Install datasette from a GitHub branch e.g. master - --template-dir DIRECTORY Path to directory containing custom templates - --plugins-dir DIRECTORY Path to directory containing custom plugins - --static STATIC MOUNT mountpoint:path-to-directory for serving static files - --install TEXT Additional packages (e.g. plugins) to install - --plugin-secret TEXT... Secrets to pass to plugins, e.g. --plugin-secret datasette- - auth-github client_id xxx - --version-note TEXT Additional note to show on /-/versions - --title TEXT Title for metadata - --license TEXT License label for metadata - --license_url TEXT License URL for metadata - --source TEXT Source label for metadata - --source_url TEXT Source URL for metadata - --about TEXT About label for metadata - --about_url TEXT About URL for metadata - -n, --name TEXT Application name to use when building - --service TEXT Cloud Run service to deploy (or over-write) - --spatialite Enable SpatialLite extension - --help Show this message and exit. + -m, --metadata FILENAME Path to JSON file containing metadata to publish + --extra-options TEXT Extra options to pass to datasette serve + --branch TEXT Install datasette from a GitHub branch e.g. master + --template-dir DIRECTORY Path to directory containing custom templates + --plugins-dir DIRECTORY Path to directory containing custom plugins + --static STATIC MOUNT mountpoint:path-to-directory for serving static files + --install TEXT Additional packages (e.g. plugins) to install + --plugin-secret ... + Secrets to pass to plugins, e.g. --plugin-secret + datasette-auth-github client_id xxx + --version-note TEXT Additional note to show on /-/versions + --title TEXT Title for metadata + --license TEXT License label for metadata + --license_url TEXT License URL for metadata + --source TEXT Source label for metadata + --source_url TEXT Source URL for metadata + --about TEXT About label for metadata + --about_url TEXT About URL for metadata + -n, --name TEXT Application name to use when building + --service TEXT Cloud Run service to deploy (or over-write) + --spatialite Enable SpatialLite extension + --help Show this message and exit. diff --git a/docs/datasette-publish-heroku-help.txt b/docs/datasette-publish-heroku-help.txt index 52748429..88d387a6 100644 --- a/docs/datasette-publish-heroku-help.txt +++ b/docs/datasette-publish-heroku-help.txt @@ -3,22 +3,23 @@ $ datasette publish heroku --help Usage: datasette publish heroku [OPTIONS] [FILES]... Options: - -m, --metadata FILENAME Path to JSON file containing metadata to publish - --extra-options TEXT Extra options to pass to datasette serve - --branch TEXT Install datasette from a GitHub branch e.g. master - --template-dir DIRECTORY Path to directory containing custom templates - --plugins-dir DIRECTORY Path to directory containing custom plugins - --static STATIC MOUNT mountpoint:path-to-directory for serving static files - --install TEXT Additional packages (e.g. plugins) to install - --plugin-secret TEXT... Secrets to pass to plugins, e.g. --plugin-secret datasette- - auth-github client_id xxx - --version-note TEXT Additional note to show on /-/versions - --title TEXT Title for metadata - --license TEXT License label for metadata - --license_url TEXT License URL for metadata - --source TEXT Source label for metadata - --source_url TEXT Source URL for metadata - --about TEXT About label for metadata - --about_url TEXT About URL for metadata - -n, --name TEXT Application name to use when deploying - --help Show this message and exit. + -m, --metadata FILENAME Path to JSON file containing metadata to publish + --extra-options TEXT Extra options to pass to datasette serve + --branch TEXT Install datasette from a GitHub branch e.g. master + --template-dir DIRECTORY Path to directory containing custom templates + --plugins-dir DIRECTORY Path to directory containing custom plugins + --static STATIC MOUNT mountpoint:path-to-directory for serving static files + --install TEXT Additional packages (e.g. plugins) to install + --plugin-secret ... + Secrets to pass to plugins, e.g. --plugin-secret + datasette-auth-github client_id xxx + --version-note TEXT Additional note to show on /-/versions + --title TEXT Title for metadata + --license TEXT License label for metadata + --license_url TEXT License URL for metadata + --source TEXT Source label for metadata + --source_url TEXT Source URL for metadata + --about TEXT About label for metadata + --about_url TEXT About URL for metadata + -n, --name TEXT Application name to use when deploying + --help Show this message and exit. diff --git a/docs/datasette-publish-nowv1-help.txt b/docs/datasette-publish-nowv1-help.txt index a625cd67..0ab0758e 100644 --- a/docs/datasette-publish-nowv1-help.txt +++ b/docs/datasette-publish-nowv1-help.txt @@ -3,26 +3,27 @@ $ datasette publish nowv1 --help Usage: datasette publish nowv1 [OPTIONS] [FILES]... Options: - -m, --metadata FILENAME Path to JSON file containing metadata to publish - --extra-options TEXT Extra options to pass to datasette serve - --branch TEXT Install datasette from a GitHub branch e.g. master - --template-dir DIRECTORY Path to directory containing custom templates - --plugins-dir DIRECTORY Path to directory containing custom plugins - --static STATIC MOUNT mountpoint:path-to-directory for serving static files - --install TEXT Additional packages (e.g. plugins) to install - --plugin-secret TEXT... Secrets to pass to plugins, e.g. --plugin-secret datasette- - auth-github client_id xxx - --version-note TEXT Additional note to show on /-/versions - --title TEXT Title for metadata - --license TEXT License label for metadata - --license_url TEXT License URL for metadata - --source TEXT Source label for metadata - --source_url TEXT Source URL for metadata - --about TEXT About label for metadata - --about_url TEXT About URL for metadata - -n, --name TEXT Application name to use when deploying - --force Pass --force option to now - --token TEXT Auth token to use for deploy - --alias TEXT Desired alias e.g. yoursite.now.sh - --spatialite Enable SpatialLite extension - --help Show this message and exit. + -m, --metadata FILENAME Path to JSON file containing metadata to publish + --extra-options TEXT Extra options to pass to datasette serve + --branch TEXT Install datasette from a GitHub branch e.g. master + --template-dir DIRECTORY Path to directory containing custom templates + --plugins-dir DIRECTORY Path to directory containing custom plugins + --static STATIC MOUNT mountpoint:path-to-directory for serving static files + --install TEXT Additional packages (e.g. plugins) to install + --plugin-secret ... + Secrets to pass to plugins, e.g. --plugin-secret + datasette-auth-github client_id xxx + --version-note TEXT Additional note to show on /-/versions + --title TEXT Title for metadata + --license TEXT License label for metadata + --license_url TEXT License URL for metadata + --source TEXT Source label for metadata + --source_url TEXT Source URL for metadata + --about TEXT About label for metadata + --about_url TEXT About URL for metadata + -n, --name TEXT Application name to use when deploying + --force Pass --force option to now + --token TEXT Auth token to use for deploy + --alias TEXT Desired alias e.g. yoursite.now.sh + --spatialite Enable SpatialLite extension + --help Show this message and exit. From 4c2f4dbd600d4127c783652a3c5c08c68e23c7b1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Jul 2019 13:06:36 -0700 Subject: [PATCH 03/10] --plugin-secret support for Heroku --- datasette/publish/heroku.py | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py index 085e6c3a..ae08f916 100644 --- a/datasette/publish/heroku.py +++ b/datasette/publish/heroku.py @@ -62,6 +62,30 @@ def publish_subcommand(publish): ) call(["heroku", "plugins:install", "heroku-builds"]) + extra_metadata = { + "title": title, + "license": license, + "license_url": license_url, + "source": source, + "source_url": source_url, + "about": about, + "about_url": about_url, + } + + environment_variables = {} + if plugin_secret: + extra_metadata["plugins"] = {} + for plugin_name, plugin_setting, setting_value in plugin_secret: + environment_variable = ( + "{}_{}".format(plugin_name, plugin_setting) + .upper() + .replace("-", "_") + ) + environment_variables[environment_variable] = setting_value + extra_metadata["plugins"].setdefault(plugin_name, {})[ + plugin_setting + ] = {"$env": environment_variable} + with temporary_heroku_directory( files, name, @@ -73,15 +97,7 @@ def publish_subcommand(publish): static, install, version_note, - { - "title": title, - "license": license, - "license_url": license_url, - "source": source, - "source_url": source_url, - "about": about, - "about_url": about_url, - }, + extra_metadata, ): app_name = None if name: @@ -105,6 +121,9 @@ def publish_subcommand(publish): create_output = check_output(cmd).decode("utf8") app_name = json.loads(create_output)["name"] + for key, value in environment_variables.items(): + call(["heroku", "config:set", "-a", app_name, "{}={}".format(key, value)]) + call(["heroku", "builds:create", "-a", app_name, "--include-vcs-ignore"]) From c92349e7ee2b1605b708dc63f93b2269d6094f2f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Jul 2019 13:18:15 -0700 Subject: [PATCH 04/10] Applied Black --- datasette/publish/heroku.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py index ae08f916..34d1f773 100644 --- a/datasette/publish/heroku.py +++ b/datasette/publish/heroku.py @@ -122,7 +122,9 @@ def publish_subcommand(publish): app_name = json.loads(create_output)["name"] for key, value in environment_variables.items(): - call(["heroku", "config:set", "-a", app_name, "{}={}".format(key, value)]) + call( + ["heroku", "config:set", "-a", app_name, "{}={}".format(key, value)] + ) call(["heroku", "builds:create", "-a", app_name, "--include-vcs-ignore"]) From 08dd0d96eddbaf8e41fbe7a1e85fd3984c469e6c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Jul 2019 13:21:50 -0700 Subject: [PATCH 05/10] white-space: pre-wrap for table SQL, closes #505 --- datasette/static/app.css | 4 ++++ datasette/templates/table.html | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 76ecdd8d..c5f5c679 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -312,3 +312,7 @@ a.not-underlined { font-style: normal; font-size: 0.8em; } + +pre.wrapped-sql { + white-space: pre-wrap; +} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index c7913f60..1841300b 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -184,11 +184,11 @@ {% endif %} {% if table_definition %} - 
{{ table_definition }}
+ 
{{ table_definition }}
{% endif %} {% if view_definition %} - 
{{ view_definition }}
+ 
{{ view_definition }}
{% endif %} {% endblock %} From 18bf0675f5f05d6a948d583db0a34e6dc1b530c4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Jul 2019 13:25:38 -0700 Subject: [PATCH 06/10] Fix nav display on 500 page, closes #545 --- datasette/templates/500.html | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/datasette/templates/500.html b/datasette/templates/500.html index 809b2a71..46573f30 100644 --- a/datasette/templates/500.html +++ b/datasette/templates/500.html @@ -2,8 +2,14 @@ {% block title %}{% if title %}{{ title }}{% else %}Error {{ status }}{% endif %}{% endblock %} +{% block nav %} +

+ home +

+ {{ super() }} +{% endblock %} + {% block content %} -
home

{% if title %}{{ title }}{% else %}Error {{ status }}{% endif %}

From 741760d05c6836396756ff6be44a4d375a83a06f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Jul 2019 16:57:04 -0700 Subject: [PATCH 07/10] Unit tests for now --plugin-secret option Also added new --show-files option to - useful for debugging. --- datasette/publish/now.py | 14 +++++++ docs/datasette-publish-nowv1-help.txt | 1 + tests/test_publish_now.py | 56 +++++++++++++++++++++++++++ 3 files changed, 71 insertions(+) diff --git a/datasette/publish/now.py b/datasette/publish/now.py index e15109d9..d7831c80 100644 --- a/datasette/publish/now.py +++ b/datasette/publish/now.py @@ -1,6 +1,7 @@ from datasette import hookimpl import click import json +import os from subprocess import run, PIPE from .common import ( @@ -24,6 +25,11 @@ def publish_subcommand(publish): @click.option("--token", help="Auth token to use for deploy") @click.option("--alias", multiple=True, help="Desired alias e.g. yoursite.now.sh") @click.option("--spatialite", is_flag=True, help="Enable SpatialLite extension") + @click.option( + "--show-files", + is_flag=True, + help="Output the generated Dockerfile and metadata.json", + ) def nowv1( files, metadata, @@ -47,6 +53,7 @@ def publish_subcommand(publish): token, alias, spatialite, + show_files, ): fail_if_publish_binary_not_installed("now", "Zeit Now", "https://zeit.co/now") if extra_options: @@ -106,6 +113,13 @@ def publish_subcommand(publish): else: done = run("now", stdout=PIPE) deployment_url = done.stdout + if show_files: + if os.path.exists("metadata.json"): + print("=== metadata.json ===\n") + print(open("metadata.json").read()) + print("\n==== Dockerfile ====\n") + print(open("Dockerfile").read()) + print("\n====================\n") if alias: # I couldn't get --target=production working, so I call # 'now alias' with arguments directly instead - but that diff --git a/docs/datasette-publish-nowv1-help.txt b/docs/datasette-publish-nowv1-help.txt index 0ab0758e..c2bf23f1 100644 --- a/docs/datasette-publish-nowv1-help.txt +++ b/docs/datasette-publish-nowv1-help.txt @@ -26,4 +26,5 @@ Options: --token TEXT Auth token to use for deploy --alias TEXT Desired alias e.g. yoursite.now.sh --spatialite Enable SpatialLite extension + --show-files Output the generated Dockerfile and metadata.json --help Show this message and exit. diff --git a/tests/test_publish_now.py b/tests/test_publish_now.py index fa1ab30a..72aa71db 100644 --- a/tests/test_publish_now.py +++ b/tests/test_publish_now.py @@ -1,6 +1,7 @@ from click.testing import CliRunner from datasette import cli from unittest import mock +import json import subprocess @@ -105,3 +106,58 @@ def test_publish_now_multiple_aliases(mock_run, mock_which): ), ] ) + + +@mock.patch("shutil.which") +@mock.patch("datasette.publish.now.run") +def test_publish_now_plugin_secrets(mock_run, mock_which): + mock_which.return_value = True + mock_run.return_value = mock.Mock(0) + mock_run.return_value.stdout = b"https://demo.example.com/" + + runner = CliRunner() + with runner.isolated_filesystem(): + open("test.db", "w").write("data") + result = runner.invoke( + cli.cli, + [ + "publish", + "now", + "test.db", + "--token", + "XXX", + "--plugin-secret", + "datasette-auth-github", + "client_id", + "x-client-id", + "--show-files", + ], + ) + dockerfile = ( + result.output.split("==== Dockerfile ====\n")[1] + .split("\n====================\n")[0] + .strip() + ) + expected = """FROM python:3.6 +COPY . /app +WORKDIR /app + +ENV DATASETTE_AUTH_GITHUB_CLIENT_ID 'x-client-id' +RUN pip install -U datasette +RUN datasette inspect test.db --inspect-file inspect-data.json +ENV PORT 8001 +EXPOSE 8001 +CMD datasette serve --host 0.0.0.0 -i test.db --cors --inspect-file inspect-data.json --metadata metadata.json --config force_https_urls:on --port $PORT""".strip() + assert expected == dockerfile + metadata = ( + result.output.split("=== metadata.json ===\n")[1] + .split("\n==== Dockerfile ====\n")[0] + .strip() + ) + assert { + "plugins": { + "datasette-auth-github": { + "client_id": {"$env": "DATASETTE_AUTH_GITHUB_CLIENT_ID"} + } + } + } == json.loads(metadata) From ba69d520ae92fb9221378d0cdc74a2aa5a43d4d8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Jul 2019 18:27:43 -0700 Subject: [PATCH 08/10] Unit test for heroku --plugin-secret --- tests/test_publish_heroku.py | 46 +++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/tests/test_publish_heroku.py b/tests/test_publish_heroku.py index 08fdeaea..4cd66219 100644 --- a/tests/test_publish_heroku.py +++ b/tests/test_publish_heroku.py @@ -46,7 +46,7 @@ def test_publish_heroku_invalid_database(mock_which): @mock.patch("datasette.publish.heroku.check_output") @mock.patch("datasette.publish.heroku.call") def test_publish_heroku(mock_call, mock_check_output, mock_which): - mock_which.return_varue = True + mock_which.return_value = True mock_check_output.side_effect = lambda s: { "['heroku', 'plugins']": b"heroku-builds", "['heroku', 'apps:list', '--json']": b"[]", @@ -60,3 +60,47 @@ def test_publish_heroku(mock_call, mock_check_output, mock_which): mock_call.assert_called_once_with( ["heroku", "builds:create", "-a", "f", "--include-vcs-ignore"] ) + + +@mock.patch("shutil.which") +@mock.patch("datasette.publish.heroku.check_output") +@mock.patch("datasette.publish.heroku.call") +def test_publish_heroku_plugin_secrets(mock_call, mock_check_output, mock_which): + mock_which.return_value = True + mock_check_output.side_effect = lambda s: { + "['heroku', 'plugins']": b"heroku-builds", + "['heroku', 'apps:list', '--json']": b"[]", + "['heroku', 'apps:create', 'datasette', '--json']": b'{"name": "f"}', + }[repr(s)] + runner = CliRunner() + with runner.isolated_filesystem(): + open("test.db", "w").write("data") + result = runner.invoke( + cli.cli, + [ + "publish", + "heroku", + "test.db", + "--plugin-secret", + "datasette-auth-github", + "client_id", + "x-client-id", + ], + ) + assert 0 == result.exit_code, result.output + mock_call.assert_has_calls( + [ + mock.call( + [ + "heroku", + "config:set", + "-a", + "f", + "DATASETTE_AUTH_GITHUB_CLIENT_ID=x-client-id", + ] + ), + mock.call( + ["heroku", "builds:create", "-a", "f", "--include-vcs-ignore"] + ), + ] + ) From c2df6f3482cbe607cf4791003bef9d85c22a6c23 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Jul 2019 18:55:42 -0700 Subject: [PATCH 09/10] --plugin-secret support for Cloud Run --- datasette/publish/cloudrun.py | 50 ++++++++++++++++++---- docs/datasette-publish-cloudrun-help.txt | 1 + tests/test_publish_cloudrun.py | 54 ++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 9 deletions(-) diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index ffedef44..32c9cd2a 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -1,6 +1,7 @@ from datasette import hookimpl import click import json +import os from subprocess import check_call, check_output from .common import ( @@ -24,6 +25,11 @@ def publish_subcommand(publish): "--service", default="", help="Cloud Run service to deploy (or over-write)" ) @click.option("--spatialite", is_flag=True, help="Enable SpatialLite extension") + @click.option( + "--show-files", + is_flag=True, + help="Output the generated Dockerfile and metadata.json", + ) def cloudrun( files, metadata, @@ -45,6 +51,7 @@ def publish_subcommand(publish): name, service, spatialite, + show_files, ): fail_if_publish_binary_not_installed( "gcloud", "Google Cloud", "https://cloud.google.com/sdk/" @@ -53,6 +60,30 @@ def publish_subcommand(publish): "gcloud config get-value project", shell=True, universal_newlines=True ).strip() + extra_metadata = { + "title": title, + "license": license, + "license_url": license_url, + "source": source, + "source_url": source_url, + "about": about, + "about_url": about_url, + } + + environment_variables = {} + if plugin_secret: + extra_metadata["plugins"] = {} + for plugin_name, plugin_setting, setting_value in plugin_secret: + environment_variable = ( + "{}_{}".format(plugin_name, plugin_setting) + .upper() + .replace("-", "_") + ) + environment_variables[environment_variable] = setting_value + extra_metadata["plugins"].setdefault(plugin_name, {})[ + plugin_setting + ] = {"$env": environment_variable} + with temporary_docker_directory( files, name, @@ -65,16 +96,17 @@ def publish_subcommand(publish): install, spatialite, version_note, - { - "title": title, - "license": license, - "license_url": license_url, - "source": source, - "source_url": source_url, - "about": about, - "about_url": about_url, - }, + extra_metadata, + environment_variables, ): + if show_files: + if os.path.exists("metadata.json"): + print("=== metadata.json ===\n") + print(open("metadata.json").read()) + print("\n==== Dockerfile ====\n") + print(open("Dockerfile").read()) + print("\n====================\n") + image_id = "gcr.io/{project}/{name}".format(project=project, name=name) check_call("gcloud builds submit --tag {}".format(image_id), shell=True) check_call( diff --git a/docs/datasette-publish-cloudrun-help.txt b/docs/datasette-publish-cloudrun-help.txt index 19e3c799..6cdc87eb 100644 --- a/docs/datasette-publish-cloudrun-help.txt +++ b/docs/datasette-publish-cloudrun-help.txt @@ -24,4 +24,5 @@ Options: -n, --name TEXT Application name to use when building --service TEXT Cloud Run service to deploy (or over-write) --spatialite Enable SpatialLite extension + --show-files Output the generated Dockerfile and metadata.json --help Show this message and exit. diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index d26786ce..1e9bb830 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -1,6 +1,7 @@ from click.testing import CliRunner from datasette import cli from unittest import mock +import json @mock.patch("shutil.which") @@ -46,3 +47,56 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which): ), ] ) + + +@mock.patch("shutil.which") +@mock.patch("datasette.publish.cloudrun.check_output") +@mock.patch("datasette.publish.cloudrun.check_call") +def test_publish_cloudrun_plugin_secrets(mock_call, mock_output, mock_which): + mock_which.return_value = True + mock_output.return_value = "myproject" + + runner = CliRunner() + with runner.isolated_filesystem(): + open("test.db", "w").write("data") + result = runner.invoke( + cli.cli, + [ + "publish", + "cloudrun", + "test.db", + "--plugin-secret", + "datasette-auth-github", + "client_id", + "x-client-id", + "--show-files", + ], + ) + dockerfile = ( + result.output.split("==== Dockerfile ====\n")[1] + .split("\n====================\n")[0] + .strip() + ) + expected = """FROM python:3.6 +COPY . /app +WORKDIR /app + +ENV DATASETTE_AUTH_GITHUB_CLIENT_ID 'x-client-id' +RUN pip install -U datasette +RUN datasette inspect test.db --inspect-file inspect-data.json +ENV PORT 8001 +EXPOSE 8001 +CMD datasette serve --host 0.0.0.0 -i test.db --cors --inspect-file inspect-data.json --metadata metadata.json --port $PORT""".strip() + assert expected == dockerfile + metadata = ( + result.output.split("=== metadata.json ===\n")[1] + .split("\n==== Dockerfile ====\n")[0] + .strip() + ) + assert { + "plugins": { + "datasette-auth-github": { + "client_id": {"$env": "DATASETTE_AUTH_GITHUB_CLIENT_ID"} + } + } + } == json.loads(metadata) From ccf80604e931fba1893b5bab11de386fed82009e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Jul 2019 18:55:55 -0700 Subject: [PATCH 10/10] Documentation for --plugin-secret option --- docs/plugins.rst | 11 ++++++++++- docs/publish.rst | 9 +++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/plugins.rst b/docs/plugins.rst index 3b3653cc..0b4dcced 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -219,6 +219,8 @@ Here is an example of some plugin configuration for a specific table:: This tells the ``datasette-cluster-map`` column which latitude and longitude columns should be used for a table called ``Street_Tree_List`` inside a database file called ``sf-trees.db``. +.. _plugins_configuration_secret: + Secret configuration values ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -236,7 +238,6 @@ Any values embedded in ``metadata.json`` will be visible to anyone who views the } } - **As values in separate files**. Your secrets can also live in files on disk. To specify a secret should be read from a file, provide the full file path like this:: { @@ -249,6 +250,14 @@ Any values embedded in ``metadata.json`` will be visible to anyone who views the } } +If you are publishing your data using the :ref:`datasette publish ` family of commands, you can use the ``--plugin-secret`` option to set these secrets at publish time. For example, using Heroku you might run the following command:: + + $ datasette publish heroku my_database.db \ + --name my-heroku-app-demo \ + --install=datasette-auth-github \ + --plugin-secret datasette-auth-github client_id your_client_id \ + --plugin-secret datasette-auth-github client_secret your_client_secret + Writing plugins that accept configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/publish.rst b/docs/publish.rst index c9039734..009ae199 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -6,6 +6,8 @@ Datasette includes tools for publishing and deploying your data to the internet. The ``datasette publish`` command will deploy a new Datasette instance containing your databases directly to a Heroku, Google Cloud or Zeit Now hosting account. You can also use ``datasette package`` to create a Docker image that bundles your databases together with the datasette application that is used to serve them. +.. _cli_publish: + datasette publish ================= @@ -99,6 +101,13 @@ You can also specify plugins you would like to install. For example, if you want datasette publish nowv1 mydatabase.db --install=datasette-vega +If a plugin has any :ref:`plugins_configuration_secret` you can use the ``--plugin-secret`` option to set those secrets at publish time. For example, using Heroku with `datasette-auth-github `__ you might run the following command:: + + $ datasette publish heroku my_database.db \ + --name my-heroku-app-demo \ + --install=datasette-auth-github \ + --plugin-secret datasette-auth-github client_id your_client_id \ + --plugin-secret datasette-auth-github client_secret your_client_secret datasette package =================