github/datasette
1
0
Fork 0
mirror of https://github.com/simonw/datasette.git synced 2025-12-10 16:51:24 +01:00
Code Issues Projects Releases Packages Wiki Activity

--plugin-secret option for datasette publish

Browse source 
Closes #543

Also added new --show-files option to publish now and publish cloudrun - handy for debugging.
This commit is contained in:
Simon Willison 2019-07-07 19:06:31 -07:00 • committed by GitHub
commit 973f8f139d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 381 additions and 86 deletions
Download patch file Download diff file Expand all files Collapse all files

51
datasette/publish/cloudrun.py
View file

@ -1,6 +1,7 @@
from datasette import hookimpl
import click
import json
import os
from subprocess import check_call, check_output
from .common import (
@ -24,6 +25,11 @@ def publish_subcommand(publish):
"--service", default="", help="Cloud Run service to deploy (or over-write)"
)
@click.option("--spatialite", is_flag=True, help="Enable SpatialLite extension")
@click.option(
"--show-files",
is_flag=True,
help="Output the generated Dockerfile and metadata.json",
)
def cloudrun(
files,
metadata,
@ -33,6 +39,7 @@ def publish_subcommand(publish):
plugins_dir,
static,
install,
plugin_secret,
version_note,
title,
license,
@ -44,6 +51,7 @@ def publish_subcommand(publish):
name,
service,
spatialite,
show_files,
):
fail_if_publish_binary_not_installed(
"gcloud", "Google Cloud", "https://cloud.google.com/sdk/"
@ -52,6 +60,30 @@ def publish_subcommand(publish):
"gcloud config get-value project", shell=True, universal_newlines=True
).strip()
extra_metadata = {
"title": title,
"license": license,
"license_url": license_url,
"source": source,
"source_url": source_url,
"about": about,
"about_url": about_url,
}
environment_variables = {}
if plugin_secret:
extra_metadata["plugins"] = {}
for plugin_name, plugin_setting, setting_value in plugin_secret:
environment_variable = (
"{}_{}".format(plugin_name, plugin_setting)
.upper()
.replace("-", "_")
)
environment_variables[environment_variable] = setting_value
extra_metadata["plugins"].setdefault(plugin_name, {})[
plugin_setting
] = {"$env": environment_variable}
with temporary_docker_directory(
files,
name,
@ -64,16 +96,17 @@ def publish_subcommand(publish):
install,
spatialite,
version_note,
{
"title": title,
"license": license,
"license_url": license_url,
"source": source,
"source_url": source_url,
"about": about,
"about_url": about_url,
},
extra_metadata,
environment_variables,
):
if show_files:
if os.path.exists("metadata.json"):
print("=== metadata.json ===\n")
print(open("metadata.json").read())
print("\n==== Dockerfile ====\n")
print(open("Dockerfile").read())
print("\n====================\n")
image_id = "gcr.io/{project}/{name}".format(project=project, name=name)
check_call("gcloud builds submit --tag {}".format(image_id), shell=True)
check_call(

15
datasette/publish/common.py
View file

@ -41,6 +41,14 @@ def add_common_publish_arguments_and_options(subcommand):
help="Additional packages (e.g. plugins) to install",
multiple=True,
),
click.option(
"--plugin-secret",
nargs=3,
type=(str, str, str),
callback=validate_plugin_secret,
multiple=True,
help="Secrets to pass to plugins, e.g. --plugin-secret datasette-auth-github client_id xxx",
),
click.option(
"--version-note", help="Additional note to show on /-/versions"
),
@ -76,3 +84,10 @@ def fail_if_publish_binary_not_installed(binary, publish_target, install_link):
err=True,
)
sys.exit(1)
def validate_plugin_secret(ctx, param, value):
for plugin_name, plugin_setting, setting_value in value:
if "'" in setting_value:
raise click.BadParameter("--plugin-secret cannot contain single quotes")
return value

40
datasette/publish/heroku.py
View file

@ -33,6 +33,7 @@ def publish_subcommand(publish):
plugins_dir,
static,
install,
plugin_secret,
version_note,
title,
license,
@ -61,6 +62,30 @@ def publish_subcommand(publish):
)
call(["heroku", "plugins:install", "heroku-builds"])
extra_metadata = {
"title": title,
"license": license,
"license_url": license_url,
"source": source,
"source_url": source_url,
"about": about,
"about_url": about_url,
}
environment_variables = {}
if plugin_secret:
extra_metadata["plugins"] = {}
for plugin_name, plugin_setting, setting_value in plugin_secret:
environment_variable = (
"{}_{}".format(plugin_name, plugin_setting)
.upper()
.replace("-", "_")
)
environment_variables[environment_variable] = setting_value
extra_metadata["plugins"].setdefault(plugin_name, {})[
plugin_setting
] = {"$env": environment_variable}
with temporary_heroku_directory(
files,
name,
@ -72,15 +97,7 @@ def publish_subcommand(publish):
static,
install,
version_note,
{
"title": title,
"license": license,
"license_url": license_url,
"source": source,
"source_url": source_url,
"about": about,
"about_url": about_url,
},
extra_metadata,
):
app_name = None
if name:
@ -104,6 +121,11 @@ def publish_subcommand(publish):
create_output = check_output(cmd).decode("utf8")
app_name = json.loads(create_output)["name"]
for key, value in environment_variables.items():
call(
["heroku", "config:set", "-a", app_name, "{}={}".format(key, value)]
)
call(["heroku", "builds:create", "-a", app_name, "--include-vcs-ignore"])

50
datasette/publish/now.py
View file

@ -1,6 +1,7 @@
from datasette import hookimpl
import click
import json
import os
from subprocess import run, PIPE
from .common import (
@ -24,6 +25,11 @@ def publish_subcommand(publish):
@click.option("--token", help="Auth token to use for deploy")
@click.option("--alias", multiple=True, help="Desired alias e.g. yoursite.now.sh")
@click.option("--spatialite", is_flag=True, help="Enable SpatialLite extension")
@click.option(
"--show-files",
is_flag=True,
help="Output the generated Dockerfile and metadata.json",
)
def nowv1(
files,
metadata,
@ -33,6 +39,7 @@ def publish_subcommand(publish):
plugins_dir,
static,
install,
plugin_secret,
version_note,
title,
license,
@ -46,6 +53,7 @@ def publish_subcommand(publish):
token,
alias,
spatialite,
show_files,
):
fail_if_publish_binary_not_installed("now", "Zeit Now", "https://zeit.co/now")
if extra_options:
@ -54,6 +62,30 @@ def publish_subcommand(publish):
extra_options = ""
extra_options += "--config force_https_urls:on"
extra_metadata = {
"title": title,
"license": license,
"license_url": license_url,
"source": source,
"source_url": source_url,
"about": about,
"about_url": about_url,
}
environment_variables = {}
if plugin_secret:
extra_metadata["plugins"] = {}
for plugin_name, plugin_setting, setting_value in plugin_secret:
environment_variable = (
"{}_{}".format(plugin_name, plugin_setting)
.upper()
.replace("-", "_")
)
environment_variables[environment_variable] = setting_value
extra_metadata["plugins"].setdefault(plugin_name, {})[
plugin_setting
] = {"$env": environment_variable}
with temporary_docker_directory(
files,
name,
@ -66,15 +98,8 @@ def publish_subcommand(publish):
install,
spatialite,
version_note,
{
"title": title,
"license": license,
"license_url": license_url,
"source": source,
"source_url": source_url,
"about": about,
"about_url": about_url,
},
extra_metadata,
environment_variables,
):
now_json = {"version": 1}
open("now.json", "w").write(json.dumps(now_json, indent=4))
@ -88,6 +113,13 @@ def publish_subcommand(publish):
else:
done = run("now", stdout=PIPE)
deployment_url = done.stdout
if show_files:
if os.path.exists("metadata.json"):
print("=== metadata.json ===\n")
print(open("metadata.json").read())
print("\n==== Dockerfile ====\n")
print(open("Dockerfile").read())
print("\n====================\n")
if alias:
# I couldn't get --target=production working, so I call
# 'now alias' with arguments directly instead - but that

10
datasette/utils/__init__.py
View file

@ -272,6 +272,7 @@ def make_dockerfile(
install,
spatialite,
version_note,
environment_variables=None,
):
cmd = ["datasette", "serve", "--host", "0.0.0.0"]
for filename in files:
@ -307,11 +308,18 @@ FROM python:3.6
COPY . /app
WORKDIR /app
{spatialite_extras}
{environment_variables}
RUN pip install -U {install_from}
RUN datasette inspect {files} --inspect-file inspect-data.json
ENV PORT 8001
EXPOSE 8001
CMD {cmd}""".format(
environment_variables="\n".join(
[
"ENV {} '{}'".format(key, value)
for key, value in (environment_variables or {}).items()
]
),
files=" ".join(files),
cmd=cmd,
install_from=" ".join(install),
@ -333,6 +341,7 @@ def temporary_docker_directory(
spatialite,
version_note,
extra_metadata=None,
environment_variables=None,
):
extra_metadata = extra_metadata or {}
tmp = tempfile.TemporaryDirectory()
@ -361,6 +370,7 @@ def temporary_docker_directory(
install,
spatialite,
version_note,
environment_variables,
)
os.chdir(datasette_dir)
if metadata_content:

42
docs/datasette-publish-cloudrun-help.txt
View file

@ -3,22 +3,26 @@ $ datasette publish cloudrun --help
Usage: datasette publish cloudrun [OPTIONS] [FILES]...
Options:
-m, --metadata FILENAME Path to JSON file containing metadata to publish
--extra-options TEXT Extra options to pass to datasette serve
--branch TEXT Install datasette from a GitHub branch e.g. master
--template-dir DIRECTORY Path to directory containing custom templates
--plugins-dir DIRECTORY Path to directory containing custom plugins
--static STATIC MOUNT mountpoint:path-to-directory for serving static files
--install TEXT Additional packages (e.g. plugins) to install
--version-note TEXT Additional note to show on /-/versions
--title TEXT Title for metadata
--license TEXT License label for metadata
--license_url TEXT License URL for metadata
--source TEXT Source label for metadata
--source_url TEXT Source URL for metadata
--about TEXT About label for metadata
--about_url TEXT About URL for metadata
-n, --name TEXT Application name to use when building
--service TEXT Cloud Run service to deploy (or over-write)
--spatialite Enable SpatialLite extension
--help Show this message and exit.
-m, --metadata FILENAME Path to JSON file containing metadata to publish
--extra-options TEXT Extra options to pass to datasette serve
--branch TEXT Install datasette from a GitHub branch e.g. master
--template-dir DIRECTORY Path to directory containing custom templates
--plugins-dir DIRECTORY Path to directory containing custom plugins
--static STATIC MOUNT mountpoint:path-to-directory for serving static files
--install TEXT Additional packages (e.g. plugins) to install
--plugin-secret <TEXT TEXT TEXT>...
Secrets to pass to plugins, e.g. --plugin-secret
datasette-auth-github client_id xxx
--version-note TEXT Additional note to show on /-/versions
--title TEXT Title for metadata
--license TEXT License label for metadata
--license_url TEXT License URL for metadata
--source TEXT Source label for metadata
--source_url TEXT Source URL for metadata
--about TEXT About label for metadata
--about_url TEXT About URL for metadata
-n, --name TEXT Application name to use when building
--service TEXT Cloud Run service to deploy (or over-write)
--spatialite Enable SpatialLite extension
--show-files Output the generated Dockerfile and metadata.json
--help Show this message and exit.

37
docs/datasette-publish-heroku-help.txt
View file

@ -3,20 +3,23 @@ $ datasette publish heroku --help
Usage: datasette publish heroku [OPTIONS] [FILES]...
Options:
-m, --metadata FILENAME Path to JSON file containing metadata to publish
--extra-options TEXT Extra options to pass to datasette serve
--branch TEXT Install datasette from a GitHub branch e.g. master
--template-dir DIRECTORY Path to directory containing custom templates
--plugins-dir DIRECTORY Path to directory containing custom plugins
--static STATIC MOUNT mountpoint:path-to-directory for serving static files
--install TEXT Additional packages (e.g. plugins) to install
--version-note TEXT Additional note to show on /-/versions
--title TEXT Title for metadata
--license TEXT License label for metadata
--license_url TEXT License URL for metadata
--source TEXT Source label for metadata
--source_url TEXT Source URL for metadata
--about TEXT About label for metadata
--about_url TEXT About URL for metadata
-n, --name TEXT Application name to use when deploying
--help Show this message and exit.
-m, --metadata FILENAME Path to JSON file containing metadata to publish
--extra-options TEXT Extra options to pass to datasette serve
--branch TEXT Install datasette from a GitHub branch e.g. master
--template-dir DIRECTORY Path to directory containing custom templates
--plugins-dir DIRECTORY Path to directory containing custom plugins
--static STATIC MOUNT mountpoint:path-to-directory for serving static files
--install TEXT Additional packages (e.g. plugins) to install
--plugin-secret <TEXT TEXT TEXT>...
Secrets to pass to plugins, e.g. --plugin-secret
datasette-auth-github client_id xxx
--version-note TEXT Additional note to show on /-/versions
--title TEXT Title for metadata
--license TEXT License label for metadata
--license_url TEXT License URL for metadata
--source TEXT Source label for metadata
--source_url TEXT Source URL for metadata
--about TEXT About label for metadata
--about_url TEXT About URL for metadata
-n, --name TEXT Application name to use when deploying
--help Show this message and exit.

46
docs/datasette-publish-nowv1-help.txt
View file

@ -3,24 +3,28 @@ $ datasette publish nowv1 --help
Usage: datasette publish nowv1 [OPTIONS] [FILES]...
Options:
-m, --metadata FILENAME Path to JSON file containing metadata to publish
--extra-options TEXT Extra options to pass to datasette serve
--branch TEXT Install datasette from a GitHub branch e.g. master
--template-dir DIRECTORY Path to directory containing custom templates
--plugins-dir DIRECTORY Path to directory containing custom plugins
--static STATIC MOUNT mountpoint:path-to-directory for serving static files
--install TEXT Additional packages (e.g. plugins) to install
--version-note TEXT Additional note to show on /-/versions
--title TEXT Title for metadata
--license TEXT License label for metadata
--license_url TEXT License URL for metadata
--source TEXT Source label for metadata
--source_url TEXT Source URL for metadata
--about TEXT About label for metadata
--about_url TEXT About URL for metadata
-n, --name TEXT Application name to use when deploying
--force Pass --force option to now
--token TEXT Auth token to use for deploy
--alias TEXT Desired alias e.g. yoursite.now.sh
--spatialite Enable SpatialLite extension
--help Show this message and exit.
-m, --metadata FILENAME Path to JSON file containing metadata to publish
--extra-options TEXT Extra options to pass to datasette serve
--branch TEXT Install datasette from a GitHub branch e.g. master
--template-dir DIRECTORY Path to directory containing custom templates
--plugins-dir DIRECTORY Path to directory containing custom plugins
--static STATIC MOUNT mountpoint:path-to-directory for serving static files
--install TEXT Additional packages (e.g. plugins) to install
--plugin-secret <TEXT TEXT TEXT>...
Secrets to pass to plugins, e.g. --plugin-secret
datasette-auth-github client_id xxx
--version-note TEXT Additional note to show on /-/versions
--title TEXT Title for metadata
--license TEXT License label for metadata
--license_url TEXT License URL for metadata
--source TEXT Source label for metadata
--source_url TEXT Source URL for metadata
--about TEXT About label for metadata
--about_url TEXT About URL for metadata
-n, --name TEXT Application name to use when deploying
--force Pass --force option to now
--token TEXT Auth token to use for deploy
--alias TEXT Desired alias e.g. yoursite.now.sh
--spatialite Enable SpatialLite extension
--show-files Output the generated Dockerfile and metadata.json
--help Show this message and exit.

11
docs/plugins.rst
View file

@ -219,6 +219,8 @@ Here is an example of some plugin configuration for a specific table::
This tells the ``datasette-cluster-map`` column which latitude and longitude columns should be used for a table called ``Street_Tree_List`` inside a database file called ``sf-trees.db``.
.. _plugins_configuration_secret:
Secret configuration values
~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -236,7 +238,6 @@ Any values embedded in ``metadata.json`` will be visible to anyone who views the
}
}
**As values in separate files**. Your secrets can also live in files on disk. To specify a secret should be read from a file, provide the full file path like this::
{
@ -249,6 +250,14 @@ Any values embedded in ``metadata.json`` will be visible to anyone who views the
}
}
If you are publishing your data using the :ref:`datasette publish <cli_publish>` family of commands, you can use the ``--plugin-secret`` option to set these secrets at publish time. For example, using Heroku you might run the following command::
$ datasette publish heroku my_database.db \
--name my-heroku-app-demo \
--install=datasette-auth-github \
--plugin-secret datasette-auth-github client_id your_client_id \
--plugin-secret datasette-auth-github client_secret your_client_secret
Writing plugins that accept configuration
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

9
docs/publish.rst
View file

@ -6,6 +6,8 @@
Datasette includes tools for publishing and deploying your data to the internet. The ``datasette publish`` command will deploy a new Datasette instance containing your databases directly to a Heroku, Google Cloud or Zeit Now hosting account. You can also use ``datasette package`` to create a Docker image that bundles your databases together with the datasette application that is used to serve them.
.. _cli_publish:
datasette publish
=================
@ -99,6 +101,13 @@ You can also specify plugins you would like to install. For example, if you want
datasette publish nowv1 mydatabase.db --install=datasette-vega
If a plugin has any :ref:`plugins_configuration_secret` you can use the ``--plugin-secret`` option to set those secrets at publish time. For example, using Heroku with `datasette-auth-github <https://github.com/simonw/datasette-auth-github>`__ you might run the following command::
$ datasette publish heroku my_database.db \
--name my-heroku-app-demo \
--install=datasette-auth-github \
--plugin-secret datasette-auth-github client_id your_client_id \
--plugin-secret datasette-auth-github client_secret your_client_secret
datasette package
=================

54
tests/test_publish_cloudrun.py
View file

@ -1,6 +1,7 @@
from click.testing import CliRunner
from datasette import cli
from unittest import mock
import json
@mock.patch("shutil.which")
@ -46,3 +47,56 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which):
),
]
)
@mock.patch("shutil.which")
@mock.patch("datasette.publish.cloudrun.check_output")
@mock.patch("datasette.publish.cloudrun.check_call")
def test_publish_cloudrun_plugin_secrets(mock_call, mock_output, mock_which):
mock_which.return_value = True
mock_output.return_value = "myproject"
runner = CliRunner()
with runner.isolated_filesystem():
open("test.db", "w").write("data")
result = runner.invoke(
cli.cli,
[
"publish",
"cloudrun",
"test.db",
"--plugin-secret",
"datasette-auth-github",
"client_id",
"x-client-id",
"--show-files",
],
)
dockerfile = (
result.output.split("==== Dockerfile ====\n")[1]
.split("\n====================\n")[0]
.strip()
)
expected = """FROM python:3.6
COPY . /app
WORKDIR /app
ENV DATASETTE_AUTH_GITHUB_CLIENT_ID 'x-client-id'
RUN pip install -U datasette
RUN datasette inspect test.db --inspect-file inspect-data.json
ENV PORT 8001
EXPOSE 8001
CMD datasette serve --host 0.0.0.0 -i test.db --cors --inspect-file inspect-data.json --metadata metadata.json --port $PORT""".strip()
assert expected == dockerfile
metadata = (
result.output.split("=== metadata.json ===\n")[1]
.split("\n==== Dockerfile ====\n")[0]
.strip()
)
assert {
"plugins": {
"datasette-auth-github": {
"client_id": {"$env": "DATASETTE_AUTH_GITHUB_CLIENT_ID"}
}
}
} == json.loads(metadata)

46
tests/test_publish_heroku.py
View file

@ -46,7 +46,7 @@ def test_publish_heroku_invalid_database(mock_which):
@mock.patch("datasette.publish.heroku.check_output")
@mock.patch("datasette.publish.heroku.call")
def test_publish_heroku(mock_call, mock_check_output, mock_which):
mock_which.return_varue = True
mock_which.return_value = True
mock_check_output.side_effect = lambda s: {
"['heroku', 'plugins']": b"heroku-builds",
"['heroku', 'apps:list', '--json']": b"[]",
@ -60,3 +60,47 @@ def test_publish_heroku(mock_call, mock_check_output, mock_which):
mock_call.assert_called_once_with(
["heroku", "builds:create", "-a", "f", "--include-vcs-ignore"]
)
@mock.patch("shutil.which")
@mock.patch("datasette.publish.heroku.check_output")
@mock.patch("datasette.publish.heroku.call")
def test_publish_heroku_plugin_secrets(mock_call, mock_check_output, mock_which):
mock_which.return_value = True
mock_check_output.side_effect = lambda s: {
"['heroku', 'plugins']": b"heroku-builds",
"['heroku', 'apps:list', '--json']": b"[]",
"['heroku', 'apps:create', 'datasette', '--json']": b'{"name": "f"}',
}[repr(s)]
runner = CliRunner()
with runner.isolated_filesystem():
open("test.db", "w").write("data")
result = runner.invoke(
cli.cli,
[
"publish",
"heroku",
"test.db",
"--plugin-secret",
"datasette-auth-github",
"client_id",
"x-client-id",
],
)
assert 0 == result.exit_code, result.output
mock_call.assert_has_calls(
[
mock.call(
[
"heroku",
"config:set",
"-a",
"f",
"DATASETTE_AUTH_GITHUB_CLIENT_ID=x-client-id",
]
),
mock.call(
["heroku", "builds:create", "-a", "f", "--include-vcs-ignore"]
),
]
)

56
tests/test_publish_now.py
View file

@ -1,6 +1,7 @@
from click.testing import CliRunner
from datasette import cli
from unittest import mock
import json
import subprocess
@ -105,3 +106,58 @@ def test_publish_now_multiple_aliases(mock_run, mock_which):
),
]
)
@mock.patch("shutil.which")
@mock.patch("datasette.publish.now.run")
def test_publish_now_plugin_secrets(mock_run, mock_which):
mock_which.return_value = True
mock_run.return_value = mock.Mock(0)
mock_run.return_value.stdout = b"https://demo.example.com/"
runner = CliRunner()
with runner.isolated_filesystem():
open("test.db", "w").write("data")
result = runner.invoke(
cli.cli,
[
"publish",
"now",
"test.db",
"--token",
"XXX",
"--plugin-secret",
"datasette-auth-github",
"client_id",
"x-client-id",
"--show-files",
],
)
dockerfile = (
result.output.split("==== Dockerfile ====\n")[1]
.split("\n====================\n")[0]
.strip()
)
expected = """FROM python:3.6
COPY . /app
WORKDIR /app
ENV DATASETTE_AUTH_GITHUB_CLIENT_ID 'x-client-id'
RUN pip install -U datasette
RUN datasette inspect test.db --inspect-file inspect-data.json
ENV PORT 8001
EXPOSE 8001
CMD datasette serve --host 0.0.0.0 -i test.db --cors --inspect-file inspect-data.json --metadata metadata.json --config force_https_urls:on --port $PORT""".strip()
assert expected == dockerfile
metadata = (
result.output.split("=== metadata.json ===\n")[1]
.split("\n==== Dockerfile ====\n")[0]
.strip()
)
assert {
"plugins": {
"datasette-auth-github": {
"client_id": {"$env": "DATASETTE_AUTH_GITHUB_CLIENT_ID"}
}
}
} == json.loads(metadata)