remote/datasette
1
0
Fork 0
mirror of https://github.com/simonw/datasette.git synced 2026-06-10 02:57:01 +02:00
Code Issues Projects Releases Packages Wiki Activity

publish_subcommand hook + default plugins mechanism, used for publish heroku/now (#349)

Browse source 
This change introduces a new plugin hook, publish_subcommand, which can be
used to implement new subcommands for the "datasette publish" command family.

I've used this new hook to refactor out the "publish now" and "publish heroku"
implementations into separate modules. I've also added unit tests for these
two publishers, mocking the subprocess.call and subprocess.check_output
functions.

As part of this, I introduced a mechanism for loading default plugins. These
are defined in the new "default_plugins" list inside datasette/app.py

Closes #217 (Plugin support for datasette publish)
Closes #348 (Unit tests for "datasette publish")
Refs #14, #59, #102, #103, #146, #236, #347
This commit is contained in:
Simon Willison 2018-07-25 22:15:59 -07:00 • committed by GitHub
commit dbbe707841
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 343 additions and 239 deletions
Download patch file Download diff file Expand all files Collapse all files

11
datasette/app.py
View file

@ -2,6 +2,7 @@ import asyncio
import click
import collections
import hashlib
import importlib
import itertools
import os
import sqlite3
@ -41,6 +42,11 @@ from .utils import (
from .inspect import inspect_hash, inspect_views, inspect_tables
from .version import __version__
default_plugins = (
"datasette.publish.heroku",
"datasette.publish.now",
)
app_root = Path(__file__).parent.parent
connections = threading.local()
@ -49,6 +55,11 @@ pm = pluggy.PluginManager("datasette")
pm.add_hookspecs(hookspecs)
pm.load_setuptools_entrypoints("datasette")
# Load default plugins
for plugin in default_plugins:
mod = importlib.import_module(plugin)
pm.register(mod, plugin)
ConfigOption = collections.namedtuple(
"ConfigOption", ("name", "default", "help")

221
datasette/cli.py
View file

@ -4,32 +4,17 @@ from click_default_group import DefaultGroup
import json
import os
import shutil
from subprocess import call, check_output
from subprocess import call
import sys
from .app import Datasette, DEFAULT_CONFIG, CONFIG_OPTIONS
from .app import Datasette, DEFAULT_CONFIG, CONFIG_OPTIONS, pm
from .utils import (
temporary_docker_directory,
temporary_heroku_directory,
value_as_boolean,
StaticMount,
ValueAsBooleanError,
)
class StaticMount(click.ParamType):
name = "static mount"
def convert(self, value, param, ctx):
if ":" not in value:
self.fail(
'"{}" should be of format mountpoint:directory'.format(value),
param, ctx
)
path, dirpath = value.split(":")
if not os.path.exists(dirpath) or not os.path.isdir(dirpath):
self.fail("%s is not a valid directory path" % value, param, ctx)
return path, dirpath
class Config(click.ParamType):
name = "config"
@ -93,202 +78,14 @@ def inspect(files, inspect_file, sqlite_extensions):
open(inspect_file, "w").write(json.dumps(app.inspect(), indent=2))
@cli.command()
@click.argument("publisher", type=click.Choice(["now", "heroku"]))
@click.argument("files", type=click.Path(exists=True), nargs=-1)
@click.option(
"-n",
"--name",
default="datasette",
help="Application name to use when deploying",
)
@click.option(
"-m",
"--metadata",
type=click.File(mode="r"),
help="Path to JSON file containing metadata to publish",
)
@click.option("--extra-options", help="Extra options to pass to datasette serve")
@click.option("--force", is_flag=True, help="Pass --force option to now")
@click.option("--branch", help="Install datasette from a GitHub branch e.g. master")
@click.option("--token", help="Auth token to use for deploy (Now only)")
@click.option(
"--template-dir",
type=click.Path(exists=True, file_okay=False, dir_okay=True),
help="Path to directory containing custom templates",
)
@click.option(
"--plugins-dir",
type=click.Path(exists=True, file_okay=False, dir_okay=True),
help="Path to directory containing custom plugins",
)
@click.option(
"--static",
type=StaticMount(),
help="mountpoint:path-to-directory for serving static files",
multiple=True,
)
@click.option(
"--install",
help="Additional packages (e.g. plugins) to install",
multiple=True,
)
@click.option(
"--spatialite", is_flag=True, help="Enable SpatialLite extension"
)
@click.option("--version-note", help="Additional note to show on /-/versions")
@click.option("--title", help="Title for metadata")
@click.option("--license", help="License label for metadata")
@click.option("--license_url", help="License URL for metadata")
@click.option("--source", help="Source label for metadata")
@click.option("--source_url", help="Source URL for metadata")
def publish(
publisher,
files,
name,
metadata,
extra_options,
force,
branch,
token,
template_dir,
plugins_dir,
static,
install,
spatialite,
version_note,
**extra_metadata
):
"""
Publish specified SQLite database files to the internet along with a datasette API.
@cli.group()
def publish():
"Publish specified SQLite database files to the internet along with a Datasette-powered interface and API"
pass
Options for PUBLISHER:
* 'now' - You must have Zeit Now installed: https://zeit.co/now
* 'heroku' - You must have Heroku installed: https://cli.heroku.com/
Example usage: datasette publish now my-database.db
"""
def _fail_if_publish_binary_not_installed(binary, publish_target, install_link):
"""Exit (with error message) if ``binary` isn't installed"""
if not shutil.which(binary):
click.secho(
"Publishing to {publish_target} requires {binary} to be installed and configured".format(
publish_target=publish_target, binary=binary
),
bg="red",
fg="white",
bold=True,
err=True,
)
click.echo(
"Follow the instructions at {install_link}".format(
install_link=install_link
),
err=True,
)
sys.exit(1)
if publisher == "now":
_fail_if_publish_binary_not_installed("now", "Zeit Now", "https://zeit.co/now")
if extra_options:
extra_options += " "
else:
extra_options = ""
extra_options += "--config force_https_urls:on"
with temporary_docker_directory(
files,
name,
metadata,
extra_options,
branch,
template_dir,
plugins_dir,
static,
install,
spatialite,
version_note,
extra_metadata,
):
args = []
if force:
args.append("--force")
if token:
args.append("--token={}".format(token))
if args:
call(["now"] + args)
else:
call("now")
elif publisher == "heroku":
_fail_if_publish_binary_not_installed(
"heroku", "Heroku", "https://cli.heroku.com"
)
if spatialite:
click.secho(
"The --spatialite option is not yet supported for Heroku",
bg="red",
fg="white",
bold=True,
err=True,
)
click.echo(
"See https://github.com/simonw/datasette/issues/301",
err=True,
)
sys.exit(1)
# Check for heroku-builds plugin
plugins = [
line.split()[0] for line in check_output(["heroku", "plugins"]).splitlines()
]
if b"heroku-builds" not in plugins:
click.echo(
"Publishing to Heroku requires the heroku-builds plugin to be installed."
)
click.confirm(
"Install it? (this will run `heroku plugins:install heroku-builds`)",
abort=True,
)
call(["heroku", "plugins:install", "heroku-builds"])
with temporary_heroku_directory(
files,
name,
metadata,
extra_options,
branch,
template_dir,
plugins_dir,
static,
install,
extra_metadata,
):
app_name = None
if name:
# Check to see if this app already exists
list_output = check_output(["heroku", "apps:list", "--json"]).decode('utf8')
apps = json.loads(list_output)
for app in apps:
if app['name'] == name:
app_name = name
break
if not app_name:
# Create a new app
cmd = ["heroku", "apps:create"]
if name:
cmd.append(name)
cmd.append("--json")
create_output = check_output(cmd).decode(
"utf8"
)
app_name = json.loads(create_output)["name"]
call(["heroku", "builds:create", "-a", app_name])
# Register publish plugins
pm.hook.publish_subcommand(publish=publish)
@cli.command()

5
datasette/hookspecs.py
View file

@ -23,3 +23,8 @@ def extra_css_urls():
@hookspec
def extra_js_urls():
"Extra JavaScript URLs added by this plugin"
@hookspec
def publish_subcommand(publish):
"Subcommands for 'datasette publish'"

0
datasette/publish/__init__.py Normal file
View file

68
datasette/publish/common.py Normal file
View file

@ -0,0 +1,68 @@
from ..utils import StaticMount
import click
import shutil
import sys
def add_common_publish_arguments_and_options(subcommand):
for decorator in reversed((
click.argument("files", type=click.Path(exists=True), nargs=-1),
click.option(
"-m",
"--metadata",
type=click.File(mode="r"),
help="Path to JSON file containing metadata to publish",
),
click.option("--extra-options", help="Extra options to pass to datasette serve"),
click.option("--branch", help="Install datasette from a GitHub branch e.g. master"),
click.option(
"--template-dir",
type=click.Path(exists=True, file_okay=False, dir_okay=True),
help="Path to directory containing custom templates",
),
click.option(
"--plugins-dir",
type=click.Path(exists=True, file_okay=False, dir_okay=True),
help="Path to directory containing custom plugins",
),
click.option(
"--static",
type=StaticMount(),
help="mountpoint:path-to-directory for serving static files",
multiple=True,
),
click.option(
"--install",
help="Additional packages (e.g. plugins) to install",
multiple=True,
),
click.option("--version-note", help="Additional note to show on /-/versions"),
click.option("--title", help="Title for metadata"),
click.option("--license", help="License label for metadata"),
click.option("--license_url", help="License URL for metadata"),
click.option("--source", help="Source label for metadata"),
click.option("--source_url", help="Source URL for metadata"),
)):
subcommand = decorator(subcommand)
return subcommand
def fail_if_publish_binary_not_installed(binary, publish_target, install_link):
"""Exit (with error message) if ``binary` isn't installed"""
if not shutil.which(binary):
click.secho(
"Publishing to {publish_target} requires {binary} to be installed and configured".format(
publish_target=publish_target, binary=binary
),
bg="red",
fg="white",
bold=True,
err=True,
)
click.echo(
"Follow the instructions at {install_link}".format(
install_link=install_link
),
err=True,
)
sys.exit(1)

99
datasette/publish/heroku.py Normal file
View file

@ -0,0 +1,99 @@
from datasette import hookimpl
import click
import json
from subprocess import call, check_output
from .common import (
add_common_publish_arguments_and_options,
fail_if_publish_binary_not_installed,
)
from ..utils import temporary_heroku_directory
@hookimpl
def publish_subcommand(publish):
@publish.command()
@add_common_publish_arguments_and_options
@click.option(
"-n",
"--name",
default="datasette",
help="Application name to use when deploying",
)
def heroku(
files,
metadata,
extra_options,
branch,
template_dir,
plugins_dir,
static,
install,
version_note,
title,
license,
license_url,
source,
source_url,
name,
):
fail_if_publish_binary_not_installed(
"heroku", "Heroku", "https://cli.heroku.com"
)
# Check for heroku-builds plugin
plugins = [
line.split()[0] for line in check_output(["heroku", "plugins"]).splitlines()
]
if b"heroku-builds" not in plugins:
click.echo(
"Publishing to Heroku requires the heroku-builds plugin to be installed."
)
click.confirm(
"Install it? (this will run `heroku plugins:install heroku-builds`)",
abort=True,
)
call(["heroku", "plugins:install", "heroku-builds"])
with temporary_heroku_directory(
files,
name,
metadata,
extra_options,
branch,
template_dir,
plugins_dir,
static,
install,
version_note,
{
"title": title,
"license": license,
"license_url": license_url,
"source": source,
"source_url": source_url,
},
):
app_name = None
if name:
# Check to see if this app already exists
list_output = check_output(["heroku", "apps:list", "--json"]).decode(
"utf8"
)
apps = json.loads(list_output)
for app in apps:
if app["name"] == name:
app_name = name
break
if not app_name:
# Create a new app
cmd = ["heroku", "apps:create"]
if name:
cmd.append(name)
cmd.append("--json")
create_output = check_output(cmd).decode("utf8")
app_name = json.loads(create_output)["name"]
call(["heroku", "builds:create", "-a", app_name])

80
datasette/publish/now.py Normal file
View file

@ -0,0 +1,80 @@
from datasette import hookimpl
import click
from subprocess import call
from .common import (
add_common_publish_arguments_and_options,
fail_if_publish_binary_not_installed,
)
from ..utils import temporary_docker_directory
@hookimpl
def publish_subcommand(publish):
@publish.command()
@add_common_publish_arguments_and_options
@click.option(
"-n",
"--name",
default="datasette",
help="Application name to use when deploying",
)
@click.option("--force", is_flag=True, help="Pass --force option to now")
@click.option("--token", help="Auth token to use for deploy (Now only)")
@click.option("--spatialite", is_flag=True, help="Enable SpatialLite extension")
def now(
files,
metadata,
extra_options,
branch,
template_dir,
plugins_dir,
static,
install,
version_note,
title,
license,
license_url,
source,
source_url,
name,
force,
token,
spatialite,
):
fail_if_publish_binary_not_installed("now", "Zeit Now", "https://zeit.co/now")
if extra_options:
extra_options += " "
else:
extra_options = ""
extra_options += "--config force_https_urls:on"
with temporary_docker_directory(
files,
name,
metadata,
extra_options,
branch,
template_dir,
plugins_dir,
static,
install,
spatialite,
version_note,
{
"title": title,
"license": license,
"license_url": license_url,
"source": source,
"source_url": source_url,
},
):
args = []
if force:
args.append("--force")
if token:
args.append("--token={}".format(token))
if args:
call(["now"] + args)
else:
call("now")

20
datasette/utils.py
View file

@ -1,6 +1,7 @@
from contextlib import contextmanager
from collections import OrderedDict
import base64
import click
import hashlib
import imp
import json
@ -376,6 +377,7 @@ def temporary_heroku_directory(
plugins_dir,
static,
install,
version_note,
extra_metadata=None
):
# FIXME: lots of duplicated code from above
@ -430,7 +432,8 @@ def temporary_heroku_directory(
os.path.join(tmp.name, 'plugins')
)
extras.extend(['--plugins-dir', 'plugins/'])
if version_note:
extras.extend(['--version-note', version_note])
if metadata:
extras.extend(['--metadata', 'metadata.json'])
if extra_options:
@ -876,3 +879,18 @@ def remove_infinites(row):
for c in row
]
return row
class StaticMount(click.ParamType):
name = "static mount"
def convert(self, value, param, ctx):
if ":" not in value:
self.fail(
'"{}" should be of format mountpoint:directory'.format(value),
param, ctx
)
path, dirpath = value.split(":")
if not os.path.exists(dirpath) or not os.path.isdir(dirpath):
self.fail("%s is not a valid directory path" % value, param, ctx)
return path, dirpath