mirror of
https://github.com/simonw/datasette.git
synced 2025-12-10 16:51:24 +01:00
Add new --internal internal.db option, deprecate legacy _internal database
Refs: - #2157 --------- Co-authored-by: Simon Willison <swillison@gmail.com>
This commit is contained in:
Alex Garcia
GitHub
parent
d28f12092d
commit
92b8bf38c0
13 changed files with 108 additions and 90 deletions
|
|
@ -256,6 +256,7 @@ class Datasette:
|
|||
pdb=False,
|
||||
crossdb=False,
|
||||
nolock=False,
|
||||
internal=None,
|
||||
):
|
||||
self._startup_invoked = False
|
||||
assert config_dir is None or isinstance(
|
||||
|
|
@ -304,17 +305,18 @@ class Datasette:
|
|||
self.add_database(
|
||||
Database(self, is_mutable=False, is_memory=True), name="_memory"
|
||||
)
|
||||
# memory_name is a random string so that each Datasette instance gets its own
|
||||
# unique in-memory named database - otherwise unit tests can fail with weird
|
||||
# errors when different instances accidentally share an in-memory database
|
||||
self.add_database(
|
||||
Database(self, memory_name=secrets.token_hex()), name="_internal"
|
||||
)
|
||||
self.internal_db_created = False
|
||||
for file in self.files:
|
||||
self.add_database(
|
||||
Database(self, file, is_mutable=file not in self.immutables)
|
||||
)
|
||||
|
||||
self.internal_db_created = False
|
||||
if internal is None:
|
||||
self._internal_database = Database(self, memory_name=secrets.token_hex())
|
||||
else:
|
||||
self._internal_database = Database(self, path=internal, mode="rwc")
|
||||
self._internal_database.name = "__INTERNAL__"
|
||||
|
||||
self.cache_headers = cache_headers
|
||||
self.cors = cors
|
||||
config_files = []
|
||||
|
|
@ -436,15 +438,14 @@ class Datasette:
|
|||
await self._refresh_schemas()
|
||||
|
||||
async def _refresh_schemas(self):
|
||||
internal_db = self.databases["_internal"]
|
||||
internal_db = self.get_internal_database()
|
||||
if not self.internal_db_created:
|
||||
await init_internal_db(internal_db)
|
||||
self.internal_db_created = True
|
||||
|
||||
current_schema_versions = {
|
||||
row["database_name"]: row["schema_version"]
|
||||
for row in await internal_db.execute(
|
||||
"select database_name, schema_version from databases"
|
||||
"select database_name, schema_version from core_databases"
|
||||
)
|
||||
}
|
||||
for database_name, db in self.databases.items():
|
||||
|
|
@ -459,7 +460,7 @@ class Datasette:
|
|||
values = [database_name, db.is_memory, schema_version]
|
||||
await internal_db.execute_write(
|
||||
"""
|
||||
INSERT OR REPLACE INTO databases (database_name, path, is_memory, schema_version)
|
||||
INSERT OR REPLACE INTO core_databases (database_name, path, is_memory, schema_version)
|
||||
VALUES {}
|
||||
""".format(
|
||||
placeholders
|
||||
|
|
@ -554,8 +555,7 @@ class Datasette:
|
|||
raise KeyError
|
||||
return matches[0]
|
||||
if name is None:
|
||||
# Return first database that isn't "_internal"
|
||||
name = [key for key in self.databases.keys() if key != "_internal"][0]
|
||||
name = [key for key in self.databases.keys()][0]
|
||||
return self.databases[name]
|
||||
|
||||
def add_database(self, db, name=None, route=None):
|
||||
|
|
@ -655,6 +655,9 @@ class Datasette:
|
|||
def _metadata(self):
|
||||
return self.metadata()
|
||||
|
||||
def get_internal_database(self):
|
||||
return self._internal_database
|
||||
|
||||
def plugin_config(self, plugin_name, database=None, table=None, fallback=True):
|
||||
"""Return config for plugin, falling back from specified database/table"""
|
||||
plugins = self.metadata(
|
||||
|
|
@ -978,7 +981,6 @@ class Datasette:
|
|||
"hash": d.hash,
|
||||
}
|
||||
for name, d in self.databases.items()
|
||||
if name != "_internal"
|
||||
]
|
||||
|
||||
def _versions(self):
|
||||
|
|
|
|||
|
|
@ -148,9 +148,6 @@ async def inspect_(files, sqlite_extensions):
|
|||
app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions)
|
||||
data = {}
|
||||
for name, database in app.databases.items():
|
||||
if name == "_internal":
|
||||
# Don't include the in-memory _internal database
|
||||
continue
|
||||
counts = await database.table_counts(limit=3600 * 1000)
|
||||
data[name] = {
|
||||
"hash": database.hash,
|
||||
|
|
@ -476,6 +473,11 @@ def uninstall(packages, yes):
|
|||
"--ssl-certfile",
|
||||
help="SSL certificate file",
|
||||
)
|
||||
@click.option(
|
||||
"--internal",
|
||||
type=click.Path(),
|
||||
help="Path to a persistent Datasette internal SQLite database",
|
||||
)
|
||||
def serve(
|
||||
files,
|
||||
immutable,
|
||||
|
|
@ -507,6 +509,7 @@ def serve(
|
|||
nolock,
|
||||
ssl_keyfile,
|
||||
ssl_certfile,
|
||||
internal,
|
||||
return_instance=False,
|
||||
):
|
||||
"""Serve up specified SQLite database files with a web UI"""
|
||||
|
|
@ -570,6 +573,7 @@ def serve(
|
|||
pdb=pdb,
|
||||
crossdb=crossdb,
|
||||
nolock=nolock,
|
||||
internal=internal,
|
||||
)
|
||||
|
||||
# if files is a single directory, use that as config_dir=
|
||||
|
|
|
|||
|
|
@ -29,7 +29,13 @@ AttachedDatabase = namedtuple("AttachedDatabase", ("seq", "name", "file"))
|
|||
|
||||
class Database:
|
||||
def __init__(
|
||||
self, ds, path=None, is_mutable=True, is_memory=False, memory_name=None
|
||||
self,
|
||||
ds,
|
||||
path=None,
|
||||
is_mutable=True,
|
||||
is_memory=False,
|
||||
memory_name=None,
|
||||
mode=None,
|
||||
):
|
||||
self.name = None
|
||||
self.route = None
|
||||
|
|
@ -50,6 +56,7 @@ class Database:
|
|||
self._write_connection = None
|
||||
# This is used to track all file connections so they can be closed
|
||||
self._all_file_connections = []
|
||||
self.mode = mode
|
||||
|
||||
@property
|
||||
def cached_table_counts(self):
|
||||
|
|
@ -90,6 +97,7 @@ class Database:
|
|||
return conn
|
||||
if self.is_memory:
|
||||
return sqlite3.connect(":memory:", uri=True)
|
||||
|
||||
# mode=ro or immutable=1?
|
||||
if self.is_mutable:
|
||||
qs = "?mode=ro"
|
||||
|
|
@ -100,6 +108,8 @@ class Database:
|
|||
assert not (write and not self.is_mutable)
|
||||
if write:
|
||||
qs = ""
|
||||
if self.mode is not None:
|
||||
qs = f"?mode={self.mode}"
|
||||
conn = sqlite3.connect(
|
||||
f"file:{self.path}{qs}", uri=True, check_same_thread=False
|
||||
)
|
||||
|
|
|
|||
|
|
@ -146,8 +146,6 @@ async def _resolve_metadata_view_permissions(datasette, actor, action, resource)
|
|||
if allow is not None:
|
||||
return actor_matches_allow(actor, allow)
|
||||
elif action == "view-database":
|
||||
if resource == "_internal" and (actor is None or actor.get("id") != "root"):
|
||||
return False
|
||||
database_allow = datasette.metadata("allow", database=resource)
|
||||
if database_allow is None:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -5,13 +5,13 @@ from datasette.utils import table_column_details
|
|||
async def init_internal_db(db):
|
||||
create_tables_sql = textwrap.dedent(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS databases (
|
||||
CREATE TABLE IF NOT EXISTS core_databases (
|
||||
database_name TEXT PRIMARY KEY,
|
||||
path TEXT,
|
||||
is_memory INTEGER,
|
||||
schema_version INTEGER
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS tables (
|
||||
CREATE TABLE IF NOT EXISTS core_tables (
|
||||
database_name TEXT,
|
||||
table_name TEXT,
|
||||
rootpage INTEGER,
|
||||
|
|
@ -19,7 +19,7 @@ async def init_internal_db(db):
|
|||
PRIMARY KEY (database_name, table_name),
|
||||
FOREIGN KEY (database_name) REFERENCES databases(database_name)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS columns (
|
||||
CREATE TABLE IF NOT EXISTS core_columns (
|
||||
database_name TEXT,
|
||||
table_name TEXT,
|
||||
cid INTEGER,
|
||||
|
|
@ -33,7 +33,7 @@ async def init_internal_db(db):
|
|||
FOREIGN KEY (database_name) REFERENCES databases(database_name),
|
||||
FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS indexes (
|
||||
CREATE TABLE IF NOT EXISTS core_indexes (
|
||||
database_name TEXT,
|
||||
table_name TEXT,
|
||||
seq INTEGER,
|
||||
|
|
@ -45,7 +45,7 @@ async def init_internal_db(db):
|
|||
FOREIGN KEY (database_name) REFERENCES databases(database_name),
|
||||
FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS foreign_keys (
|
||||
CREATE TABLE IF NOT EXISTS core_foreign_keys (
|
||||
database_name TEXT,
|
||||
table_name TEXT,
|
||||
id INTEGER,
|
||||
|
|
@ -69,12 +69,16 @@ async def populate_schema_tables(internal_db, db):
|
|||
database_name = db.name
|
||||
|
||||
def delete_everything(conn):
|
||||
conn.execute("DELETE FROM tables WHERE database_name = ?", [database_name])
|
||||
conn.execute("DELETE FROM columns WHERE database_name = ?", [database_name])
|
||||
conn.execute("DELETE FROM core_tables WHERE database_name = ?", [database_name])
|
||||
conn.execute(
|
||||
"DELETE FROM foreign_keys WHERE database_name = ?", [database_name]
|
||||
"DELETE FROM core_columns WHERE database_name = ?", [database_name]
|
||||
)
|
||||
conn.execute(
|
||||
"DELETE FROM core_foreign_keys WHERE database_name = ?", [database_name]
|
||||
)
|
||||
conn.execute(
|
||||
"DELETE FROM core_indexes WHERE database_name = ?", [database_name]
|
||||
)
|
||||
conn.execute("DELETE FROM indexes WHERE database_name = ?", [database_name])
|
||||
|
||||
await internal_db.execute_write_fn(delete_everything)
|
||||
|
||||
|
|
@ -133,14 +137,14 @@ async def populate_schema_tables(internal_db, db):
|
|||
|
||||
await internal_db.execute_write_many(
|
||||
"""
|
||||
INSERT INTO tables (database_name, table_name, rootpage, sql)
|
||||
INSERT INTO core_tables (database_name, table_name, rootpage, sql)
|
||||
values (?, ?, ?, ?)
|
||||
""",
|
||||
tables_to_insert,
|
||||
)
|
||||
await internal_db.execute_write_many(
|
||||
"""
|
||||
INSERT INTO columns (
|
||||
INSERT INTO core_columns (
|
||||
database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden
|
||||
) VALUES (
|
||||
:database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden
|
||||
|
|
@ -150,7 +154,7 @@ async def populate_schema_tables(internal_db, db):
|
|||
)
|
||||
await internal_db.execute_write_many(
|
||||
"""
|
||||
INSERT INTO foreign_keys (
|
||||
INSERT INTO core_foreign_keys (
|
||||
database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match
|
||||
) VALUES (
|
||||
:database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match
|
||||
|
|
@ -160,7 +164,7 @@ async def populate_schema_tables(internal_db, db):
|
|||
)
|
||||
await internal_db.execute_write_many(
|
||||
"""
|
||||
INSERT INTO indexes (
|
||||
INSERT INTO core_indexes (
|
||||
database_name, table_name, seq, name, "unique", origin, partial
|
||||
) VALUES (
|
||||
:database_name, :table_name, :seq, :name, :unique, :origin, :partial
|
||||
|
|
|
|||
|
|
@ -950,9 +950,9 @@ class TableCreateView(BaseView):
|
|||
|
||||
|
||||
async def _table_columns(datasette, database_name):
|
||||
internal = datasette.get_database("_internal")
|
||||
result = await internal.execute(
|
||||
"select table_name, name from columns where database_name = ?",
|
||||
internal_db = datasette.get_internal_database()
|
||||
result = await internal_db.execute(
|
||||
"select table_name, name from core_columns where database_name = ?",
|
||||
[database_name],
|
||||
)
|
||||
table_columns = {}
|
||||
|
|
|
|||
|
|
@ -238,7 +238,7 @@ class CreateTokenView(BaseView):
|
|||
# Build list of databases and tables the user has permission to view
|
||||
database_with_tables = []
|
||||
for database in self.ds.databases.values():
|
||||
if database.name in ("_internal", "_memory"):
|
||||
if database.name == "_memory":
|
||||
continue
|
||||
if not await self.ds.permission_allowed(
|
||||
request.actor, "view-database", database.name
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue