Initial project layout + database table creation tools

.gitignore

@@ -0,0 +1,8 @@
+.venv
+__pycache__/
+*.py[cod]
+*$py.class
+venv
+.eggs
+.pytest_cache
+*.egg-info

docs/.gitignore

@@ -0,0 +1 @@
+_build

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = sqlite-utils
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/conf.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = []
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = ".rst"
+
+# The master toctree document.
+master_doc = "index"
+
+# General information about the project.
+project = "sqlite-utils"
+copyright = "2018, Simon Willison"
+author = "Simon Willison"
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = ""
+# The full version, including alpha/beta/rc tags.
+release = ""
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = "sphinx"
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "sphinx_rtd_theme"
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# This is required for the alabaster theme
+# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
+html_sidebars = {
+    "**": [
+        "relations.html",  # needs 'show_related': True theme option to display
+        "searchbox.html",
+    ]
+}
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = "sqlite-utils-doc"
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (
+        master_doc,
+        "sqlite-utils.tex",
+        "sqlite-utils documentation",
+        "Simon Willison",
+        "manual",
+    )
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [(master_doc, "sqlite-utils", "sqlite-utils documentation", [author], 1)]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (
+        master_doc,
+        "sqlite-utils",
+        "sqlite-utils documentation",
+        author,
+        "sqlite-utils",
+        "Python utility functions for manipulating SQLite databases",
+        "Miscellaneous",
+    )
+]

docs/index.rst

@@ -0,0 +1,6 @@
+sqlite-utils
+============
+
+*Python utility functions for manipulating SQLite databases*
+
+Work in progress.

setup.py

@@ -0,0 +1,41 @@
+from setuptools import setup, find_packages
+import io
+import os
+
+
+def get_long_description():
+    with io.open(
+        os.path.join(os.path.dirname(os.path.abspath(__file__)), "README.md"),
+        encoding="utf8",
+    ) as fp:
+        return fp.read()
+
+
+setup(
+    name="sqlite-utils",
+    description="Python utility functions for manipulating SQLite databases",
+    long_description=get_long_description(),
+    long_description_content_type="text/markdown",
+    author="Simon Willison",
+    version="0.1",
+    license="Apache License, Version 2.0",
+    packages=find_packages(),
+    install_requires=["click==6.7"],
+    setup_requires=["pytest-runner"],
+    tests_require=["pytest==3.2.3"],
+    entry_points="""
+        [console_scripts]
+        sqlite-utils=sqlite_utils.cli:cli
+    """,
+    url="https://github.com/simonw/sqlite-utils",
+    classifiers=[
+        "Development Status :: 3 - Alpha",
+        "Intended Audience :: Developers",
+        "Intended Audience :: Science/Research",
+        "Intended Audience :: End Users/Desktop",
+        "Topic :: Database",
+        "License :: OSI Approved :: Apache Software License",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+    ],
+)

sqlite_utils/cli.py

@@ -0,0 +1,6 @@
+import click
+
+
+@click.command()
+def cli(*args):
+    click.echo(repr(args))

sqlite_utils/db.py

@@ -0,0 +1,168 @@
+import sqlite3
+from collections import namedtuple
+
+Column = namedtuple(
+    "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk")
+)
+
+
+class Database:
+    def __init__(self, filename_or_conn):
+        if isinstance(filename_or_conn, str):
+            self.conn = sqlite3.connect(filename_or_conn)
+        else:
+            self.conn = filename_or_conn
+
+    def __getitem__(self, table_name):
+        return Table(self, table_name)
+
+    @property
+    def tables(self):
+        return [
+            r[0]
+            for r in self.conn.execute(
+                "select name from sqlite_master where type = 'table'"
+            ).fetchall()
+        ]
+
+    def create_table(self, name, columns, pk=None):
+        sql = """CREATE TABLE {table} (
+            {columns}
+        );
+        """.format(
+            table=name,
+            columns=",\n".join(
+                "   {col_name} {col_type} {primary_key}".format(
+                    col_name=col_name,
+                    col_type={
+                        float: "FLOAT",
+                        int: "INTEGER",
+                        bool: "INTEGER",
+                        str: "TEXT",
+                        None.__class__: "TEXT",
+                    }[col_type],
+                    primary_key=" PRIMARY KEY" if (pk == col_name) else "",
+                )
+                for col_name, col_type in columns.items()
+            ),
+        )
+        self.conn.execute(sql)
+        return self[name]
+
+
+class Table:
+    def __init__(self, db, name):
+        self.db = db
+        self.name = name
+        self.exists = self.name in self.db.tables
+
+    @property
+    def columns(self):
+        if not self.exists:
+            return []
+        rows = self.db.conn.execute(
+            "PRAGMA table_info([{}])".format(self.name)
+        ).fetchall()
+        return [Column(*row) for row in rows]
+
+    def create(self, columns, pk=None, foreign_keys=None):
+        # Ignore columns in foreign_keys list
+        columns = {
+            name: value
+            for name, value in columns.items()
+            if name not in {fk[0] for fk in (foreign_keys or [])}
+        }
+        self.db.create_table(self.name, columns, pk=pk)
+        if foreign_keys:
+            for args in foreign_keys:
+                self.add_foreign_key(*args)
+        self.exists = True
+
+    def drop(self):
+        return self.db.conn.execute("DROP TABLE {}".format(self.name))
+
+    def add_foreign_key(self, column, column_type, other_table, other_column):
+        sql = """
+            ALTER TABLE {table} ADD COLUMN {column} {column_type}
+            REFERENCES {other_table}({other_column});
+        """.format(
+            table=self.name,
+            column=column,
+            column_type=column_type,
+            other_table=other_table,
+            other_column=other_column,
+        )
+        result = self.db.conn.execute(sql)
+        self.db.conn.commit()
+        return result
+
+    def detect_column_types(self, records):
+        all_column_types = {}
+        for record in records:
+            for key, value in record.items():
+                all_column_types.setdefault(key, set()).add(type(value))
+        column_types = {}
+        for key, types in all_column_types.items():
+            if len(types) == 1:
+                t = list(types)[0]
+            elif {int, bool}.issuperset(types):
+                t = int
+            elif {int, float, bool}.issuperset(types):
+                t = float
+            else:
+                t = str
+            column_types[key] = t
+        return column_types
+
+    def insert(self, record, pk=None, foreign_keys=None, upsert=False):
+        return self.insert_all(
+            [record], pk=pk, foreign_keys=foreign_keys, upsert=upsert
+        )
+
+    def insert_all(
+        self, records, pk=None, foreign_keys=None, upsert=False, batch_size=100
+    ):
+        """
+        Like .insert() but takes a list of records and ensures that the table
+        that it creates (if table does not exist) has columns for ALL of that
+        data
+        """
+        if not self.exists:
+            self.create(self.detect_column_types(records), pk, foreign_keys)
+        all_columns = set()
+        for record in records:
+            all_columns.update(record.keys())
+        all_columns = list(sorted(all_columns))
+        for chunk in chunks(records, batch_size):
+            sql = """
+                INSERT {upsert} INTO {table} ({columns}) VALUES {rows};
+            """.format(
+                upsert="OR REPLACE" if upsert else "",
+                table=self.name,
+                columns=", ".join(all_columns),
+                rows=", ".join(
+                    """
+                    ({placeholders})
+                """.format(
+                        placeholders=", ".join(["?"] * len(all_columns))
+                    )
+                    for record in chunk
+                ),
+            )
+            values = []
+            for record in chunk:
+                values.extend(record.get(key, None) for key in all_columns)
+            result = self.db.conn.execute(sql, values)
+            self.db.conn.commit()
+        return result
+
+    def upsert(self, record, pk=None, foreign_keys=None):
+        return self.insert(record, pk=pk, foreign_keys=foreign_keys, upsert=True)
+
+    def upsert_all(self, records, pk=None, foreign_keys=None):
+        return self.insert_all(records, pk=pk, foreign_keys=foreign_keys, upsert=True)
+
+
+def chunks(sequence, size):
+    for i in range(0, len(sequence), size):
+        yield sequence[i : i + size]

tests/test_create.py

@@ -0,0 +1,40 @@
+from sqlite_utils import db
+import sqlite3
+import pytest
+
+
+@pytest.fixture
+def fresh_db():
+    return db.Database(sqlite3.connect(":memory:"))
+
+
+def test_create_table(fresh_db):
+    assert [] == fresh_db.tables
+    table = fresh_db.create_table(
+        "test_table",
+        {"text_col": str, "float_col": float, "int_col": int, "bool_col": bool},
+    )
+    assert ["test_table"] == fresh_db.tables
+    assert [
+        {"name": "text_col", "type": "TEXT"},
+        {"name": "float_col", "type": "FLOAT"},
+        {"name": "int_col", "type": "INTEGER"},
+        {"name": "bool_col", "type": "INTEGER"},
+    ] == [{"name": col.name, "type": col.type} for col in table.columns]
+
+
+@pytest.mark.parametrize(
+    "example,expected_columns",
+    (
+        (
+            {"name": "Ravi", "age": 63},
+            [{"name": "name", "type": "TEXT"}, {"name": "age", "type": "INTEGER"}],
+        ),
+    ),
+)
+def test_create_table_from_example(fresh_db, example, expected_columns):
+    fresh_db["people"].insert(example)
+    assert ["people"] == fresh_db.tables
+    assert expected_columns == [
+        {"name": col.name, "type": col.type} for col in fresh_db["people"].columns
+    ]