diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4449289 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.venv +__pycache__/ +*.py[cod] +*$py.class +venv +.eggs +.pytest_cache +*.egg-info diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000..e35d885 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +_build diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..3744f40 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = sqlite-utils +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..1136754 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = ".rst" + +# The master toctree document. +master_doc = "index" + +# General information about the project. +project = "sqlite-utils" +copyright = "2018, Simon Willison" +author = "Simon Willison" + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = "" +# The full version, including alpha/beta/rc tags. +release = "" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# This is required for the alabaster theme +# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars +html_sidebars = { + "**": [ + "relations.html", # needs 'show_related': True theme option to display + "searchbox.html", + ] +} + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = "sqlite-utils-doc" + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ( + master_doc, + "sqlite-utils.tex", + "sqlite-utils documentation", + "Simon Willison", + "manual", + ) +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(master_doc, "sqlite-utils", "sqlite-utils documentation", [author], 1)] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "sqlite-utils", + "sqlite-utils documentation", + author, + "sqlite-utils", + "Python utility functions for manipulating SQLite databases", + "Miscellaneous", + ) +] diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..490e994 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,6 @@ +sqlite-utils +============ + +*Python utility functions for manipulating SQLite databases* + +Work in progress. diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..315edf5 --- /dev/null +++ b/setup.py @@ -0,0 +1,41 @@ +from setuptools import setup, find_packages +import io +import os + + +def get_long_description(): + with io.open( + os.path.join(os.path.dirname(os.path.abspath(__file__)), "README.md"), + encoding="utf8", + ) as fp: + return fp.read() + + +setup( + name="sqlite-utils", + description="Python utility functions for manipulating SQLite databases", + long_description=get_long_description(), + long_description_content_type="text/markdown", + author="Simon Willison", + version="0.1", + license="Apache License, Version 2.0", + packages=find_packages(), + install_requires=["click==6.7"], + setup_requires=["pytest-runner"], + tests_require=["pytest==3.2.3"], + entry_points=""" + [console_scripts] + sqlite-utils=sqlite_utils.cli:cli + """, + url="https://github.com/simonw/sqlite-utils", + classifiers=[ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Intended Audience :: End Users/Desktop", + "Topic :: Database", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + ], +) diff --git a/sqlite_utils/__init__.py b/sqlite_utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py new file mode 100644 index 0000000..6177c6b --- /dev/null +++ b/sqlite_utils/cli.py @@ -0,0 +1,6 @@ +import click + + +@click.command() +def cli(*args): + click.echo(repr(args)) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py new file mode 100644 index 0000000..1b441cf --- /dev/null +++ b/sqlite_utils/db.py @@ -0,0 +1,168 @@ +import sqlite3 +from collections import namedtuple + +Column = namedtuple( + "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk") +) + + +class Database: + def __init__(self, filename_or_conn): + if isinstance(filename_or_conn, str): + self.conn = sqlite3.connect(filename_or_conn) + else: + self.conn = filename_or_conn + + def __getitem__(self, table_name): + return Table(self, table_name) + + @property + def tables(self): + return [ + r[0] + for r in self.conn.execute( + "select name from sqlite_master where type = 'table'" + ).fetchall() + ] + + def create_table(self, name, columns, pk=None): + sql = """CREATE TABLE {table} ( + {columns} + ); + """.format( + table=name, + columns=",\n".join( + " {col_name} {col_type} {primary_key}".format( + col_name=col_name, + col_type={ + float: "FLOAT", + int: "INTEGER", + bool: "INTEGER", + str: "TEXT", + None.__class__: "TEXT", + }[col_type], + primary_key=" PRIMARY KEY" if (pk == col_name) else "", + ) + for col_name, col_type in columns.items() + ), + ) + self.conn.execute(sql) + return self[name] + + +class Table: + def __init__(self, db, name): + self.db = db + self.name = name + self.exists = self.name in self.db.tables + + @property + def columns(self): + if not self.exists: + return [] + rows = self.db.conn.execute( + "PRAGMA table_info([{}])".format(self.name) + ).fetchall() + return [Column(*row) for row in rows] + + def create(self, columns, pk=None, foreign_keys=None): + # Ignore columns in foreign_keys list + columns = { + name: value + for name, value in columns.items() + if name not in {fk[0] for fk in (foreign_keys or [])} + } + self.db.create_table(self.name, columns, pk=pk) + if foreign_keys: + for args in foreign_keys: + self.add_foreign_key(*args) + self.exists = True + + def drop(self): + return self.db.conn.execute("DROP TABLE {}".format(self.name)) + + def add_foreign_key(self, column, column_type, other_table, other_column): + sql = """ + ALTER TABLE {table} ADD COLUMN {column} {column_type} + REFERENCES {other_table}({other_column}); + """.format( + table=self.name, + column=column, + column_type=column_type, + other_table=other_table, + other_column=other_column, + ) + result = self.db.conn.execute(sql) + self.db.conn.commit() + return result + + def detect_column_types(self, records): + all_column_types = {} + for record in records: + for key, value in record.items(): + all_column_types.setdefault(key, set()).add(type(value)) + column_types = {} + for key, types in all_column_types.items(): + if len(types) == 1: + t = list(types)[0] + elif {int, bool}.issuperset(types): + t = int + elif {int, float, bool}.issuperset(types): + t = float + else: + t = str + column_types[key] = t + return column_types + + def insert(self, record, pk=None, foreign_keys=None, upsert=False): + return self.insert_all( + [record], pk=pk, foreign_keys=foreign_keys, upsert=upsert + ) + + def insert_all( + self, records, pk=None, foreign_keys=None, upsert=False, batch_size=100 + ): + """ + Like .insert() but takes a list of records and ensures that the table + that it creates (if table does not exist) has columns for ALL of that + data + """ + if not self.exists: + self.create(self.detect_column_types(records), pk, foreign_keys) + all_columns = set() + for record in records: + all_columns.update(record.keys()) + all_columns = list(sorted(all_columns)) + for chunk in chunks(records, batch_size): + sql = """ + INSERT {upsert} INTO {table} ({columns}) VALUES {rows}; + """.format( + upsert="OR REPLACE" if upsert else "", + table=self.name, + columns=", ".join(all_columns), + rows=", ".join( + """ + ({placeholders}) + """.format( + placeholders=", ".join(["?"] * len(all_columns)) + ) + for record in chunk + ), + ) + values = [] + for record in chunk: + values.extend(record.get(key, None) for key in all_columns) + result = self.db.conn.execute(sql, values) + self.db.conn.commit() + return result + + def upsert(self, record, pk=None, foreign_keys=None): + return self.insert(record, pk=pk, foreign_keys=foreign_keys, upsert=True) + + def upsert_all(self, records, pk=None, foreign_keys=None): + return self.insert_all(records, pk=pk, foreign_keys=foreign_keys, upsert=True) + + +def chunks(sequence, size): + for i in range(0, len(sequence), size): + yield sequence[i : i + size] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_create.py b/tests/test_create.py new file mode 100644 index 0000000..168d4e4 --- /dev/null +++ b/tests/test_create.py @@ -0,0 +1,40 @@ +from sqlite_utils import db +import sqlite3 +import pytest + + +@pytest.fixture +def fresh_db(): + return db.Database(sqlite3.connect(":memory:")) + + +def test_create_table(fresh_db): + assert [] == fresh_db.tables + table = fresh_db.create_table( + "test_table", + {"text_col": str, "float_col": float, "int_col": int, "bool_col": bool}, + ) + assert ["test_table"] == fresh_db.tables + assert [ + {"name": "text_col", "type": "TEXT"}, + {"name": "float_col", "type": "FLOAT"}, + {"name": "int_col", "type": "INTEGER"}, + {"name": "bool_col", "type": "INTEGER"}, + ] == [{"name": col.name, "type": col.type} for col in table.columns] + + +@pytest.mark.parametrize( + "example,expected_columns", + ( + ( + {"name": "Ravi", "age": 63}, + [{"name": "name", "type": "TEXT"}, {"name": "age", "type": "INTEGER"}], + ), + ), +) +def test_create_table_from_example(fresh_db, example, expected_columns): + fresh_db["people"].insert(example) + assert ["people"] == fresh_db.tables + assert expected_columns == [ + {"name": col.name, "type": col.type} for col in fresh_db["people"].columns + ]