Remove develop_server.sh in favour of pelican serving static files itself

Competing static site generators integrate the functionality of regenerating
content and serving it into their main executable. In pelican this
functionality used to be in an external script `develop_server.sh` which
resides in the blog base directory. This has the disadvantage that changes in
pelican can break the `develop_server.sh` scripts which will not automatically
be upgraded together with pelican by package managers. Thus, pelican should
integrate this functionality into its main executable.

To this end, this commit removes `develop_server.sh` and adds three command
line options to the pelican executable:

 * `-l/--listen` starts the HTTP server (`-s/--serve` was already taken)
 * `-p/--port` specifies the port to listen at
 * `-b/--bind` specifies the IP to bind to

`--listen` and `--autoreload` can be used together to achieve the same
effect that other static site generators offer: Serve files via HTTP
while at the same time auto-generating the content.

Since the `develop_server.sh` script was removed, pelican-quickstart looses the
`develop` option.

Since the `develop_server.sh` script was removed, the Makefile looses the
`stopserver` target and the `devserver` target is replaced by running `pelican
-l` in the foreground.

Since pelican now offers the `--listen` option, the fabfile uses that instead
of starting the socketserver itself.

docs/install.rst

@@ -103,7 +103,6 @@ can optionally add yourself if you plan to create non-chronological content)::
     ├── content
     │   └── (pages)
     ├── output
-    ├── develop_server.sh
     ├── fabfile.py
     ├── Makefile
     ├── pelicanconf.py       # Main settings file

docs/publish.rst

@@ -201,10 +201,7 @@ separate terminal sessions, but you can run both at once via::
     make devserver
 
 The above command will simultaneously run Pelican in regeneration mode as well
-as serve the output at http://localhost:8000. Once you are done testing your
+as serve the output at http://localhost:8000.
-changes, you should stop the development server via::
-
-    ./develop_server.sh stop
 
 When you're ready to publish your site, you can upload it via the method(s) you
 chose during the ``pelican-quickstart`` questionnaire. For this example, we'll

docs/quickstart.rst

@@ -61,11 +61,10 @@ ignored for now.)
 Preview your site
 -----------------
 
-Open a new terminal session and run the following commands to switch to your
+Open a new terminal session, navigate to your site directory and run the
-``output`` directory and launch Pelican's web server::
+following command to launch Pelican's web server::
 
-    cd ~/projects/yoursite/output
+    pelican --listen
-    python -m pelican.server
 
 Preview your site by navigating to http://localhost:8000/ in your browser.
 

docs/settings.rst

@@ -328,6 +328,15 @@ Basic settings
    A list of metadata fields containing reST/Markdown content to be parsed and
    translated to HTML.
 
+.. data:: PORT = 8000
+
+   The TCP port to serve content from the output folder via HTTP when pelican
+   is run with --listen
+
+.. data:: BIND = ''
+
+   The IP to which to bind the HTTP server.
+
 
 URL settings
 ============

pelican/__init__.py

@@ -5,10 +5,12 @@ import argparse
 import collections
 import locale
 import logging
+import multiprocessing
 import os
 import re
 import sys
 import time
+import traceback
 
 import six
 
@@ -20,6 +22,7 @@ from pelican.generators import (ArticlesGenerator, PagesGenerator,
                                 SourceFileGenerator, StaticGenerator,
                                 TemplatePagesGenerator)
 from pelican.readers import Readers
+from pelican.server import ComplexHTTPRequestHandler, RootedHTTPServer
 from pelican.settings import read_settings
 from pelican.utils import (clean_output_dir, file_watcher,
                            folder_watcher, maybe_pluralize)
@@ -336,7 +339,24 @@ def parse_arguments():
                         help=('Only enable log de-duplication for levels equal'
                               ' to or above the specified value'))
 
-    return parser.parse_args()
+    parser.add_argument('-l', '--listen', dest='listen', action='store_true',
+                        help='Serve content files via HTTP and port 8000.')
+
+    parser.add_argument('-p', '--port', dest='port', type=int,
+                        help='Port to serve HTTP files at. (default: 8000)')
+
+    parser.add_argument('-b', '--bind', dest='bind',
+                        help='IP to bind to when serving files via HTTP '
+                        '(default: 127.0.0.1)')
+
+    args = parser.parse_args()
+
+    if args.port is not None and not args.listen:
+        logger.warning('--port without --listen has no effect')
+    if args.bind is not None and not args.listen:
+        logger.warning('--bind without --listen has no effect')
+
+    return args
 
 
 def get_config(args):
@@ -359,6 +379,10 @@ def get_config(args):
         config['WRITE_SELECTED'] = args.selected_paths.split(',')
     if args.relative_paths:
         config['RELATIVE_URLS'] = args.relative_paths
+    if args.port is not None:
+        config['PORT'] = args.port
+    if args.bind is not None:
+        config['BIND'] = args.bind
     config['DEBUG'] = args.verbosity == logging.DEBUG
 
     # argparse returns bytes in Py2. There is no definite answer as to which
@@ -391,6 +415,100 @@ def get_instance(args):
     return cls(settings), settings
 
 
+def autoreload(watchers, args, old_static, reader_descs, excqueue=None):
+    while True:
+        try:
+            # Check source dir for changed files ending with the given
+            # extension in the settings. In the theme dir is no such
+            # restriction; all files are recursively checked if they
+            # have changed, no matter what extension the filenames
+            # have.
+            modified = {k: next(v) for k, v in watchers.items()}
+
+            if modified['settings']:
+                pelican, settings = get_instance(args)
+
+                # Adjust static watchers if there are any changes
+                new_static = settings.get("STATIC_PATHS", [])
+
+                # Added static paths
+                # Add new watchers and set them as modified
+                new_watchers = set(new_static).difference(old_static)
+                for static_path in new_watchers:
+                    static_key = '[static]%s' % static_path
+                    watchers[static_key] = folder_watcher(
+                        os.path.join(pelican.path, static_path),
+                        [''],
+                        pelican.ignore_files)
+                    modified[static_key] = next(watchers[static_key])
+
+                # Removed static paths
+                # Remove watchers and modified values
+                old_watchers = set(old_static).difference(new_static)
+                for static_path in old_watchers:
+                    static_key = '[static]%s' % static_path
+                    watchers.pop(static_key)
+                    modified.pop(static_key)
+
+                # Replace old_static with the new one
+                old_static = new_static
+
+            if any(modified.values()):
+                print('\n-> Modified: {}. re-generating...'.format(
+                    ', '.join(k for k, v in modified.items() if v)))
+
+                if modified['content'] is None:
+                    logger.warning(
+                        'No valid files found in content for '
+                        + 'the active readers:\n'
+                        + '\n'.join(reader_descs))
+
+                if modified['theme'] is None:
+                    logger.warning('Empty theme folder. Using `basic` '
+                                   'theme.')
+
+                pelican.run()
+
+        except KeyboardInterrupt as e:
+            logger.warning("Keyboard interrupt, quitting.")
+            if excqueue is not None:
+                excqueue.put(traceback.format_exception_only(type(e), e)[-1])
+            return
+
+        except Exception as e:
+            if (args.verbosity == logging.DEBUG):
+                if excqueue is not None:
+                    excqueue.put(
+                        traceback.format_exception_only(type(e), e)[-1])
+                else:
+                    raise
+            logger.warning(
+                'Caught exception "%s". Reloading.', e)
+
+        finally:
+            time.sleep(.5)  # sleep to avoid cpu load
+
+
+def listen(server, port, output, excqueue=None):
+    RootedHTTPServer.allow_reuse_address = True
+    try:
+        httpd = RootedHTTPServer(
+            output, (server, port), ComplexHTTPRequestHandler)
+    except OSError as e:
+        logging.error("Could not listen on port %s, server %s.", port, server)
+        if excqueue is not None:
+            excqueue.put(traceback.format_exception_only(type(e), e)[-1])
+        return
+
+    logging.info("Serving at port %s, server %s.", port, server)
+    try:
+        httpd.serve_forever()
+    except Exception as e:
+        if excqueue is not None:
+            excqueue.put(traceback.format_exception_only(type(e), e)[-1])
+        return
+
+
 def main():
     args = parse_arguments()
     logs_dedup_min_level = getattr(logging, args.logs_dedup_min_level)
@@ -426,76 +544,28 @@ def main():
                 [''],
                 pelican.ignore_files)
 
-        if args.autoreload:
+        if args.autoreload and args.listen:
+            excqueue = multiprocessing.Queue()
+            p1 = multiprocessing.Process(
+                target=autoreload,
+                args=(watchers, args, old_static, reader_descs, excqueue))
+            p2 = multiprocessing.Process(
+                target=listen,
+                args=(settings.get('BIND'), settings.get('PORT'),
+                      settings.get("OUTPUT_PATH"), excqueue))
+            p1.start()
+            p2.start()
+            exc = excqueue.get()
+            p1.terminate()
+            p2.terminate()
+            logger.critical(exc)
+        elif args.autoreload:
             print('  --- AutoReload Mode: Monitoring `content`, `theme` and'
                   ' `settings` for changes. ---')
-
+            autoreload(watchers, args, old_static, reader_descs)
-            while True:
+        elif args.listen:
-                try:
+            listen(settings.get('BIND'), settings.get('PORT'),
-                    # Check source dir for changed files ending with the given
+                   settings.get("OUTPUT_PATH"))
-                    # extension in the settings. In the theme dir is no such
-                    # restriction; all files are recursively checked if they
-                    # have changed, no matter what extension the filenames
-                    # have.
-                    modified = {k: next(v) for k, v in watchers.items()}
-
-                    if modified['settings']:
-                        pelican, settings = get_instance(args)
-
-                        # Adjust static watchers if there are any changes
-                        new_static = settings.get("STATIC_PATHS", [])
-
-                        # Added static paths
-                        # Add new watchers and set them as modified
-                        new_watchers = set(new_static).difference(old_static)
-                        for static_path in new_watchers:
-                            static_key = '[static]%s' % static_path
-                            watchers[static_key] = folder_watcher(
-                                os.path.join(pelican.path, static_path),
-                                [''],
-                                pelican.ignore_files)
-                            modified[static_key] = next(watchers[static_key])
-
-                        # Removed static paths
-                        # Remove watchers and modified values
-                        old_watchers = set(old_static).difference(new_static)
-                        for static_path in old_watchers:
-                            static_key = '[static]%s' % static_path
-                            watchers.pop(static_key)
-                            modified.pop(static_key)
-
-                        # Replace old_static with the new one
-                        old_static = new_static
-
-                    if any(modified.values()):
-                        print('\n-> Modified: {}. re-generating...'.format(
-                            ', '.join(k for k, v in modified.items() if v)))
-
-                        if modified['content'] is None:
-                            logger.warning(
-                                'No valid files found in content for '
-                                + 'the active readers:\n'
-                                + '\n'.join(reader_descs))
-
-                        if modified['theme'] is None:
-                            logger.warning('Empty theme folder. Using `basic` '
-                                           'theme.')
-
-                        pelican.run()
-
-                except KeyboardInterrupt:
-                    logger.warning("Keyboard interrupt, quitting.")
-                    break
-
-                except Exception as e:
-                    if (args.verbosity == logging.DEBUG):
-                        raise
-                    logger.warning(
-                        'Caught exception "%s". Reloading.', e)
-
-                finally:
-                    time.sleep(.5)  # sleep to avoid cpu load
-
         else:
             if next(watchers['content']) is None:
                 logger.warning(

pelican/server.py

@@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals
 import argparse
 import logging
 import os
+import posixpath
 import ssl
 import sys
 
@@ -12,8 +13,9 @@ try:
 except ImportError:
     magic_from_file = None
 
+from six.moves import BaseHTTPServer
 from six.moves import SimpleHTTPServer as srvmod
-from six.moves import socketserver
+from six.moves import urllib
 
 
 def parse_arguments():
@@ -33,6 +35,9 @@ def parse_arguments():
     parser.add_argument('--key', default="./key.pem", nargs="?",
                         help='Path to certificate key file. ' +
                         'Relative to current directory')
+    parser.add_argument('path', default=".",
+                        help='Path to pelican source directory to serve. ' +
+                        'Relative to current directory')
     return parser.parse_args()
 
 
@@ -40,6 +45,26 @@ class ComplexHTTPRequestHandler(srvmod.SimpleHTTPRequestHandler):
     SUFFIXES = ['', '.html', '/index.html']
     RSTRIP_PATTERNS = ['', '/']
 
+    def translate_path(self, path):
+        # abandon query parameters
+        path = path.split('?', 1)[0]
+        path = path.split('#', 1)[0]
+        # Don't forget explicit trailing slash when normalizing. Issue17324
+        trailing_slash = path.rstrip().endswith('/')
+        path = urllib.parse.unquote(path)
+        path = posixpath.normpath(path)
+        words = path.split('/')
+        words = filter(None, words)
+        path = self.base_path
+        for word in words:
+            if os.path.dirname(word) or word in (os.curdir, os.pardir):
+                # Ignore components that are not a simple file/directory name
+                continue
+            path = os.path.join(path, word)
+        if trailing_slash:
+            path += '/'
+        return path
+
     def do_GET(self):
         # cut off a query string
         if '?' in self.path:
@@ -83,11 +108,17 @@ class ComplexHTTPRequestHandler(srvmod.SimpleHTTPRequestHandler):
         return mimetype
 
 
+class RootedHTTPServer(BaseHTTPServer.HTTPServer):
+    def __init__(self, base_path, *args, **kwargs):
+        BaseHTTPServer.HTTPServer.__init__(self, *args, **kwargs)
+        self.RequestHandlerClass.base_path = base_path
+
+
 if __name__ == '__main__':
     args = parse_arguments()
-    socketserver.TCPServer.allow_reuse_address = True
+    RootedHTTPServer.allow_reuse_address = True
     try:
-        httpd = socketserver.TCPServer(
+        httpd = RootedHTTPServer(
             (args.server, args.port),
             ComplexHTTPRequestHandler)
         if args.ssl:
@@ -97,7 +128,6 @@ if __name__ == '__main__':
     except ssl.SSLError as e:
         logging.error("Couldn't open certificate file %s or key file %s",
                       args.cert, args.key)
-    except OSError as e:
         logging.error("Could not listen on port %s, server %s.",
                       args.port, args.server)
         sys.exit(getattr(e, 'exitcode', 1))

pelican/settings.py

@@ -150,6 +150,8 @@ DEFAULT_CONFIG = {
     'LOAD_CONTENT_CACHE': False,
     'WRITE_SELECTED': [],
     'FORMATTED_FIELDS': ['summary'],
+    'PORT': 8000,
+    'BIND': '',
 }
 
 PYGMENTS_RST_OPTIONS = None

pelican/tools/pelican_quickstart.py

@@ -263,8 +263,6 @@ needed by Pelican.
 
     automation = ask('Do you want to generate a Fabfile/Makefile '
                      'to automate generation and publishing?', bool, True)
-    develop = ask('Do you want an auto-reload HTTP script '
-                  'to assist with theme and site development?', bool, True)
 
     if automation:
         if ask('Do you want to upload your website using FTP?',
@@ -380,29 +378,6 @@ needed by Pelican.
         except OSError as e:
             print('Error: {0}'.format(e))
 
-    if develop:
-        conf_shell = dict()
-        for key, value in CONF.items():
-            if isinstance(value, six.string_types) and ' ' in value:
-                value = '"' + value.replace('"', '\\"') + '"'
-            conf_shell[key] = value
-        try:
-            with codecs.open(os.path.join(CONF['basedir'],
-                                          'develop_server.sh'),
-                             'w', 'utf-8') as fd:
-                py_v = '${PY:-python}'
-                if six.PY3:
-                    py_v = '${PY:-python3}'
-                _template = _jinja_env.get_template('develop_server.sh.jinja2')
-                fd.write(_template.render(py_v=py_v, **conf_shell))
-                fd.close()
-
-                # mode 0o755
-                os.chmod((os.path.join(CONF['basedir'],
-                                       'develop_server.sh')), 493)
-        except OSError as e:
-            print('Error: {0}'.format(e))
-
     print('Done. Your new project is available at %s' % CONF['basedir'])
 
 

pelican/tools/templates/Makefile.jinja2

@@ -60,9 +60,7 @@ help:
 	@echo '   make publish                        generate using production settings '
 	@echo '   make serve [PORT=8000]              serve site at http://localhost:8000'
 	@echo '   make serve-global [SERVER=0.0.0.0]  serve (as root) to $(SERVER):80    '
-	@echo '   make devserver [PORT=8000]          start/restart develop_server.sh    '
+	@echo '   make devserver [PORT=8000]          serve and regenerate together      '
-	@echo '   make stopserver                     stop local server                  '
-{% if ssh %}
 	@echo '   make ssh_upload                     upload the web site via SSH        '
 	@echo '   make rsync_upload                   upload the web site via rsync+ssh  '
 {% endif %}
@@ -97,30 +95,26 @@ regenerate:
 
 serve:
 ifdef PORT
-	cd $(OUTPUTDIR) && $(PY) -m pelican.server $(PORT)
+	$$(PELICAN) -l $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(CONFFILE) $$(PELICANOPTS) -p $$(PORT)
 else
-	cd $(OUTPUTDIR) && $(PY) -m pelican.server
+	$$(PELICAN) -l $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(CONFFILE) $$(PELICANOPTS)
 endif
 
 serve-global:
 ifdef SERVER
-	cd $(OUTPUTDIR) && $(PY) -m pelican.server 80 $(SERVER)
+	$$(PELICAN) -l $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(CONFFILE) $$(PELICANOPTS) -p $$(PORT) -b $$(SERVER)
 else
-	cd $(OUTPUTDIR) && $(PY) -m pelican.server 80 0.0.0.0
+	$$(PELICAN) -l $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(CONFFILE) $$(PELICANOPTS) -p $$(PORT) -b 0.0.0.0
 endif
 
 
 devserver:
 ifdef PORT
-	$(BASEDIR)/develop_server.sh restart $(PORT)
+	$$(PELICAN) -lr $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(CONFFILE) $$(PELICANOPTS) -p $$(PORT)
 else
-	$(BASEDIR)/develop_server.sh restart
+	$$(PELICAN) -lr $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(CONFFILE) $$(PELICANOPTS)
 endif
 
-stopserver:
-	$(BASEDIR)/develop_server.sh stop
-	@echo 'Stopped Pelican and SimpleHTTPServer processes running in background.'
-
 publish:
 	$(PELICAN) $(INPUTDIR) -o $(OUTPUTDIR) -s $(PUBLISHCONF) $(PELICANOPTS)
 

pelican/tools/templates/develop_server.sh.jinja2

@@ -1,103 +0,0 @@
-#!/usr/bin/env bash
-##
-# This section should match your Makefile
-##
-PY={{py_v}}
-PELICAN=${PELICAN:-pelican}
-PELICANOPTS={{pelicanopts}}
-
-BASEDIR=$(pwd)
-INPUTDIR="$BASEDIR"/content
-OUTPUTDIR="$BASEDIR"/output
-CONFFILE="$BASEDIR"/pelicanconf.py
-
-###
-# Don't change stuff below here unless you are sure
-###
-
-SRV_PID="$BASEDIR"/srv.pid
-PELICAN_PID="$BASEDIR"/pelican.pid
-
-function usage(){
-  echo "usage: $0 (stop) (start) (restart) [port]"
-  echo "This starts Pelican in debug and reload mode and then launches"
-  echo "an HTTP server to help site development. It doesn't read"
-  echo "your Pelican settings, so if you edit any paths in your Makefile"
-  echo "you will need to edit your settings as well."
-  exit 3
-}
-
-function alive() {
-  kill -0 $1 >/dev/null 2>&1
-}
-
-function shut_down(){
-  PID=$(cat "$SRV_PID")
-  if [[ $? -eq 0 ]]; then
-    if alive $PID; then
-      echo "Stopping HTTP server"
-      kill $PID
-    else
-      echo "Stale PID, deleting"
-    fi
-    rm "$SRV_PID"
-  else
-    echo "HTTP server PIDFile not found"
-  fi
-
-  PID=$(cat "$PELICAN_PID")
-  if [[ $? -eq 0 ]]; then
-    if alive $PID; then
-      echo "Killing Pelican"
-      kill $PID
-    else
-      echo "Stale PID, deleting"
-    fi
-    rm "$PELICAN_PID"
-  else
-    echo "Pelican PIDFile not found"
-  fi
-}
-
-function start_up(){
-  local port=$1
-  echo "Starting up Pelican and HTTP server"
-  shift
-  $PELICAN --debug --autoreload -r "$INPUT"DIR -o "$OUTPUTDIR" -s "$CONFFILE" $PELICANOPTS &
-  pelican_pid=$!
-  echo $pelican_pid > "$PELICAN_PID"
-  mkdir -p "$OUTPUTDIR" && cd "$OUTPUTDIR"
-  $PY -m pelican.server $port &
-  srv_pid=$!
-  echo $srv_pid > "$SRV_PID"
-  cd "$BASEDIR"
-  sleep 1
-  if ! alive $pelican_pid ; then
-    echo "Pelican didn't start. Is the Pelican package installed?"
-    return 1
-  elif ! alive $srv_pid ; then
-    echo "The HTTP server didn't start. Is there another service using port" $port "?"
-    return 1
-  fi
-  echo 'Pelican and HTTP server processes now running in background.'
-}
-
-###
-#  MAIN
-###
-[[ ($# -eq 0) || ($# -gt 2) ]] && usage
-port=''
-[[ $# -eq 2 ]] && port=$2
-
-if [[ $1 == "stop" ]]; then
-  shut_down
-elif [[ $1 == "restart" ]]; then
-  shut_down
-  start_up $port
-elif [[ $1 == "start" ]]; then
-  if ! start_up $port; then
-    shut_down
-  fi
-else
-  usage
-fi

pelican/tools/templates/fabfile.py.jinja2

@@ -3,12 +3,6 @@ import fabric.contrib.project as project
 import os
 import shutil
 import sys
-try:
-    import socketserver
-except ImportError:
-    import SocketServer as socketserver
-
-from pelican.server import ComplexHTTPRequestHandler
 
 # Local path configuration (can be absolute or relative to fabfile)
 env.deploy_path = 'output'
@@ -55,15 +49,11 @@ def regenerate():
 
 def serve():
     """Serve site at http://localhost:8000/"""
-    os.chdir(env.deploy_path)
+    local('pelican -l -s pelicanconf.py')
 
-    class AddressReuseTCPServer(socketserver.TCPServer):
+def devserver():
-        allow_reuse_address = True
+    """Serve site at http://localhost:8000/ and regenerate automatically"""
-
+    local('pelican -r -l -s pelicanconf.py')
-    server = AddressReuseTCPServer(('', PORT), ComplexHTTPRequestHandler)
-
-    sys.stderr.write('Serving on port {0} ...\n'.format(PORT))
-    server.serve_forever()
 
 def reserve():
     """`build`, then `serve`"""