Merge pull request #2351 from Lucas-C/rm_develop_server.sh

Remove develop_server.sh in favour of pelican serving static files itself - close #2176

docs/install.rst

@@ -103,7 +103,6 @@ can optionally add yourself if you plan to create non-chronological content)::
     ├── content
     │   └── (pages)
     ├── output
-    ├── develop_server.sh
     ├── fabfile.py
     ├── Makefile
     ├── pelicanconf.py       # Main settings file

docs/publish.rst

@@ -201,10 +201,7 @@ separate terminal sessions, but you can run both at once via::
     make devserver
 
 The above command will simultaneously run Pelican in regeneration mode as well
-as serve the output at http://localhost:8000. Once you are done testing your
+as serve the output at http://localhost:8000.
-changes, you should stop the development server via::
-
-    ./develop_server.sh stop
 
 When you're ready to publish your site, you can upload it via the method(s) you
 chose during the ``pelican-quickstart`` questionnaire. For this example, we'll

docs/quickstart.rst

@@ -61,11 +61,10 @@ ignored for now.)
 Preview your site
 -----------------
 
-Open a new terminal session and run the following commands to switch to your
+Open a new terminal session, navigate to your site directory and run the
-``output`` directory and launch Pelican's web server::
+following command to launch Pelican's web server::
 
-    cd ~/projects/yoursite/output
+    pelican --listen
-    python -m pelican.server
 
 Preview your site by navigating to http://localhost:8000/ in your browser.
 

docs/settings.rst

@@ -328,6 +328,15 @@ Basic settings
    A list of metadata fields containing reST/Markdown content to be parsed and
    translated to HTML.
 
+.. data:: PORT = 8000
+
+   The TCP port to serve content from the output folder via HTTP when pelican
+   is run with --listen
+
+.. data:: BIND = ''
+
+   The IP to which to bind the HTTP server.
+
 
 URL settings
 ============

pelican/__init__.py

@@ -5,10 +5,12 @@ import argparse
 import collections
 import locale
 import logging
+import multiprocessing
 import os
 import re
 import sys
 import time
+import traceback
 
 import six
 
@@ -20,6 +22,7 @@ from pelican.generators import (ArticlesGenerator, PagesGenerator,
                                 SourceFileGenerator, StaticGenerator,
                                 TemplatePagesGenerator)
 from pelican.readers import Readers
+from pelican.server import ComplexHTTPRequestHandler, RootedHTTPServer
 from pelican.settings import read_settings
 from pelican.utils import (clean_output_dir, file_watcher,
                            folder_watcher, maybe_pluralize)
@@ -336,7 +339,24 @@ def parse_arguments():
                         help=('Only enable log de-duplication for levels equal'
                               ' to or above the specified value'))
 
-    return parser.parse_args()
+    parser.add_argument('-l', '--listen', dest='listen', action='store_true',
+                        help='Serve content files via HTTP and port 8000.')
+
+    parser.add_argument('-p', '--port', dest='port', type=int,
+                        help='Port to serve HTTP files at. (default: 8000)')
+
+    parser.add_argument('-b', '--bind', dest='bind',
+                        help='IP to bind to when serving files via HTTP '
+                        '(default: 127.0.0.1)')
+
+    args = parser.parse_args()
+
+    if args.port is not None and not args.listen:
+        logger.warning('--port without --listen has no effect')
+    if args.bind is not None and not args.listen:
+        logger.warning('--bind without --listen has no effect')
+
+    return args
 
 
 def get_config(args):
@@ -359,6 +379,10 @@ def get_config(args):
         config['WRITE_SELECTED'] = args.selected_paths.split(',')
     if args.relative_paths:
         config['RELATIVE_URLS'] = args.relative_paths
+    if args.port is not None:
+        config['PORT'] = args.port
+    if args.bind is not None:
+        config['BIND'] = args.bind
     config['DEBUG'] = args.verbosity == logging.DEBUG
 
     # argparse returns bytes in Py2. There is no definite answer as to which
@@ -391,6 +415,100 @@ def get_instance(args):
     return cls(settings), settings
 
 
+def autoreload(watchers, args, old_static, reader_descs, excqueue=None):
+    while True:
+        try:
+            # Check source dir for changed files ending with the given
+            # extension in the settings. In the theme dir is no such
+            # restriction; all files are recursively checked if they
+            # have changed, no matter what extension the filenames
+            # have.
+            modified = {k: next(v) for k, v in watchers.items()}
+
+            if modified['settings']:
+                pelican, settings = get_instance(args)
+
+                # Adjust static watchers if there are any changes
+                new_static = settings.get("STATIC_PATHS", [])
+
+                # Added static paths
+                # Add new watchers and set them as modified
+                new_watchers = set(new_static).difference(old_static)
+                for static_path in new_watchers:
+                    static_key = '[static]%s' % static_path
+                    watchers[static_key] = folder_watcher(
+                        os.path.join(pelican.path, static_path),
+                        [''],
+                        pelican.ignore_files)
+                    modified[static_key] = next(watchers[static_key])
+
+                # Removed static paths
+                # Remove watchers and modified values
+                old_watchers = set(old_static).difference(new_static)
+                for static_path in old_watchers:
+                    static_key = '[static]%s' % static_path
+                    watchers.pop(static_key)
+                    modified.pop(static_key)
+
+                # Replace old_static with the new one
+                old_static = new_static
+
+            if any(modified.values()):
+                print('\n-> Modified: {}. re-generating...'.format(
+                    ', '.join(k for k, v in modified.items() if v)))
+
+                if modified['content'] is None:
+                    logger.warning(
+                        'No valid files found in content for '
+                        + 'the active readers:\n'
+                        + '\n'.join(reader_descs))
+
+                if modified['theme'] is None:
+                    logger.warning('Empty theme folder. Using `basic` '
+                                   'theme.')
+
+                pelican.run()
+
+        except KeyboardInterrupt as e:
+            logger.warning("Keyboard interrupt, quitting.")
+            if excqueue is not None:
+                excqueue.put(traceback.format_exception_only(type(e), e)[-1])
+            return
+
+        except Exception as e:
+            if (args.verbosity == logging.DEBUG):
+                if excqueue is not None:
+                    excqueue.put(
+                        traceback.format_exception_only(type(e), e)[-1])
+                else:
+                    raise
+            logger.warning(
+                'Caught exception "%s". Reloading.', e)
+
+        finally:
+            time.sleep(.5)  # sleep to avoid cpu load
+
+
+def listen(server, port, output, excqueue=None):
+    RootedHTTPServer.allow_reuse_address = True
+    try:
+        httpd = RootedHTTPServer(
+            output, (server, port), ComplexHTTPRequestHandler)
+    except OSError as e:
+        logging.error("Could not listen on port %s, server %s.", port, server)
+        if excqueue is not None:
+            excqueue.put(traceback.format_exception_only(type(e), e)[-1])
+        return
+
+    logging.info("Serving at port %s, server %s.", port, server)
+    try:
+        httpd.serve_forever()
+    except Exception as e:
+        if excqueue is not None:
+            excqueue.put(traceback.format_exception_only(type(e), e)[-1])
+        return
+
+
 def main():
     args = parse_arguments()
     logs_dedup_min_level = getattr(logging, args.logs_dedup_min_level)
@@ -426,76 +544,28 @@ def main():
                 [''],
                 pelican.ignore_files)
 
-        if args.autoreload:
+        if args.autoreload and args.listen:
+            excqueue = multiprocessing.Queue()
+            p1 = multiprocessing.Process(
+                target=autoreload,
+                args=(watchers, args, old_static, reader_descs, excqueue))
+            p2 = multiprocessing.Process(
+                target=listen,
+                args=(settings.get('BIND'), settings.get('PORT'),
+                      settings.get("OUTPUT_PATH"), excqueue))
+            p1.start()
+            p2.start()
+            exc = excqueue.get()
+            p1.terminate()
+            p2.terminate()
+            logger.critical(exc)
+        elif args.autoreload:
             print('  --- AutoReload Mode: Monitoring `content`, `theme` and'
                   ' `settings` for changes. ---')
-
+            autoreload(watchers, args, old_static, reader_descs)
-            while True:
+        elif args.listen:
-                try:
+            listen(settings.get('BIND'), settings.get('PORT'),
-                    # Check source dir for changed files ending with the given
+                   settings.get("OUTPUT_PATH"))
-                    # extension in the settings. In the theme dir is no such
-                    # restriction; all files are recursively checked if they
-                    # have changed, no matter what extension the filenames
-                    # have.
-                    modified = {k: next(v) for k, v in watchers.items()}
-
-                    if modified['settings']:
-                        pelican, settings = get_instance(args)
-
-                        # Adjust static watchers if there are any changes
-                        new_static = settings.get("STATIC_PATHS", [])
-
-                        # Added static paths
-                        # Add new watchers and set them as modified
-                        new_watchers = set(new_static).difference(old_static)
-                        for static_path in new_watchers:
-                            static_key = '[static]%s' % static_path
-                            watchers[static_key] = folder_watcher(
-                                os.path.join(pelican.path, static_path),
-                                [''],
-                                pelican.ignore_files)
-                            modified[static_key] = next(watchers[static_key])
-
-                        # Removed static paths
-                        # Remove watchers and modified values
-                        old_watchers = set(old_static).difference(new_static)
-                        for static_path in old_watchers:
-                            static_key = '[static]%s' % static_path
-                            watchers.pop(static_key)
-                            modified.pop(static_key)
-
-                        # Replace old_static with the new one
-                        old_static = new_static
-
-                    if any(modified.values()):
-                        print('\n-> Modified: {}. re-generating...'.format(
-                            ', '.join(k for k, v in modified.items() if v)))
-
-                        if modified['content'] is None:
-                            logger.warning(
-                                'No valid files found in content for '
-                                + 'the active readers:\n'
-                                + '\n'.join(reader_descs))
-
-                        if modified['theme'] is None:
-                            logger.warning('Empty theme folder. Using `basic` '
-                                           'theme.')
-
-                        pelican.run()
-
-                except KeyboardInterrupt:
-                    logger.warning("Keyboard interrupt, quitting.")
-                    break
-
-                except Exception as e:
-                    if (args.verbosity == logging.DEBUG):
-                        raise
-                    logger.warning(
-                        'Caught exception "%s". Reloading.', e)
-
-                finally:
-                    time.sleep(.5)  # sleep to avoid cpu load
-
         else:
             if next(watchers['content']) is None:
                 logger.warning(

pelican/server.py

@@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals
 import argparse
 import logging
 import os
+import posixpath
 import ssl
 import sys
 
@@ -12,8 +13,9 @@ try:
 except ImportError:
     magic_from_file = None
 
+from six.moves import BaseHTTPServer
 from six.moves import SimpleHTTPServer as srvmod
-from six.moves import socketserver
+from six.moves import urllib
 
 
 def parse_arguments():
@@ -33,6 +35,9 @@ def parse_arguments():
     parser.add_argument('--key', default="./key.pem", nargs="?",
                         help='Path to certificate key file. ' +
                         'Relative to current directory')
+    parser.add_argument('path', default=".",
+                        help='Path to pelican source directory to serve. ' +
+                        'Relative to current directory')
     return parser.parse_args()
 
 
@@ -40,6 +45,26 @@ class ComplexHTTPRequestHandler(srvmod.SimpleHTTPRequestHandler):
     SUFFIXES = ['', '.html', '/index.html']
     RSTRIP_PATTERNS = ['', '/']
 
+    def translate_path(self, path):
+        # abandon query parameters
+        path = path.split('?', 1)[0]
+        path = path.split('#', 1)[0]
+        # Don't forget explicit trailing slash when normalizing. Issue17324
+        trailing_slash = path.rstrip().endswith('/')
+        path = urllib.parse.unquote(path)
+        path = posixpath.normpath(path)
+        words = path.split('/')
+        words = filter(None, words)
+        path = self.base_path
+        for word in words:
+            if os.path.dirname(word) or word in (os.curdir, os.pardir):
+                # Ignore components that are not a simple file/directory name
+                continue
+            path = os.path.join(path, word)
+        if trailing_slash:
+            path += '/'
+        return path
+
     def do_GET(self):
         # cut off a query string
         if '?' in self.path:
@@ -83,11 +108,17 @@ class ComplexHTTPRequestHandler(srvmod.SimpleHTTPRequestHandler):
         return mimetype
 
 
+class RootedHTTPServer(BaseHTTPServer.HTTPServer):
+    def __init__(self, base_path, *args, **kwargs):
+        BaseHTTPServer.HTTPServer.__init__(self, *args, **kwargs)
+        self.RequestHandlerClass.base_path = base_path
+
+
 if __name__ == '__main__':
     args = parse_arguments()
-    socketserver.TCPServer.allow_reuse_address = True
+    RootedHTTPServer.allow_reuse_address = True
     try:
-        httpd = socketserver.TCPServer(
+        httpd = RootedHTTPServer(
             (args.server, args.port),
             ComplexHTTPRequestHandler)
         if args.ssl:
@@ -97,7 +128,6 @@ if __name__ == '__main__':
     except ssl.SSLError as e:
         logging.error("Couldn't open certificate file %s or key file %s",
                       args.cert, args.key)
-    except OSError as e:
         logging.error("Could not listen on port %s, server %s.",
                       args.port, args.server)
         sys.exit(getattr(e, 'exitcode', 1))

pelican/settings.py

@@ -150,6 +150,8 @@ DEFAULT_CONFIG = {
     'LOAD_CONTENT_CACHE': False,
     'WRITE_SELECTED': [],
     'FORMATTED_FIELDS': ['summary'],
+    'PORT': 8000,
+    'BIND': '',
 }
 
 PYGMENTS_RST_OPTIONS = None

pelican/tools/pelican_quickstart.py

@@ -263,8 +263,6 @@ needed by Pelican.
 
     automation = ask('Do you want to generate a Fabfile/Makefile '
                      'to automate generation and publishing?', bool, True)
-    develop = ask('Do you want an auto-reload HTTP script '
-                  'to assist with theme and site development?', bool, True)
 
     if automation:
         if ask('Do you want to upload your website using FTP?',
@@ -380,29 +378,6 @@ needed by Pelican.
         except OSError as e:
             print('Error: {0}'.format(e))
 
-    if develop:
-        conf_shell = dict()
-        for key, value in CONF.items():
-            if isinstance(value, six.string_types) and ' ' in value:
-                value = '"' + value.replace('"', '\\"') + '"'
-            conf_shell[key] = value
-        try:
-            with codecs.open(os.path.join(CONF['basedir'],
-                                          'develop_server.sh'),
-                             'w', 'utf-8') as fd:
-                py_v = '${PY:-python}'
-                if six.PY3:
-                    py_v = '${PY:-python3}'
-                _template = _jinja_env.get_template('develop_server.sh.jinja2')
-                fd.write(_template.render(py_v=py_v, **conf_shell))
-                fd.close()
-
-                # mode 0o755
-                os.chmod((os.path.join(CONF['basedir'],
-                                       'develop_server.sh')), 493)
-        except OSError as e:
-            print('Error: {0}'.format(e))
-
     print('Done. Your new project is available at %s' % CONF['basedir'])
 
 

pelican/tools/templates/Makefile.jinja2

@@ -60,9 +60,7 @@ help:
 	@echo '   make publish                        generate using production settings '
 	@echo '   make serve [PORT=8000]              serve site at http://localhost:8000'
 	@echo '   make serve-global [SERVER=0.0.0.0]  serve (as root) to $(SERVER):80    '
-	@echo '   make devserver [PORT=8000]          start/restart develop_server.sh    '
+	@echo '   make devserver [PORT=8000]          serve and regenerate together      '
-	@echo '   make stopserver                     stop local server                  '
-{% if ssh %}
 	@echo '   make ssh_upload                     upload the web site via SSH        '
 	@echo '   make rsync_upload                   upload the web site via rsync+ssh  '
 {% endif %}
@@ -97,30 +95,26 @@ regenerate:
 
 serve:
 ifdef PORT
-	cd $(OUTPUTDIR) && $(PY) -m pelican.server $(PORT)
+	$$(PELICAN) -l $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(CONFFILE) $$(PELICANOPTS) -p $$(PORT)
 else
-	cd $(OUTPUTDIR) && $(PY) -m pelican.server
+	$$(PELICAN) -l $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(CONFFILE) $$(PELICANOPTS)
 endif
 
 serve-global:
 ifdef SERVER
-	cd $(OUTPUTDIR) && $(PY) -m pelican.server 80 $(SERVER)
+	$$(PELICAN) -l $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(CONFFILE) $$(PELICANOPTS) -p $$(PORT) -b $$(SERVER)
 else
-	cd $(OUTPUTDIR) && $(PY) -m pelican.server 80 0.0.0.0
+	$$(PELICAN) -l $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(CONFFILE) $$(PELICANOPTS) -p $$(PORT) -b 0.0.0.0
 endif
 
 
 devserver:
 ifdef PORT
-	$(BASEDIR)/develop_server.sh restart $(PORT)
+	$$(PELICAN) -lr $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(CONFFILE) $$(PELICANOPTS) -p $$(PORT)
 else
-	$(BASEDIR)/develop_server.sh restart
+	$$(PELICAN) -lr $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(CONFFILE) $$(PELICANOPTS)
 endif
 
-stopserver:
-	$(BASEDIR)/develop_server.sh stop
-	@echo 'Stopped Pelican and SimpleHTTPServer processes running in background.'
-
 publish:
 	$(PELICAN) $(INPUTDIR) -o $(OUTPUTDIR) -s $(PUBLISHCONF) $(PELICANOPTS)
 

pelican/tools/templates/develop_server.sh.jinja2

@@ -1,103 +0,0 @@
-#!/usr/bin/env bash
-##
-# This section should match your Makefile
-##
-PY={{py_v}}
-PELICAN=${PELICAN:-pelican}
-PELICANOPTS={{pelicanopts}}
-
-BASEDIR=$(pwd)
-INPUTDIR="$BASEDIR"/content
-OUTPUTDIR="$BASEDIR"/output
-CONFFILE="$BASEDIR"/pelicanconf.py
-
-###
-# Don't change stuff below here unless you are sure
-###
-
-SRV_PID="$BASEDIR"/srv.pid
-PELICAN_PID="$BASEDIR"/pelican.pid
-
-function usage(){
-  echo "usage: $0 (stop) (start) (restart) [port]"
-  echo "This starts Pelican in debug and reload mode and then launches"
-  echo "an HTTP server to help site development. It doesn't read"
-  echo "your Pelican settings, so if you edit any paths in your Makefile"
-  echo "you will need to edit your settings as well."
-  exit 3
-}
-
-function alive() {
-  kill -0 $1 >/dev/null 2>&1
-}
-
-function shut_down(){
-  PID=$(cat "$SRV_PID")
-  if [[ $? -eq 0 ]]; then
-    if alive $PID; then
-      echo "Stopping HTTP server"
-      kill $PID
-    else
-      echo "Stale PID, deleting"
-    fi
-    rm "$SRV_PID"
-  else
-    echo "HTTP server PIDFile not found"
-  fi
-
-  PID=$(cat "$PELICAN_PID")
-  if [[ $? -eq 0 ]]; then
-    if alive $PID; then
-      echo "Killing Pelican"
-      kill $PID
-    else
-      echo "Stale PID, deleting"
-    fi
-    rm "$PELICAN_PID"
-  else
-    echo "Pelican PIDFile not found"
-  fi
-}
-
-function start_up(){
-  local port=$1
-  echo "Starting up Pelican and HTTP server"
-  shift
-  $PELICAN --debug --autoreload -r "$INPUT"DIR -o "$OUTPUTDIR" -s "$CONFFILE" $PELICANOPTS &
-  pelican_pid=$!
-  echo $pelican_pid > "$PELICAN_PID"
-  mkdir -p "$OUTPUTDIR" && cd "$OUTPUTDIR"
-  $PY -m pelican.server $port &
-  srv_pid=$!
-  echo $srv_pid > "$SRV_PID"
-  cd "$BASEDIR"
-  sleep 1
-  if ! alive $pelican_pid ; then
-    echo "Pelican didn't start. Is the Pelican package installed?"
-    return 1
-  elif ! alive $srv_pid ; then
-    echo "The HTTP server didn't start. Is there another service using port" $port "?"
-    return 1
-  fi
-  echo 'Pelican and HTTP server processes now running in background.'
-}
-
-###
-#  MAIN
-###
-[[ ($# -eq 0) || ($# -gt 2) ]] && usage
-port=''
-[[ $# -eq 2 ]] && port=$2
-
-if [[ $1 == "stop" ]]; then
-  shut_down
-elif [[ $1 == "restart" ]]; then
-  shut_down
-  start_up $port
-elif [[ $1 == "start" ]]; then
-  if ! start_up $port; then
-    shut_down
-  fi
-else
-  usage
-fi

pelican/tools/templates/fabfile.py.jinja2

@@ -3,12 +3,6 @@ import fabric.contrib.project as project
 import os
 import shutil
 import sys
-try:
-    import socketserver
-except ImportError:
-    import SocketServer as socketserver
-
-from pelican.server import ComplexHTTPRequestHandler
 
 # Local path configuration (can be absolute or relative to fabfile)
 env.deploy_path = 'output'
@@ -55,15 +49,11 @@ def regenerate():
 
 def serve():
     """Serve site at http://localhost:8000/"""
-    os.chdir(env.deploy_path)
+    local('pelican -l -s pelicanconf.py')
 
-    class AddressReuseTCPServer(socketserver.TCPServer):
+def devserver():
-        allow_reuse_address = True
+    """Serve site at http://localhost:8000/ and regenerate automatically"""
-
+    local('pelican -r -l -s pelicanconf.py')
-    server = AddressReuseTCPServer(('', PORT), ComplexHTTPRequestHandler)
-
-    sys.stderr.write('Serving on port {0} ...\n'.format(PORT))
-    server.serve_forever()
 
 def reserve():
     """`build`, then `serve`"""