diff --git a/docs/install.rst b/docs/install.rst index 02470f62..8075632f 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -103,7 +103,6 @@ can optionally add yourself if you plan to create non-chronological content):: ├── content │   └── (pages) ├── output - ├── develop_server.sh ├── fabfile.py ├── Makefile ├── pelicanconf.py # Main settings file diff --git a/docs/publish.rst b/docs/publish.rst index a9b0ec39..7fb4b587 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -201,10 +201,7 @@ separate terminal sessions, but you can run both at once via:: make devserver The above command will simultaneously run Pelican in regeneration mode as well -as serve the output at http://localhost:8000. Once you are done testing your -changes, you should stop the development server via:: - - ./develop_server.sh stop +as serve the output at http://localhost:8000. When you're ready to publish your site, you can upload it via the method(s) you chose during the ``pelican-quickstart`` questionnaire. For this example, we'll diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 752c1857..022322c1 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -61,11 +61,10 @@ ignored for now.) Preview your site ----------------- -Open a new terminal session and run the following commands to switch to your -``output`` directory and launch Pelican's web server:: +Open a new terminal session, navigate to your site directory and run the +following command to launch Pelican's web server:: - cd ~/projects/yoursite/output - python -m pelican.server + pelican --listen Preview your site by navigating to http://localhost:8000/ in your browser. diff --git a/docs/settings.rst b/docs/settings.rst index 0ae26ca4..dbd960d5 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -328,6 +328,15 @@ Basic settings A list of metadata fields containing reST/Markdown content to be parsed and translated to HTML. +.. data:: PORT = 8000 + + The TCP port to serve content from the output folder via HTTP when pelican + is run with --listen + +.. data:: BIND = '' + + The IP to which to bind the HTTP server. + URL settings ============ diff --git a/pelican/__init__.py b/pelican/__init__.py index 87d8bc24..8dfbb7ba 100644 --- a/pelican/__init__.py +++ b/pelican/__init__.py @@ -5,10 +5,12 @@ import argparse import collections import locale import logging +import multiprocessing import os import re import sys import time +import traceback import six @@ -20,6 +22,7 @@ from pelican.generators import (ArticlesGenerator, PagesGenerator, SourceFileGenerator, StaticGenerator, TemplatePagesGenerator) from pelican.readers import Readers +from pelican.server import ComplexHTTPRequestHandler, RootedHTTPServer from pelican.settings import read_settings from pelican.utils import (clean_output_dir, file_watcher, folder_watcher, maybe_pluralize) @@ -336,7 +339,24 @@ def parse_arguments(): help=('Only enable log de-duplication for levels equal' ' to or above the specified value')) - return parser.parse_args() + parser.add_argument('-l', '--listen', dest='listen', action='store_true', + help='Serve content files via HTTP and port 8000.') + + parser.add_argument('-p', '--port', dest='port', type=int, + help='Port to serve HTTP files at. (default: 8000)') + + parser.add_argument('-b', '--bind', dest='bind', + help='IP to bind to when serving files via HTTP ' + '(default: 127.0.0.1)') + + args = parser.parse_args() + + if args.port is not None and not args.listen: + logger.warning('--port without --listen has no effect') + if args.bind is not None and not args.listen: + logger.warning('--bind without --listen has no effect') + + return args def get_config(args): @@ -359,6 +379,10 @@ def get_config(args): config['WRITE_SELECTED'] = args.selected_paths.split(',') if args.relative_paths: config['RELATIVE_URLS'] = args.relative_paths + if args.port is not None: + config['PORT'] = args.port + if args.bind is not None: + config['BIND'] = args.bind config['DEBUG'] = args.verbosity == logging.DEBUG # argparse returns bytes in Py2. There is no definite answer as to which @@ -391,6 +415,100 @@ def get_instance(args): return cls(settings), settings +def autoreload(watchers, args, old_static, reader_descs, excqueue=None): + while True: + try: + # Check source dir for changed files ending with the given + # extension in the settings. In the theme dir is no such + # restriction; all files are recursively checked if they + # have changed, no matter what extension the filenames + # have. + modified = {k: next(v) for k, v in watchers.items()} + + if modified['settings']: + pelican, settings = get_instance(args) + + # Adjust static watchers if there are any changes + new_static = settings.get("STATIC_PATHS", []) + + # Added static paths + # Add new watchers and set them as modified + new_watchers = set(new_static).difference(old_static) + for static_path in new_watchers: + static_key = '[static]%s' % static_path + watchers[static_key] = folder_watcher( + os.path.join(pelican.path, static_path), + [''], + pelican.ignore_files) + modified[static_key] = next(watchers[static_key]) + + # Removed static paths + # Remove watchers and modified values + old_watchers = set(old_static).difference(new_static) + for static_path in old_watchers: + static_key = '[static]%s' % static_path + watchers.pop(static_key) + modified.pop(static_key) + + # Replace old_static with the new one + old_static = new_static + + if any(modified.values()): + print('\n-> Modified: {}. re-generating...'.format( + ', '.join(k for k, v in modified.items() if v))) + + if modified['content'] is None: + logger.warning( + 'No valid files found in content for ' + + 'the active readers:\n' + + '\n'.join(reader_descs)) + + if modified['theme'] is None: + logger.warning('Empty theme folder. Using `basic` ' + 'theme.') + + pelican.run() + + except KeyboardInterrupt as e: + logger.warning("Keyboard interrupt, quitting.") + if excqueue is not None: + excqueue.put(traceback.format_exception_only(type(e), e)[-1]) + return + + except Exception as e: + if (args.verbosity == logging.DEBUG): + if excqueue is not None: + excqueue.put( + traceback.format_exception_only(type(e), e)[-1]) + else: + raise + logger.warning( + 'Caught exception "%s". Reloading.', e) + + finally: + time.sleep(.5) # sleep to avoid cpu load + + +def listen(server, port, output, excqueue=None): + RootedHTTPServer.allow_reuse_address = True + try: + httpd = RootedHTTPServer( + output, (server, port), ComplexHTTPRequestHandler) + except OSError as e: + logging.error("Could not listen on port %s, server %s.", port, server) + if excqueue is not None: + excqueue.put(traceback.format_exception_only(type(e), e)[-1]) + return + + logging.info("Serving at port %s, server %s.", port, server) + try: + httpd.serve_forever() + except Exception as e: + if excqueue is not None: + excqueue.put(traceback.format_exception_only(type(e), e)[-1]) + return + + def main(): args = parse_arguments() logs_dedup_min_level = getattr(logging, args.logs_dedup_min_level) @@ -426,76 +544,28 @@ def main(): [''], pelican.ignore_files) - if args.autoreload: + if args.autoreload and args.listen: + excqueue = multiprocessing.Queue() + p1 = multiprocessing.Process( + target=autoreload, + args=(watchers, args, old_static, reader_descs, excqueue)) + p2 = multiprocessing.Process( + target=listen, + args=(settings.get('BIND'), settings.get('PORT'), + settings.get("OUTPUT_PATH"), excqueue)) + p1.start() + p2.start() + exc = excqueue.get() + p1.terminate() + p2.terminate() + logger.critical(exc) + elif args.autoreload: print(' --- AutoReload Mode: Monitoring `content`, `theme` and' ' `settings` for changes. ---') - - while True: - try: - # Check source dir for changed files ending with the given - # extension in the settings. In the theme dir is no such - # restriction; all files are recursively checked if they - # have changed, no matter what extension the filenames - # have. - modified = {k: next(v) for k, v in watchers.items()} - - if modified['settings']: - pelican, settings = get_instance(args) - - # Adjust static watchers if there are any changes - new_static = settings.get("STATIC_PATHS", []) - - # Added static paths - # Add new watchers and set them as modified - new_watchers = set(new_static).difference(old_static) - for static_path in new_watchers: - static_key = '[static]%s' % static_path - watchers[static_key] = folder_watcher( - os.path.join(pelican.path, static_path), - [''], - pelican.ignore_files) - modified[static_key] = next(watchers[static_key]) - - # Removed static paths - # Remove watchers and modified values - old_watchers = set(old_static).difference(new_static) - for static_path in old_watchers: - static_key = '[static]%s' % static_path - watchers.pop(static_key) - modified.pop(static_key) - - # Replace old_static with the new one - old_static = new_static - - if any(modified.values()): - print('\n-> Modified: {}. re-generating...'.format( - ', '.join(k for k, v in modified.items() if v))) - - if modified['content'] is None: - logger.warning( - 'No valid files found in content for ' - + 'the active readers:\n' - + '\n'.join(reader_descs)) - - if modified['theme'] is None: - logger.warning('Empty theme folder. Using `basic` ' - 'theme.') - - pelican.run() - - except KeyboardInterrupt: - logger.warning("Keyboard interrupt, quitting.") - break - - except Exception as e: - if (args.verbosity == logging.DEBUG): - raise - logger.warning( - 'Caught exception "%s". Reloading.', e) - - finally: - time.sleep(.5) # sleep to avoid cpu load - + autoreload(watchers, args, old_static, reader_descs) + elif args.listen: + listen(settings.get('BIND'), settings.get('PORT'), + settings.get("OUTPUT_PATH")) else: if next(watchers['content']) is None: logger.warning( diff --git a/pelican/server.py b/pelican/server.py index 99a5ed75..1f4c6041 100644 --- a/pelican/server.py +++ b/pelican/server.py @@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals import argparse import logging import os +import posixpath import ssl import sys @@ -12,8 +13,9 @@ try: except ImportError: magic_from_file = None +from six.moves import BaseHTTPServer from six.moves import SimpleHTTPServer as srvmod -from six.moves import socketserver +from six.moves import urllib def parse_arguments(): @@ -33,6 +35,9 @@ def parse_arguments(): parser.add_argument('--key', default="./key.pem", nargs="?", help='Path to certificate key file. ' + 'Relative to current directory') + parser.add_argument('path', default=".", + help='Path to pelican source directory to serve. ' + + 'Relative to current directory') return parser.parse_args() @@ -40,6 +45,26 @@ class ComplexHTTPRequestHandler(srvmod.SimpleHTTPRequestHandler): SUFFIXES = ['', '.html', '/index.html'] RSTRIP_PATTERNS = ['', '/'] + def translate_path(self, path): + # abandon query parameters + path = path.split('?', 1)[0] + path = path.split('#', 1)[0] + # Don't forget explicit trailing slash when normalizing. Issue17324 + trailing_slash = path.rstrip().endswith('/') + path = urllib.parse.unquote(path) + path = posixpath.normpath(path) + words = path.split('/') + words = filter(None, words) + path = self.base_path + for word in words: + if os.path.dirname(word) or word in (os.curdir, os.pardir): + # Ignore components that are not a simple file/directory name + continue + path = os.path.join(path, word) + if trailing_slash: + path += '/' + return path + def do_GET(self): # cut off a query string if '?' in self.path: @@ -83,11 +108,17 @@ class ComplexHTTPRequestHandler(srvmod.SimpleHTTPRequestHandler): return mimetype +class RootedHTTPServer(BaseHTTPServer.HTTPServer): + def __init__(self, base_path, *args, **kwargs): + BaseHTTPServer.HTTPServer.__init__(self, *args, **kwargs) + self.RequestHandlerClass.base_path = base_path + + if __name__ == '__main__': args = parse_arguments() - socketserver.TCPServer.allow_reuse_address = True + RootedHTTPServer.allow_reuse_address = True try: - httpd = socketserver.TCPServer( + httpd = RootedHTTPServer( (args.server, args.port), ComplexHTTPRequestHandler) if args.ssl: @@ -97,7 +128,6 @@ if __name__ == '__main__': except ssl.SSLError as e: logging.error("Couldn't open certificate file %s or key file %s", args.cert, args.key) - except OSError as e: logging.error("Could not listen on port %s, server %s.", args.port, args.server) sys.exit(getattr(e, 'exitcode', 1)) diff --git a/pelican/settings.py b/pelican/settings.py index eac6ff19..6849249e 100644 --- a/pelican/settings.py +++ b/pelican/settings.py @@ -150,6 +150,8 @@ DEFAULT_CONFIG = { 'LOAD_CONTENT_CACHE': False, 'WRITE_SELECTED': [], 'FORMATTED_FIELDS': ['summary'], + 'PORT': 8000, + 'BIND': '', } PYGMENTS_RST_OPTIONS = None diff --git a/pelican/tools/pelican_quickstart.py b/pelican/tools/pelican_quickstart.py index c77a4226..6ab62f8f 100755 --- a/pelican/tools/pelican_quickstart.py +++ b/pelican/tools/pelican_quickstart.py @@ -263,8 +263,6 @@ needed by Pelican. automation = ask('Do you want to generate a Fabfile/Makefile ' 'to automate generation and publishing?', bool, True) - develop = ask('Do you want an auto-reload HTTP script ' - 'to assist with theme and site development?', bool, True) if automation: if ask('Do you want to upload your website using FTP?', @@ -380,29 +378,6 @@ needed by Pelican. except OSError as e: print('Error: {0}'.format(e)) - if develop: - conf_shell = dict() - for key, value in CONF.items(): - if isinstance(value, six.string_types) and ' ' in value: - value = '"' + value.replace('"', '\\"') + '"' - conf_shell[key] = value - try: - with codecs.open(os.path.join(CONF['basedir'], - 'develop_server.sh'), - 'w', 'utf-8') as fd: - py_v = '${PY:-python}' - if six.PY3: - py_v = '${PY:-python3}' - _template = _jinja_env.get_template('develop_server.sh.jinja2') - fd.write(_template.render(py_v=py_v, **conf_shell)) - fd.close() - - # mode 0o755 - os.chmod((os.path.join(CONF['basedir'], - 'develop_server.sh')), 493) - except OSError as e: - print('Error: {0}'.format(e)) - print('Done. Your new project is available at %s' % CONF['basedir']) diff --git a/pelican/tools/templates/Makefile.jinja2 b/pelican/tools/templates/Makefile.jinja2 index 42424cdd..3e07a367 100644 --- a/pelican/tools/templates/Makefile.jinja2 +++ b/pelican/tools/templates/Makefile.jinja2 @@ -60,9 +60,7 @@ help: @echo ' make publish generate using production settings ' @echo ' make serve [PORT=8000] serve site at http://localhost:8000' @echo ' make serve-global [SERVER=0.0.0.0] serve (as root) to $(SERVER):80 ' - @echo ' make devserver [PORT=8000] start/restart develop_server.sh ' - @echo ' make stopserver stop local server ' -{% if ssh %} + @echo ' make devserver [PORT=8000] serve and regenerate together ' @echo ' make ssh_upload upload the web site via SSH ' @echo ' make rsync_upload upload the web site via rsync+ssh ' {% endif %} @@ -97,30 +95,26 @@ regenerate: serve: ifdef PORT - cd $(OUTPUTDIR) && $(PY) -m pelican.server $(PORT) + $$(PELICAN) -l $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(CONFFILE) $$(PELICANOPTS) -p $$(PORT) else - cd $(OUTPUTDIR) && $(PY) -m pelican.server + $$(PELICAN) -l $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(CONFFILE) $$(PELICANOPTS) endif serve-global: ifdef SERVER - cd $(OUTPUTDIR) && $(PY) -m pelican.server 80 $(SERVER) + $$(PELICAN) -l $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(CONFFILE) $$(PELICANOPTS) -p $$(PORT) -b $$(SERVER) else - cd $(OUTPUTDIR) && $(PY) -m pelican.server 80 0.0.0.0 + $$(PELICAN) -l $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(CONFFILE) $$(PELICANOPTS) -p $$(PORT) -b 0.0.0.0 endif devserver: ifdef PORT - $(BASEDIR)/develop_server.sh restart $(PORT) + $$(PELICAN) -lr $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(CONFFILE) $$(PELICANOPTS) -p $$(PORT) else - $(BASEDIR)/develop_server.sh restart + $$(PELICAN) -lr $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(CONFFILE) $$(PELICANOPTS) endif -stopserver: - $(BASEDIR)/develop_server.sh stop - @echo 'Stopped Pelican and SimpleHTTPServer processes running in background.' - publish: $(PELICAN) $(INPUTDIR) -o $(OUTPUTDIR) -s $(PUBLISHCONF) $(PELICANOPTS) diff --git a/pelican/tools/templates/develop_server.sh.jinja2 b/pelican/tools/templates/develop_server.sh.jinja2 deleted file mode 100755 index 7d20fd22..00000000 --- a/pelican/tools/templates/develop_server.sh.jinja2 +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env bash -## -# This section should match your Makefile -## -PY={{py_v}} -PELICAN=${PELICAN:-pelican} -PELICANOPTS={{pelicanopts}} - -BASEDIR=$(pwd) -INPUTDIR="$BASEDIR"/content -OUTPUTDIR="$BASEDIR"/output -CONFFILE="$BASEDIR"/pelicanconf.py - -### -# Don't change stuff below here unless you are sure -### - -SRV_PID="$BASEDIR"/srv.pid -PELICAN_PID="$BASEDIR"/pelican.pid - -function usage(){ - echo "usage: $0 (stop) (start) (restart) [port]" - echo "This starts Pelican in debug and reload mode and then launches" - echo "an HTTP server to help site development. It doesn't read" - echo "your Pelican settings, so if you edit any paths in your Makefile" - echo "you will need to edit your settings as well." - exit 3 -} - -function alive() { - kill -0 $1 >/dev/null 2>&1 -} - -function shut_down(){ - PID=$(cat "$SRV_PID") - if [[ $? -eq 0 ]]; then - if alive $PID; then - echo "Stopping HTTP server" - kill $PID - else - echo "Stale PID, deleting" - fi - rm "$SRV_PID" - else - echo "HTTP server PIDFile not found" - fi - - PID=$(cat "$PELICAN_PID") - if [[ $? -eq 0 ]]; then - if alive $PID; then - echo "Killing Pelican" - kill $PID - else - echo "Stale PID, deleting" - fi - rm "$PELICAN_PID" - else - echo "Pelican PIDFile not found" - fi -} - -function start_up(){ - local port=$1 - echo "Starting up Pelican and HTTP server" - shift - $PELICAN --debug --autoreload -r "$INPUT"DIR -o "$OUTPUTDIR" -s "$CONFFILE" $PELICANOPTS & - pelican_pid=$! - echo $pelican_pid > "$PELICAN_PID" - mkdir -p "$OUTPUTDIR" && cd "$OUTPUTDIR" - $PY -m pelican.server $port & - srv_pid=$! - echo $srv_pid > "$SRV_PID" - cd "$BASEDIR" - sleep 1 - if ! alive $pelican_pid ; then - echo "Pelican didn't start. Is the Pelican package installed?" - return 1 - elif ! alive $srv_pid ; then - echo "The HTTP server didn't start. Is there another service using port" $port "?" - return 1 - fi - echo 'Pelican and HTTP server processes now running in background.' -} - -### -# MAIN -### -[[ ($# -eq 0) || ($# -gt 2) ]] && usage -port='' -[[ $# -eq 2 ]] && port=$2 - -if [[ $1 == "stop" ]]; then - shut_down -elif [[ $1 == "restart" ]]; then - shut_down - start_up $port -elif [[ $1 == "start" ]]; then - if ! start_up $port; then - shut_down - fi -else - usage -fi diff --git a/pelican/tools/templates/fabfile.py.jinja2 b/pelican/tools/templates/fabfile.py.jinja2 index eda60c42..467f9ec2 100644 --- a/pelican/tools/templates/fabfile.py.jinja2 +++ b/pelican/tools/templates/fabfile.py.jinja2 @@ -3,12 +3,6 @@ import fabric.contrib.project as project import os import shutil import sys -try: - import socketserver -except ImportError: - import SocketServer as socketserver - -from pelican.server import ComplexHTTPRequestHandler # Local path configuration (can be absolute or relative to fabfile) env.deploy_path = 'output' @@ -55,15 +49,11 @@ def regenerate(): def serve(): """Serve site at http://localhost:8000/""" - os.chdir(env.deploy_path) + local('pelican -l -s pelicanconf.py') - class AddressReuseTCPServer(socketserver.TCPServer): - allow_reuse_address = True - - server = AddressReuseTCPServer(('', PORT), ComplexHTTPRequestHandler) - - sys.stderr.write('Serving on port {0} ...\n'.format(PORT)) - server.serve_forever() +def devserver(): + """Serve site at http://localhost:8000/ and regenerate automatically""" + local('pelican -r -l -s pelicanconf.py') def reserve(): """`build`, then `serve`"""