add live page reloading feature #1326

pelican/__init__.py

@@ -22,7 +22,7 @@ from pelican.generators import (ArticlesGenerator, PagesGenerator,
                                 TemplatePagesGenerator)
 from pelican.readers import Readers
 from pelican.settings import read_settings
-from pelican.utils import clean_output_dir, folder_watcher, file_watcher
+from pelican.utils import clean_output_dir, folder_watcher, file_watcher, send_browser_reload_req
 from pelican.writers import Writer
 
 __version__ = "3.5.0"
@@ -146,7 +146,7 @@ class Pelican(object):
         context = self.settings.copy()
         # Share these among all the generators and content objects:
         context['filenames'] = {}  # maps source path to Content object or None
-        context['localsiteurl'] = self.settings['SITEURL'] 
+        context['localsiteurl'] = self.settings['SITEURL']
 
         generators = [
             cls(
@@ -226,7 +226,7 @@ class Pelican(object):
                 logger.debug('Found writer: %s', writer)
             else:
                 logger.warning(
-                    '%s writers found, using only first one: %s', 
+                    '%s writers found, using only first one: %s',
                     writers_found, writer)
             return writer(self.output_path, settings=self.settings)
 
@@ -293,6 +293,11 @@ def parse_arguments():
                         dest='selected_paths', default=None,
                         help='Comma separated list of selected paths to write')
 
+    parser.add_argument('--browser-reload', nargs='?', type=int,
+                        dest='browser_reload',
+                        help='Relaunch pelican and reload browser '
+                        'each time a modification occurs')
+
     return parser.parse_args()
 
 
@@ -401,6 +406,9 @@ def main():
                         # restore original caching policy
                         pelican.settings['LOAD_CONTENT_CACHE'] = original_load_cache
 
+                        if args.browser_reload:
+                            send_browser_reload_req(args.browser_reload)
+
                 except KeyboardInterrupt:
                     logger.warning("Keyboard interrupt, quitting.")
                     break

pelican/dev_server2.py

@@ -0,0 +1,144 @@
+import os
+import sys
+import time
+import subprocess
+import shlex
+import signal
+import six
+
+PY = "python"
+if six.PY3:
+    PY += "3"
+PELICAN = "pelican"
+PELICANOPTS = ""
+
+BASEDIR = os.getcwd()
+INPUTDIR = BASEDIR + "/content"
+OUTPUTDIR = BASEDIR + "/output"
+CONFFILE = BASEDIR + "/pelicanconf.py"
+
+HTTPD_PORT = 8000
+RELOAD_PORT = 9000
+
+PIDS_FILE = BASEDIR + "/devser.pids"
+
+
+def alive(pid):
+    try:
+        os.kill(pid, 0)
+    except OSError:
+        return False
+    else:
+        return True
+
+
+def print_usage():
+    # copy from develop_server.sh
+    message = """usage: $0 (stop) (start) (restart) [httpd_port] [reload_port]
+    This starts Pelican in debug and reload mode and then launches
+    an HTTP server to help site development. It doesn't read
+    your Pelican settings, so if you edit any paths in your Makefile
+    you will need to edit your settings as well.
+    """
+    print(message)
+
+
+def start_up():
+    pelican_com = "{0} --debug --autoreload --browser-reload {1} " \
+                  "-r {2} -o {3} -s {4} {5}" \
+                  .format(PELICAN, RELOAD_PORT,
+                          INPUTDIR, OUTPUTDIR, CONFFILE, PELICANOPTS)
+    pelican_com = shlex.split(pelican_com)
+    p = subprocess.Popen(pelican_com)
+    peli_pid = p.pid
+
+    os.chdir(OUTPUTDIR)
+
+    httpd_com = "{0} -m pelican.reload_httpd {1} {2}" \
+                .format(PY, HTTPD_PORT, RELOAD_PORT)
+    httpd_com = shlex.split(httpd_com)
+    p = subprocess.Popen(httpd_com)
+    http_pid = p.pid
+
+    reload_com = "{0} -m pelican.reload_server {1}".format(PY, RELOAD_PORT)
+    reload_com = shlex.split(reload_com)
+    p = subprocess.Popen(reload_com)
+    relo_pid = p.pid
+
+    os.chdir(BASEDIR)
+
+    with open(PIDS_FILE, "w") as f:
+        f.write("{0},{1},{2}".format(peli_pid, http_pid, relo_pid))
+    # save pids
+
+    time.sleep(1)
+    if not alive(peli_pid):
+        print("Pelican didn't start. Is the Pelican package installed?")
+        return
+    if not alive(http_pid):
+        print("The HTTP server didn't start. "
+              "Is there another service using port {0} ?".format(HTTPD_PORT))
+        return
+    if not alive(relo_pid):
+        print("The reload server didn't start. "
+              "Is there another service using port {0} ?".format(RELOAD_PORT))
+        return
+
+    print('Pelican and HTTP server processes now running in background.')
+
+
+def shut_down():
+    pids = None
+    f = None
+    try:
+        f = open(PIDS_FILE)
+        pids = dict(zip(["peli", "http", "relo"],
+                    map(int, f.read().strip().split(","))))
+    except IOError:
+        print("shutdown servers failed ({0})".format(PIDS_FILE))
+    finally:
+        if f:
+            f.close()
+    if pids:
+        print("shutdown servers ->", pids)
+        pid = pids["peli"]
+        if alive(pid):
+            os.kill(pid, signal.SIGKILL)
+        else:
+            print("pelican already died")
+
+        pid = pids["http"]
+        if alive(pid):
+            os.kill(pid, signal.SIGKILL)
+        else:
+            print("httpd server already died")
+
+        pid = pids["relo"]
+        if alive(pid):
+            os.kill(pid, signal.SIGKILL)
+        else:
+            print("reload server already died")
+
+
+if len(sys.argv) >= 3:
+    HTTPD_PORT = int(sys.argv[2])
+if len(sys.argv) >= 4:
+    RELOAD_PORT = int(sys.argv[3])
+if HTTPD_PORT == RELOAD_PORT:
+    RELOAD_PORT += 1
+    print("conflicting ports, "
+          "fallback http={0} reload={1}".format(HTTPD_PORT, RELOAD_PORT))
+
+if len(sys.argv) >= 2:
+    command = sys.argv[1]
+    if command == "start":
+        start_up()
+    elif command == "restart":
+        shut_down()
+        start_up()
+    elif command == "stop":
+        shut_down()
+    else:
+        print_usage()
+else:
+    print_usage()

pelican/reload_httpd.py

@@ -0,0 +1,175 @@
+from __future__ import print_function
+import codecs
+import os
+import sys
+import logging
+try:
+    import HTMLParser as htmlparser
+except ImportError:
+    import html.parser as htmlparser  # NOQA
+
+try:
+    import SimpleHTTPServer as srvmod
+except ImportError:
+    import http.server as srvmod  # NOQA
+
+try:
+    import SocketServer as socketserver
+except ImportError:
+    import socketserver  # NOQA
+
+PORT = 8000
+RELOAD_PORT = 9000
+
+if len(sys.argv) == 2 or len(sys.argv) == 3:
+    port = int(sys.argv[1])
+    if port:
+        PORT = port
+if len(sys.argv) == 3:
+    re_p = int(sys.argv[2])
+    if re_p:
+        RELOAD_PORT = re_p
+
+if PORT == RELOAD_PORT:
+    RELOAD_PORT = PORT + 1
+    logging.warning("a conflict of ports! fallback: httpd=%d reload=%d",
+                    PORT, RELOAD_PORT)
+
+
+class HeadFinder(htmlparser.HTMLParser):
+    def __init__(self):
+        htmlparser.HTMLParser.__init__(self)
+        self.head_pos = None
+
+    def handle_endtag(self, tag):
+        if tag == "head":
+            self.head_pos = self.getpos()
+
+LIVERELOAD_INJECTION_CODE = """
+<script>
+        var PELICAN_LIVE_RELOAD_SOCKET;
+        function PELICAN_LIVE_RELOAD_SETUP() {
+            PELICAN_LIVE_RELOAD_SOCKET =
+                new WebSocket("ws://localhost:""" + str(RELOAD_PORT) + """");
+            PELICAN_LIVE_RELOAD_SOCKET.onmessage = function(e) {
+                console.log("get message" + e.data);
+                if(e.data == "RELOAD") { window.location.reload(true); }
+            }
+        }
+        function PELICAN_LIVE_RELOAD_CLOSE() {
+            if(PELICAN_LIVE_RELOAD_SOCKET) {
+                if(PELICAN_LIVE_RELOAD_SOCKET.readyState == 1) {
+                    PELICAN_LIVE_RELOAD_SOCKET.close();
+                }
+            }
+        }
+        if(window.addEventListener) {
+            window.addEventListener("load", PELICAN_LIVE_RELOAD_SETUP, false);
+            window.addEventListener("beforeunload",PELICAN_LIVE_RELOAD_CLOSE,false);
+        }
+        else{
+            window.attachEvent("onload", PELICAN_LIVE_RELOAD_SETUP);
+            window.attachEvent("onbeforeunload",PELICAN_LIVE_RELOAD_CLOSE);
+        }
+</script>
+"""
+
+SUFFIXES = ['', '.html', '/index.html']
+
+
+class ComplexHTTPRequestHandler(srvmod.SimpleHTTPRequestHandler):
+    def write_html(self, f):
+        parser = HeadFinder()
+        output = ""
+        for i, line in enumerate(f):
+            if parser.head_pos is None:
+                parser.feed(line)
+                if parser.head_pos is not None:
+                    offset = parser.head_pos[1]
+                    output += line[:offset]
+                    output += LIVERELOAD_INJECTION_CODE
+                    output += line[offset:]
+                else:
+                    output += line
+            else:
+                output += line
+        return output.encode(encoding="utf-8")
+
+    def do_GET2(self):
+        path = self.translate_path(self.path)
+        if os.path.isdir(path):
+            for index in ['index.html', 'index.htm']:
+                _path = os.path.join(path, index)
+                if os.path.exists(_path):
+                    path = _path
+                    break
+            else:
+                self.send_error(404, "File not found")
+                return
+        # check mime type
+        mime = ""
+        ext = os.path.splitext(path)[1].lower()
+        if ext in self.extensions_map:
+            mime = self.extensions_map[ext]
+        else:
+            self.send_error(404, "MIME error")
+            return
+        # open file and response
+        f = None
+        try:
+            if mime == 'text/html':
+                f = codecs.open(path, 'r', encoding="utf-8")
+            else:
+                f = open(path, 'rb')
+        except IOError:
+            self.send_error(404, "File not found")
+            return None
+        # send data
+        self.send_response(200)
+        self.send_header("Content-type", mime)
+        if mime == 'text/html':
+            output = self.write_html(f)
+            self.send_header("Content-Length", len(output))
+            self.end_headers()
+            self.wfile.write(output)
+        else:
+            self.send_header("Content-Length", str(os.path.getsize(path)))
+            self.end_headers()
+            src = f.read()
+            self.request.sendall(src)
+        f.close()
+
+    def do_GET(self):
+        # we are trying to detect the file by having a fallback mechanism
+        found = False
+        for suffix in SUFFIXES:
+            if not hasattr(self, 'original_path'):
+                self.original_path = self.path
+            self.path = self.original_path + suffix
+            print(self.path)
+            path = self.translate_path(self.path)
+            if os.path.exists(path):
+                self.do_GET2()
+                logging.info("Found: %s" % self.path)
+                found = True
+                break
+            logging.info("Tried to find file %s, but it doesn't exist. ", self.path)
+        if not found:
+            logging.warning("Unable to find file %s or variations.", self.path)
+
+Handler = ComplexHTTPRequestHandler
+
+socketserver.TCPServer.allow_reuse_address = True
+try:
+    httpd = socketserver.TCPServer(("", PORT), Handler)
+except OSError as e:
+    logging.error("Could not listen on port %s", PORT)
+    sys.exit(getattr(e, 'exitcode', 1))
+
+
+logging.info("Serving at port %s", PORT)
+try:
+    httpd.serve_forever()
+except KeyboardInterrupt as e:
+    logging.info("Shutting down server")
+    httpd.socket.close()

pelican/reload_server.py

@@ -0,0 +1,152 @@
+import sys
+import re
+import six
+import base64
+import hashlib
+import struct
+import socket
+import time
+import threading
+import logging
+try:
+    import SocketServer as socketserver
+except:
+    import socketserver
+
+last_reload_time = time.time()
+cond_last_reload_time = threading.Condition()
+force_close = False
+
+
+def set_reload_time():
+    global last_reload_time
+    last_reload_time = time.time()
+
+
+def get_reload_time():
+    return last_reload_time
+
+
+class ReloadWebScoketHandler(socketserver.BaseRequestHandler):
+
+    def handle(self):
+        self.request.settimeout(5)
+        data = self.request.recv(4096)
+        if six.PY3:
+            data = data.decode("utf-8")
+        headers = dict(re.findall(r"(.*): (.*)\r\n", data))
+        # if Pelican send reload-request
+        if "PELICAN-RELOAD-REQUEST" in headers:
+            logging.info("reloading browser pages")
+            with cond_last_reload_time:
+                set_reload_time()
+                cond_last_reload_time.notify_all()
+            return
+
+        # is it websocket request?
+        if "Upgrade" not in headers or \
+           headers["Upgrade"] not in "websocket" or \
+           "Sec-WebSocket-Version" not in headers or \
+           headers["Sec-WebSocket-Version"] != "13":
+            return
+        key = ""
+        if "Sec-WebSocket-Key" in headers:
+            key = headers["Sec-WebSocket-Key"]
+        else:
+            return
+        self.ws_handshake(key)
+
+        reload_time = time.time()
+        while(self.heartbeat()):
+            with cond_last_reload_time:
+                cond_last_reload_time.wait(30)
+                if force_close:
+                    break
+                if reload_time < get_reload_time():
+                    self.send_dataframe(0x1, b"RELOAD")
+                    break
+
+    def ws_handshake(self, key):
+        magic = b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
+        h = hashlib.sha1()
+        if isinstance(key, six.text_type):
+            key = key.encode("utf-8")
+        h.update(key)
+        h.update(magic)
+        accept = base64.b64encode(h.digest())
+        output = \
+            b"HTTP/1.1 101 Switching Protocols\r\n" \
+            b"Upgrade: websocket\r\n" \
+            b"Connection: Upgrade\r\n" \
+            b"Sec-WebSocket-Accept: " + accept + b"\r\n\r\n"
+        self.request.sendall(output)
+
+    def send_dataframe(self, opecode, payload):
+        # payload length <= 125
+        b0 = 0x80 + opecode
+        b1 = len(payload)
+        dataframe = struct.pack("!BB", b0, b1) + payload
+        self.request.sendall(dataframe)
+
+    def recv_dataframe(self):
+        # if we recv invalid data, return -1, ""
+        dataframe = self.request.recv(4096)
+        b0, b1 = struct.unpack("!BB", dataframe[0:2])
+        opecode = 0x0f & b0
+        # payload_len = 0x7f & b1
+        # this implement requires [payload <= 125]
+        # 0b 1000 **** 1*** ****
+        if not (0x80 <= b0 <= 0x8f) or not (0x80 <= b1 <= 0xfd) or \
+           not (opecode in [0x1, 0x8, 0xA]):
+            return -1, ""
+        mask = dataframe[2:6]
+        payload = dataframe[6:]
+        if six.PY2:
+            mask = [ord(x) for x in mask]
+            payload = [ord(x) for x in payload]
+        # decode payload
+        dec_payload = ""
+        for i in range(len(payload)):
+            dec_payload += chr(payload[i] ^ mask[i % 4])
+        return opecode, dec_payload
+
+    def heartbeat(self):
+        # checking whether the connection is alive
+        try:
+            self.send_dataframe(0x9, b"PELICAN-PING")
+            opecode, payload = self.recv_dataframe()
+            if opecode == 0x8 or opecode == -1:  # close or invalid
+                self.send_dataframe(0x8, b"")  # close
+                return False
+        except socket.timeout:
+            return False
+        except IOError:
+            return False
+
+        return True
+
+
+class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
+    pass
+socketserver.TCPServer.allow_reuse_address = True
+ThreadedTCPServer.daemon_threads = True
+
+PORT = len(sys.argv) == 2 and int(sys.argv[1]) or 9000
+
+
+try:
+    server = ThreadedTCPServer(("", PORT), ReloadWebScoketHandler)
+except OSError as e:
+    logging.error("Could not listen on port %s", PORT)
+    sys.exit(getattr(e, 'exitcode', 1))
+
+logging.info("Serving at port %s", PORT)
+
+try:
+    server.serve_forever()
+except KeyboardInterrupt as e:
+    logging.info("Shutting down server")
+finally:
+    with cond_last_reload_time:
+        force_close = True
+        cond_last_reload_time.notify_all()

pelican/tools/templates/Makefile.in

@@ -42,6 +42,8 @@ help:
 	@echo '   make serve [PORT=8000]           serve site at http://localhost:8000'
 	@echo '   make devserver [PORT=8000]       start/restart develop_server.sh    '
 	@echo '   make stopserver                  stop local server                  '
+	@echo '   make devserver2_start [PORT=8000,PORT2=9000] start livereload server'
+	@echo '   make devserver2_stop             stop livereload server             '
 	@echo '   make ssh_upload                  upload the web site via SSH        '
 	@echo '   make rsync_upload                upload the web site via rsync+ssh  '
 	@echo '   make dropbox_upload              upload the web site via Dropbox    '
@@ -81,6 +83,12 @@ stopserver:
 	kill -9 `cat srv.pid`
 	@echo 'Stopped Pelican and SimpleHTTPServer processes running in background.'
 
+devserver2_start:
+	$(PY) -m pelican.dev_server2 restart $(PORT) $(PORT2)
+
+devserver2_stop:
+	$(PY) -m pelican.dev_server2 stop
+
 publish:
 	$$(PELICAN) $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(PUBLISHCONF) $$(PELICANOPTS)
 

pelican/utils.py

@@ -15,6 +15,7 @@ import traceback
 import pickle
 import hashlib
 import datetime
+import socket
 
 from collections import Hashable
 from contextlib import contextmanager
@@ -340,7 +341,7 @@ def clean_output_dir(path, retention):
                 shutil.rmtree(file)
                 logger.debug("Deleted directory %s", file)
             except Exception as e:
-                logger.error("Unable to delete directory %s; %s", 
+                logger.error("Unable to delete directory %s; %s",
                         file, e)
         elif os.path.isfile(file) or os.path.islink(file):
             try:
@@ -750,4 +751,17 @@ def is_selected_for_writing(settings, path):
         return path in settings['WRITE_SELECTED']
     else:
         return True
-        
+
+
+def send_browser_reload_req(port):
+    logger.info("sending browser-reload request port=%d", port)
+    s = None
+    try:
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.connect(("", port))
+        s.send(b"PELICAN-RELOAD-REQUEST: true\r\n\r\n")
+    except IOError:
+        logger.warning("sending browser-reload request failed")
+    finally:
+        if s:
+            s.close()