diff --git a/pelican/__init__.py b/pelican/__init__.py index 967360c9..21e877c0 100644 --- a/pelican/__init__.py +++ b/pelican/__init__.py @@ -22,7 +22,7 @@ from pelican.generators import (ArticlesGenerator, PagesGenerator, TemplatePagesGenerator) from pelican.readers import Readers from pelican.settings import read_settings -from pelican.utils import clean_output_dir, folder_watcher, file_watcher +from pelican.utils import clean_output_dir, folder_watcher, file_watcher, send_browser_reload_req from pelican.writers import Writer __version__ = "3.5.0" @@ -146,7 +146,7 @@ class Pelican(object): context = self.settings.copy() # Share these among all the generators and content objects: context['filenames'] = {} # maps source path to Content object or None - context['localsiteurl'] = self.settings['SITEURL'] + context['localsiteurl'] = self.settings['SITEURL'] generators = [ cls( @@ -226,7 +226,7 @@ class Pelican(object): logger.debug('Found writer: %s', writer) else: logger.warning( - '%s writers found, using only first one: %s', + '%s writers found, using only first one: %s', writers_found, writer) return writer(self.output_path, settings=self.settings) @@ -293,6 +293,11 @@ def parse_arguments(): dest='selected_paths', default=None, help='Comma separated list of selected paths to write') + parser.add_argument('--browser-reload', nargs='?', type=int, + dest='browser_reload', + help='Relaunch pelican and reload browser ' + 'each time a modification occurs') + return parser.parse_args() @@ -401,6 +406,9 @@ def main(): # restore original caching policy pelican.settings['LOAD_CONTENT_CACHE'] = original_load_cache + if args.browser_reload: + send_browser_reload_req(args.browser_reload) + except KeyboardInterrupt: logger.warning("Keyboard interrupt, quitting.") break diff --git a/pelican/dev_server2.py b/pelican/dev_server2.py new file mode 100644 index 00000000..233bb5e0 --- /dev/null +++ b/pelican/dev_server2.py @@ -0,0 +1,144 @@ +import os +import sys +import time +import subprocess +import shlex +import signal +import six + +PY = "python" +if six.PY3: + PY += "3" +PELICAN = "pelican" +PELICANOPTS = "" + +BASEDIR = os.getcwd() +INPUTDIR = BASEDIR + "/content" +OUTPUTDIR = BASEDIR + "/output" +CONFFILE = BASEDIR + "/pelicanconf.py" + +HTTPD_PORT = 8000 +RELOAD_PORT = 9000 + +PIDS_FILE = BASEDIR + "/devser.pids" + + +def alive(pid): + try: + os.kill(pid, 0) + except OSError: + return False + else: + return True + + +def print_usage(): + # copy from develop_server.sh + message = """usage: $0 (stop) (start) (restart) [httpd_port] [reload_port] + This starts Pelican in debug and reload mode and then launches + an HTTP server to help site development. It doesn't read + your Pelican settings, so if you edit any paths in your Makefile + you will need to edit your settings as well. + """ + print(message) + + +def start_up(): + pelican_com = "{0} --debug --autoreload --browser-reload {1} " \ + "-r {2} -o {3} -s {4} {5}" \ + .format(PELICAN, RELOAD_PORT, + INPUTDIR, OUTPUTDIR, CONFFILE, PELICANOPTS) + pelican_com = shlex.split(pelican_com) + p = subprocess.Popen(pelican_com) + peli_pid = p.pid + + os.chdir(OUTPUTDIR) + + httpd_com = "{0} -m pelican.reload_httpd {1} {2}" \ + .format(PY, HTTPD_PORT, RELOAD_PORT) + httpd_com = shlex.split(httpd_com) + p = subprocess.Popen(httpd_com) + http_pid = p.pid + + reload_com = "{0} -m pelican.reload_server {1}".format(PY, RELOAD_PORT) + reload_com = shlex.split(reload_com) + p = subprocess.Popen(reload_com) + relo_pid = p.pid + + os.chdir(BASEDIR) + + with open(PIDS_FILE, "w") as f: + f.write("{0},{1},{2}".format(peli_pid, http_pid, relo_pid)) + # save pids + + time.sleep(1) + if not alive(peli_pid): + print("Pelican didn't start. Is the Pelican package installed?") + return + if not alive(http_pid): + print("The HTTP server didn't start. " + "Is there another service using port {0} ?".format(HTTPD_PORT)) + return + if not alive(relo_pid): + print("The reload server didn't start. " + "Is there another service using port {0} ?".format(RELOAD_PORT)) + return + + print('Pelican and HTTP server processes now running in background.') + + +def shut_down(): + pids = None + f = None + try: + f = open(PIDS_FILE) + pids = dict(zip(["peli", "http", "relo"], + map(int, f.read().strip().split(",")))) + except IOError: + print("shutdown servers failed ({0})".format(PIDS_FILE)) + finally: + if f: + f.close() + if pids: + print("shutdown servers ->", pids) + pid = pids["peli"] + if alive(pid): + os.kill(pid, signal.SIGKILL) + else: + print("pelican already died") + + pid = pids["http"] + if alive(pid): + os.kill(pid, signal.SIGKILL) + else: + print("httpd server already died") + + pid = pids["relo"] + if alive(pid): + os.kill(pid, signal.SIGKILL) + else: + print("reload server already died") + + +if len(sys.argv) >= 3: + HTTPD_PORT = int(sys.argv[2]) +if len(sys.argv) >= 4: + RELOAD_PORT = int(sys.argv[3]) +if HTTPD_PORT == RELOAD_PORT: + RELOAD_PORT += 1 + print("conflicting ports, " + "fallback http={0} reload={1}".format(HTTPD_PORT, RELOAD_PORT)) + +if len(sys.argv) >= 2: + command = sys.argv[1] + if command == "start": + start_up() + elif command == "restart": + shut_down() + start_up() + elif command == "stop": + shut_down() + else: + print_usage() +else: + print_usage() diff --git a/pelican/reload_httpd.py b/pelican/reload_httpd.py new file mode 100644 index 00000000..7bb2dccf --- /dev/null +++ b/pelican/reload_httpd.py @@ -0,0 +1,175 @@ +from __future__ import print_function +import codecs +import os +import sys +import logging +try: + import HTMLParser as htmlparser +except ImportError: + import html.parser as htmlparser # NOQA + +try: + import SimpleHTTPServer as srvmod +except ImportError: + import http.server as srvmod # NOQA + +try: + import SocketServer as socketserver +except ImportError: + import socketserver # NOQA + +PORT = 8000 +RELOAD_PORT = 9000 + +if len(sys.argv) == 2 or len(sys.argv) == 3: + port = int(sys.argv[1]) + if port: + PORT = port +if len(sys.argv) == 3: + re_p = int(sys.argv[2]) + if re_p: + RELOAD_PORT = re_p + +if PORT == RELOAD_PORT: + RELOAD_PORT = PORT + 1 + logging.warning("a conflict of ports! fallback: httpd=%d reload=%d", + PORT, RELOAD_PORT) + + +class HeadFinder(htmlparser.HTMLParser): + def __init__(self): + htmlparser.HTMLParser.__init__(self) + self.head_pos = None + + def handle_endtag(self, tag): + if tag == "head": + self.head_pos = self.getpos() + +LIVERELOAD_INJECTION_CODE = """ + +""" + +SUFFIXES = ['', '.html', '/index.html'] + + +class ComplexHTTPRequestHandler(srvmod.SimpleHTTPRequestHandler): + def write_html(self, f): + parser = HeadFinder() + output = "" + for i, line in enumerate(f): + if parser.head_pos is None: + parser.feed(line) + if parser.head_pos is not None: + offset = parser.head_pos[1] + output += line[:offset] + output += LIVERELOAD_INJECTION_CODE + output += line[offset:] + else: + output += line + else: + output += line + return output.encode(encoding="utf-8") + + def do_GET2(self): + path = self.translate_path(self.path) + if os.path.isdir(path): + for index in ['index.html', 'index.htm']: + _path = os.path.join(path, index) + if os.path.exists(_path): + path = _path + break + else: + self.send_error(404, "File not found") + return + # check mime type + mime = "" + ext = os.path.splitext(path)[1].lower() + if ext in self.extensions_map: + mime = self.extensions_map[ext] + else: + self.send_error(404, "MIME error") + return + # open file and response + f = None + try: + if mime == 'text/html': + f = codecs.open(path, 'r', encoding="utf-8") + else: + f = open(path, 'rb') + except IOError: + self.send_error(404, "File not found") + return None + # send data + self.send_response(200) + self.send_header("Content-type", mime) + if mime == 'text/html': + output = self.write_html(f) + self.send_header("Content-Length", len(output)) + self.end_headers() + self.wfile.write(output) + else: + self.send_header("Content-Length", str(os.path.getsize(path))) + self.end_headers() + src = f.read() + self.request.sendall(src) + f.close() + + def do_GET(self): + # we are trying to detect the file by having a fallback mechanism + found = False + for suffix in SUFFIXES: + if not hasattr(self, 'original_path'): + self.original_path = self.path + self.path = self.original_path + suffix + print(self.path) + path = self.translate_path(self.path) + if os.path.exists(path): + self.do_GET2() + logging.info("Found: %s" % self.path) + found = True + break + logging.info("Tried to find file %s, but it doesn't exist. ", self.path) + if not found: + logging.warning("Unable to find file %s or variations.", self.path) + +Handler = ComplexHTTPRequestHandler + +socketserver.TCPServer.allow_reuse_address = True +try: + httpd = socketserver.TCPServer(("", PORT), Handler) +except OSError as e: + logging.error("Could not listen on port %s", PORT) + sys.exit(getattr(e, 'exitcode', 1)) + + +logging.info("Serving at port %s", PORT) +try: + httpd.serve_forever() +except KeyboardInterrupt as e: + logging.info("Shutting down server") + httpd.socket.close() diff --git a/pelican/reload_server.py b/pelican/reload_server.py new file mode 100644 index 00000000..57a01c1a --- /dev/null +++ b/pelican/reload_server.py @@ -0,0 +1,152 @@ +import sys +import re +import six +import base64 +import hashlib +import struct +import socket +import time +import threading +import logging +try: + import SocketServer as socketserver +except: + import socketserver + +last_reload_time = time.time() +cond_last_reload_time = threading.Condition() +force_close = False + + +def set_reload_time(): + global last_reload_time + last_reload_time = time.time() + + +def get_reload_time(): + return last_reload_time + + +class ReloadWebScoketHandler(socketserver.BaseRequestHandler): + + def handle(self): + self.request.settimeout(5) + data = self.request.recv(4096) + if six.PY3: + data = data.decode("utf-8") + headers = dict(re.findall(r"(.*): (.*)\r\n", data)) + # if Pelican send reload-request + if "PELICAN-RELOAD-REQUEST" in headers: + logging.info("reloading browser pages") + with cond_last_reload_time: + set_reload_time() + cond_last_reload_time.notify_all() + return + + # is it websocket request? + if "Upgrade" not in headers or \ + headers["Upgrade"] not in "websocket" or \ + "Sec-WebSocket-Version" not in headers or \ + headers["Sec-WebSocket-Version"] != "13": + return + key = "" + if "Sec-WebSocket-Key" in headers: + key = headers["Sec-WebSocket-Key"] + else: + return + self.ws_handshake(key) + + reload_time = time.time() + while(self.heartbeat()): + with cond_last_reload_time: + cond_last_reload_time.wait(30) + if force_close: + break + if reload_time < get_reload_time(): + self.send_dataframe(0x1, b"RELOAD") + break + + def ws_handshake(self, key): + magic = b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + h = hashlib.sha1() + if isinstance(key, six.text_type): + key = key.encode("utf-8") + h.update(key) + h.update(magic) + accept = base64.b64encode(h.digest()) + output = \ + b"HTTP/1.1 101 Switching Protocols\r\n" \ + b"Upgrade: websocket\r\n" \ + b"Connection: Upgrade\r\n" \ + b"Sec-WebSocket-Accept: " + accept + b"\r\n\r\n" + self.request.sendall(output) + + def send_dataframe(self, opecode, payload): + # payload length <= 125 + b0 = 0x80 + opecode + b1 = len(payload) + dataframe = struct.pack("!BB", b0, b1) + payload + self.request.sendall(dataframe) + + def recv_dataframe(self): + # if we recv invalid data, return -1, "" + dataframe = self.request.recv(4096) + b0, b1 = struct.unpack("!BB", dataframe[0:2]) + opecode = 0x0f & b0 + # payload_len = 0x7f & b1 + # this implement requires [payload <= 125] + # 0b 1000 **** 1*** **** + if not (0x80 <= b0 <= 0x8f) or not (0x80 <= b1 <= 0xfd) or \ + not (opecode in [0x1, 0x8, 0xA]): + return -1, "" + mask = dataframe[2:6] + payload = dataframe[6:] + if six.PY2: + mask = [ord(x) for x in mask] + payload = [ord(x) for x in payload] + # decode payload + dec_payload = "" + for i in range(len(payload)): + dec_payload += chr(payload[i] ^ mask[i % 4]) + return opecode, dec_payload + + def heartbeat(self): + # checking whether the connection is alive + try: + self.send_dataframe(0x9, b"PELICAN-PING") + opecode, payload = self.recv_dataframe() + if opecode == 0x8 or opecode == -1: # close or invalid + self.send_dataframe(0x8, b"") # close + return False + except socket.timeout: + return False + except IOError: + return False + + return True + + +class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): + pass +socketserver.TCPServer.allow_reuse_address = True +ThreadedTCPServer.daemon_threads = True + +PORT = len(sys.argv) == 2 and int(sys.argv[1]) or 9000 + + +try: + server = ThreadedTCPServer(("", PORT), ReloadWebScoketHandler) +except OSError as e: + logging.error("Could not listen on port %s", PORT) + sys.exit(getattr(e, 'exitcode', 1)) + +logging.info("Serving at port %s", PORT) + +try: + server.serve_forever() +except KeyboardInterrupt as e: + logging.info("Shutting down server") +finally: + with cond_last_reload_time: + force_close = True + cond_last_reload_time.notify_all() diff --git a/pelican/tools/templates/Makefile.in b/pelican/tools/templates/Makefile.in index 8534595e..f963ce36 100644 --- a/pelican/tools/templates/Makefile.in +++ b/pelican/tools/templates/Makefile.in @@ -42,6 +42,8 @@ help: @echo ' make serve [PORT=8000] serve site at http://localhost:8000' @echo ' make devserver [PORT=8000] start/restart develop_server.sh ' @echo ' make stopserver stop local server ' + @echo ' make devserver2_start [PORT=8000,PORT2=9000] start livereload server' + @echo ' make devserver2_stop stop livereload server ' @echo ' make ssh_upload upload the web site via SSH ' @echo ' make rsync_upload upload the web site via rsync+ssh ' @echo ' make dropbox_upload upload the web site via Dropbox ' @@ -81,6 +83,12 @@ stopserver: kill -9 `cat srv.pid` @echo 'Stopped Pelican and SimpleHTTPServer processes running in background.' +devserver2_start: + $(PY) -m pelican.dev_server2 restart $(PORT) $(PORT2) + +devserver2_stop: + $(PY) -m pelican.dev_server2 stop + publish: $$(PELICAN) $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(PUBLISHCONF) $$(PELICANOPTS) diff --git a/pelican/utils.py b/pelican/utils.py index f216b027..34ab2580 100644 --- a/pelican/utils.py +++ b/pelican/utils.py @@ -15,6 +15,7 @@ import traceback import pickle import hashlib import datetime +import socket from collections import Hashable from contextlib import contextmanager @@ -340,7 +341,7 @@ def clean_output_dir(path, retention): shutil.rmtree(file) logger.debug("Deleted directory %s", file) except Exception as e: - logger.error("Unable to delete directory %s; %s", + logger.error("Unable to delete directory %s; %s", file, e) elif os.path.isfile(file) or os.path.islink(file): try: @@ -750,4 +751,17 @@ def is_selected_for_writing(settings, path): return path in settings['WRITE_SELECTED'] else: return True - + + +def send_browser_reload_req(port): + logger.info("sending browser-reload request port=%d", port) + s = None + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect(("", port)) + s.send(b"PELICAN-RELOAD-REQUEST: true\r\n\r\n") + except IOError: + logger.warning("sending browser-reload request failed") + finally: + if s: + s.close()