diff --git a/pelican/tools/__init__.py b/pelican/tools/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/pelican/tools/pelican_import.py b/pelican/tools/pelican_import.py deleted file mode 100755 index b74da750..00000000 --- a/pelican/tools/pelican_import.py +++ /dev/null @@ -1,1027 +0,0 @@ -#!/usr/bin/env python - -import argparse -import logging -import os -import re -import subprocess -import sys -import time -from collections import defaultdict -from html import unescape -from urllib.error import URLError -from urllib.parse import quote, urlparse, urlsplit, urlunsplit -from urllib.request import urlretrieve - -# because logging.setLoggerClass has to be called before logging.getLogger -from pelican.log import init -from pelican.settings import read_settings -from pelican.utils import SafeDatetime, slugify - - -logger = logging.getLogger(__name__) - - -def decode_wp_content(content, br=True): - pre_tags = {} - if content.strip() == "": - return "" - - content += "\n" - if "") - last_pre = pre_parts.pop() - content = "" - pre_index = 0 - - for pre_part in pre_parts: - start = pre_part.find("" - content = content + pre_part[0:start] + name - pre_index += 1 - content = content + last_pre - - content = re.sub(r'
\s*
', "\n\n", content) - allblocks = ('(?:table|thead|tfoot|caption|col|colgroup|tbody|tr|' - 'td|th|div|dl|dd|dt|ul|ol|li|pre|select|option|form|' - 'map|area|blockquote|address|math|style|p|h[1-6]|hr|' - 'fieldset|noscript|samp|legend|section|article|aside|' - 'hgroup|header|footer|nav|figure|figcaption|details|' - 'menu|summary)') - content = re.sub(r'(<' + allblocks + r'[^>]*>)', "\n\\1", content) - content = re.sub(r'()', "\\1\n\n", content) - # content = content.replace("\r\n", "\n") - if " inside object/embed - content = re.sub(r'\s*]*)>\s*', "", content) - content = re.sub(r'\s*\s*', '', content) - # content = re.sub(r'/\n\n+/', '\n\n', content) - pgraphs = filter(lambda s: s != "", re.split(r'\n\s*\n', content)) - content = "" - for p in pgraphs: - content = content + "

" + p.strip() + "

\n" - # under certain strange conditions it could create - # a P of entirely whitespace - content = re.sub(r'

\s*

', '', content) - content = re.sub( - r'

([^<]+)', - "

\\1

", - content) - # don't wrap tags - content = re.sub( - r'

\s*(]*>)\s*

', - "\\1", - content) - # problem with nested lists - content = re.sub(r'

(', "\\1", content) - content = re.sub(r'

]*)>', "

", content) - content = content.replace('

', '

') - content = re.sub(r'

\s*(]*>)', "\\1", content) - content = re.sub(r'(]*>)\s*

', "\\1", content) - if br: - def _preserve_newline(match): - return match.group(0).replace("\n", "") - content = re.sub( - r'/<(script|style).*?<\/\\1>/s', - _preserve_newline, - content) - # optionally make line breaks - content = re.sub(r'(?)\s*\n', "
\n", content) - content = content.replace("", "\n") - content = re.sub( - r'(]*>)\s*
', "\\1", - content) - content = re.sub( - r'
(\s*]*>)', - '\\1', - content) - content = re.sub(r'\n

', "

", content) - - if pre_tags: - def _multi_replace(dic, string): - pattern = r'|'.join(map(re.escape, dic.keys())) - return re.sub(pattern, lambda m: dic[m.group()], string) - content = _multi_replace(pre_tags, content) - - return content - - -def xml_to_soup(xml): - """Opens an xml file""" - try: - from bs4 import BeautifulSoup - except ImportError: - error = ('Missing dependency "BeautifulSoup4" and "lxml" required to ' - 'import XML files.') - sys.exit(error) - with open(xml, encoding='utf-8') as infile: - xmlfile = infile.read() - soup = BeautifulSoup(xmlfile, "xml") - return soup - - -def get_filename(post_name, post_id): - if post_name is None or post_name.isspace(): - return post_id - else: - return post_name - - -def wp2fields(xml, wp_custpost=False): - """Opens a wordpress XML file, and yield Pelican fields""" - - soup = xml_to_soup(xml) - items = soup.rss.channel.findAll('item') - for item in items: - - if item.find('status').string in ["publish", "draft"]: - - try: - # Use HTMLParser due to issues with BeautifulSoup 3 - title = unescape(item.title.contents[0]) - except IndexError: - title = 'No title [%s]' % item.find('post_name').string - logger.warning('Post "%s" is lacking a proper title', title) - - post_name = item.find('post_name').string - post_id = item.find('post_id').string - filename = get_filename(post_name, post_id) - - content = item.find('encoded').string - raw_date = item.find('post_date').string - if raw_date == '0000-00-00 00:00:00': - date = None - else: - date_object = SafeDatetime.strptime( - raw_date, '%Y-%m-%d %H:%M:%S') - date = date_object.strftime('%Y-%m-%d %H:%M') - author = item.find('creator').string - - categories = [cat.string for cat - in item.findAll('category', {'domain': 'category'})] - - tags = [tag.string for tag - in item.findAll('category', {'domain': 'post_tag'})] - # To publish a post the status should be 'published' - status = 'published' if item.find('status').string == "publish" \ - else item.find('status').string - - kind = 'article' - post_type = item.find('post_type').string - if post_type == 'page': - kind = 'page' - elif wp_custpost: - if post_type == 'post': - pass - # Old behaviour was to name everything not a page as an - # article.Theoretically all attachments have status == inherit - # so no attachments should be here. But this statement is to - # maintain existing behaviour in case that doesn't hold true. - elif post_type == 'attachment': - pass - else: - kind = post_type - yield (title, content, filename, date, author, categories, - tags, status, kind, 'wp-html') - - -def blogger2fields(xml): - """Opens a blogger XML file, and yield Pelican fields""" - - soup = xml_to_soup(xml) - entries = soup.feed.findAll('entry') - for entry in entries: - raw_kind = entry.find( - 'category', {'scheme': 'http://schemas.google.com/g/2005#kind'} - ).get('term') - if raw_kind == 'http://schemas.google.com/blogger/2008/kind#post': - kind = 'article' - elif raw_kind == 'http://schemas.google.com/blogger/2008/kind#comment': - kind = 'comment' - elif raw_kind == 'http://schemas.google.com/blogger/2008/kind#page': - kind = 'page' - else: - continue - - try: - assert kind != 'comment' - filename = entry.find('link', {'rel': 'alternate'})['href'] - filename = os.path.splitext(os.path.basename(filename))[0] - except (AssertionError, TypeError, KeyError): - filename = entry.find('id').string.split('.')[-1] - - title = entry.find('title').string or '' - - content = entry.find('content').string - raw_date = entry.find('published').string - if hasattr(SafeDatetime, 'fromisoformat'): - date_object = SafeDatetime.fromisoformat(raw_date) - else: - date_object = SafeDatetime.strptime( - raw_date[:23], '%Y-%m-%dT%H:%M:%S.%f') - date = date_object.strftime('%Y-%m-%d %H:%M') - author = entry.find('author').find('name').string - - # blogger posts only have tags, no category - tags = [tag.get('term') for tag in entry.findAll( - 'category', {'scheme': 'http://www.blogger.com/atom/ns#'})] - - # Drafts have yes - status = 'published' - try: - if entry.find('control').find('draft').string == 'yes': - status = 'draft' - except AttributeError: - pass - - yield (title, content, filename, date, author, None, tags, status, - kind, 'html') - - -def dc2fields(file): - """Opens a Dotclear export file, and yield pelican fields""" - try: - from bs4 import BeautifulSoup - except ImportError: - error = ('Missing dependency ' - '"BeautifulSoup4" and "lxml" required ' - 'to import Dotclear files.') - sys.exit(error) - - in_cat = False - in_post = False - category_list = {} - posts = [] - - with open(file, encoding='utf-8') as f: - - for line in f: - # remove final \n - line = line[:-1] - - if line.startswith('[category'): - in_cat = True - elif line.startswith('[post'): - in_post = True - elif in_cat: - fields = line.split('","') - if not line: - in_cat = False - else: - # remove 1st and last "" - fields[0] = fields[0][1:] - # fields[-1] = fields[-1][:-1] - category_list[fields[0]] = fields[2] - elif in_post: - if not line: - in_post = False - break - else: - posts.append(line) - - print("%i posts read." % len(posts)) - - settings = read_settings() - subs = settings['SLUG_REGEX_SUBSTITUTIONS'] - for post in posts: - fields = post.split('","') - - # post_id = fields[0][1:] - # blog_id = fields[1] - # user_id = fields[2] - cat_id = fields[3] - # post_dt = fields[4] - # post_tz = fields[5] - post_creadt = fields[6] - # post_upddt = fields[7] - # post_password = fields[8] - # post_type = fields[9] - post_format = fields[10] - # post_url = fields[11] - # post_lang = fields[12] - post_title = fields[13] - post_excerpt = fields[14] - post_excerpt_xhtml = fields[15] - post_content = fields[16] - post_content_xhtml = fields[17] - # post_notes = fields[18] - # post_words = fields[19] - # post_status = fields[20] - # post_selected = fields[21] - # post_position = fields[22] - # post_open_comment = fields[23] - # post_open_tb = fields[24] - # nb_comment = fields[25] - # nb_trackback = fields[26] - post_meta = fields[27] - # redirect_url = fields[28][:-1] - - # remove seconds - post_creadt = ':'.join(post_creadt.split(':')[0:2]) - - author = '' - categories = [] - tags = [] - - if cat_id: - categories = [category_list[id].strip() for id - in cat_id.split(',')] - - # Get tags related to a post - tag = (post_meta.replace('{', '') - .replace('}', '') - .replace('a:1:s:3:\\"tag\\";a:', '') - .replace('a:0:', '')) - if len(tag) > 1: - if int(len(tag[:1])) == 1: - newtag = tag.split('"')[1] - tags.append( - BeautifulSoup( - newtag, - 'xml' - ) - # bs4 always outputs UTF-8 - .decode('utf-8') - ) - else: - i = 1 - j = 1 - while(i <= int(tag[:1])): - newtag = tag.split('"')[j].replace('\\', '') - tags.append( - BeautifulSoup( - newtag, - 'xml' - ) - # bs4 always outputs UTF-8 - .decode('utf-8') - ) - i = i + 1 - if j < int(tag[:1]) * 2: - j = j + 2 - - """ - dotclear2 does not use markdown by default unless - you use the markdown plugin - Ref: http://plugins.dotaddict.org/dc2/details/formatting-markdown - """ - if post_format == "markdown": - content = post_excerpt + post_content - else: - content = post_excerpt_xhtml + post_content_xhtml - content = content.replace('\\n', '') - post_format = "html" - - kind = 'article' # TODO: Recognise pages - status = 'published' # TODO: Find a way for draft posts - - yield (post_title, content, slugify(post_title, regex_subs=subs), - post_creadt, author, categories, tags, status, kind, - post_format) - - -def posterous2fields(api_token, email, password): - """Imports posterous posts""" - import base64 - from datetime import timedelta - import json - import urllib.request as urllib_request - - def get_posterous_posts(api_token, email, password, page=1): - base64string = base64.encodestring( - ("{}:{}".format(email, password)).encode('utf-8')).replace('\n', '') - url = ("http://posterous.com/api/v2/users/me/sites/primary/" - "posts?api_token=%s&page=%d") % (api_token, page) - request = urllib_request.Request(url) - request.add_header('Authorization', 'Basic %s' % base64string.decode()) - handle = urllib_request.urlopen(request) - posts = json.loads(handle.read().decode('utf-8')) - return posts - - page = 1 - posts = get_posterous_posts(api_token, email, password, page) - settings = read_settings() - subs = settings['SLUG_REGEX_SUBSTITUTIONS'] - while len(posts) > 0: - posts = get_posterous_posts(api_token, email, password, page) - page += 1 - - for post in posts: - slug = post.get('slug') - if not slug: - slug = slugify(post.get('title'), regex_subs=subs) - tags = [tag.get('name') for tag in post.get('tags')] - raw_date = post.get('display_date') - date_object = SafeDatetime.strptime( - raw_date[:-6], '%Y/%m/%d %H:%M:%S') - offset = int(raw_date[-5:]) - delta = timedelta(hours=(offset / 100)) - date_object -= delta - date = date_object.strftime('%Y-%m-%d %H:%M') - kind = 'article' # TODO: Recognise pages - status = 'published' # TODO: Find a way for draft posts - - yield (post.get('title'), post.get('body_cleaned'), - slug, date, post.get('user').get('display_name'), - [], tags, status, kind, 'html') - - -def tumblr2fields(api_key, blogname): - """ Imports Tumblr posts (API v2)""" - import json - import urllib.request as urllib_request - - def get_tumblr_posts(api_key, blogname, offset=0): - url = ("https://api.tumblr.com/v2/blog/%s.tumblr.com/" - "posts?api_key=%s&offset=%d&filter=raw") % ( - blogname, api_key, offset) - request = urllib_request.Request(url) - handle = urllib_request.urlopen(request) - posts = json.loads(handle.read().decode('utf-8')) - return posts.get('response').get('posts') - - offset = 0 - posts = get_tumblr_posts(api_key, blogname, offset) - settings = read_settings() - subs = settings['SLUG_REGEX_SUBSTITUTIONS'] - while len(posts) > 0: - for post in posts: - title = \ - post.get('title') or \ - post.get('source_title') or \ - post.get('type').capitalize() - slug = post.get('slug') or slugify(title, regex_subs=subs) - tags = post.get('tags') - timestamp = post.get('timestamp') - date = SafeDatetime.fromtimestamp(int(timestamp)).strftime( - "%Y-%m-%d %H:%M:%S") - slug = SafeDatetime.fromtimestamp(int(timestamp)).strftime( - "%Y-%m-%d-") + slug - format = post.get('format') - content = post.get('body') - type = post.get('type') - if type == 'photo': - if format == 'markdown': - fmtstr = '![%s](%s)' - else: - fmtstr = '%s' - content = '' - for photo in post.get('photos'): - content += '\n'.join( - fmtstr % (photo.get('caption'), - photo.get('original_size').get('url'))) - content += '\n\n' + post.get('caption') - elif type == 'quote': - if format == 'markdown': - fmtstr = '\n\n— %s' - else: - fmtstr = '

— %s

' - content = post.get('text') + fmtstr % post.get('source') - elif type == 'link': - if format == 'markdown': - fmtstr = '[via](%s)\n\n' - else: - fmtstr = '

via

\n' - content = fmtstr % post.get('url') + post.get('description') - elif type == 'audio': - if format == 'markdown': - fmtstr = '[via](%s)\n\n' - else: - fmtstr = '

via

\n' - content = fmtstr % post.get('source_url') + \ - post.get('caption') + \ - post.get('player') - elif type == 'video': - if format == 'markdown': - fmtstr = '[via](%s)\n\n' - else: - fmtstr = '

via

\n' - source = fmtstr % post.get('source_url') - caption = post.get('caption') - players = '\n'.join(player.get('embed_code') - for player in post.get('player')) - content = source + caption + players - elif type == 'answer': - title = post.get('question') - content = ('

' - '%s' - ': %s' - '

\n' - ' %s' % (post.get('asking_name'), - post.get('asking_url'), - post.get('question'), - post.get('answer'))) - - content = content.rstrip() + '\n' - kind = 'article' - status = 'published' # TODO: Find a way for draft posts - - yield (title, content, slug, date, post.get('blog_name'), [type], - tags, status, kind, format) - - offset += len(posts) - posts = get_tumblr_posts(api_key, blogname, offset) - - -def feed2fields(file): - """Read a feed and yield pelican fields""" - import feedparser - d = feedparser.parse(file) - settings = read_settings() - subs = settings['SLUG_REGEX_SUBSTITUTIONS'] - for entry in d.entries: - date = (time.strftime('%Y-%m-%d %H:%M', entry.updated_parsed) - if hasattr(entry, 'updated_parsed') else None) - author = entry.author if hasattr(entry, 'author') else None - tags = ([e['term'] for e in entry.tags] - if hasattr(entry, 'tags') else None) - - slug = slugify(entry.title, regex_subs=subs) - kind = 'article' - yield (entry.title, entry.description, slug, date, - author, [], tags, None, kind, 'html') - - -def build_header(title, date, author, categories, tags, slug, - status=None, attachments=None): - """Build a header from a list of fields""" - - from docutils.utils import column_width - - header = '{}\n{}\n'.format(title, '#' * column_width(title)) - if date: - header += ':date: %s\n' % date - if author: - header += ':author: %s\n' % author - if categories: - header += ':category: %s\n' % ', '.join(categories) - if tags: - header += ':tags: %s\n' % ', '.join(tags) - if slug: - header += ':slug: %s\n' % slug - if status: - header += ':status: %s\n' % status - if attachments: - header += ':attachments: %s\n' % ', '.join(attachments) - header += '\n' - return header - - -def build_asciidoc_header(title, date, author, categories, tags, slug, - status=None, attachments=None): - """Build a header from a list of fields""" - - header = '= %s\n' % title - if author: - header += '%s\n' % author - if date: - header += '%s\n' % date - if categories: - header += ':category: %s\n' % ', '.join(categories) - if tags: - header += ':tags: %s\n' % ', '.join(tags) - if slug: - header += ':slug: %s\n' % slug - if status: - header += ':status: %s\n' % status - if attachments: - header += ':attachments: %s\n' % ', '.join(attachments) - header += '\n' - return header - - -def build_markdown_header(title, date, author, categories, tags, - slug, status=None, attachments=None): - """Build a header from a list of fields""" - header = 'Title: %s\n' % title - if date: - header += 'Date: %s\n' % date - if author: - header += 'Author: %s\n' % author - if categories: - header += 'Category: %s\n' % ', '.join(categories) - if tags: - header += 'Tags: %s\n' % ', '.join(tags) - if slug: - header += 'Slug: %s\n' % slug - if status: - header += 'Status: %s\n' % status - if attachments: - header += 'Attachments: %s\n' % ', '.join(attachments) - header += '\n' - return header - - -def get_ext(out_markup, in_markup='html'): - if out_markup == 'asciidoc': - ext = '.adoc' - elif in_markup == 'markdown' or out_markup == 'markdown': - ext = '.md' - else: - ext = '.rst' - return ext - - -def get_out_filename(output_path, filename, ext, kind, - dirpage, dircat, categories, wp_custpost, slug_subs): - filename = os.path.basename(filename) - - # Enforce filename restrictions for various filesystems at once; see - # https://en.wikipedia.org/wiki/Filename#Reserved_characters_and_words - # we do not need to filter words because an extension will be appended - filename = re.sub(r'[<>:"/\\|?*^% ]', '-', filename) # invalid chars - filename = filename.lstrip('.') # should not start with a dot - if not filename: - filename = '_' - filename = filename[:249] # allow for 5 extra characters - - out_filename = os.path.join(output_path, filename + ext) - # option to put page posts in pages/ subdirectory - if dirpage and kind == 'page': - pages_dir = os.path.join(output_path, 'pages') - if not os.path.isdir(pages_dir): - os.mkdir(pages_dir) - out_filename = os.path.join(pages_dir, filename + ext) - elif not dirpage and kind == 'page': - pass - # option to put wp custom post types in directories with post type - # names. Custom post types can also have categories so option to - # create subdirectories with category names - elif kind != 'article': - if wp_custpost: - typename = slugify(kind, regex_subs=slug_subs) - else: - typename = '' - kind = 'article' - if dircat and (len(categories) > 0): - catname = slugify( - categories[0], regex_subs=slug_subs, preserve_case=True) - else: - catname = '' - out_filename = os.path.join(output_path, typename, - catname, filename + ext) - if not os.path.isdir(os.path.join(output_path, typename, catname)): - os.makedirs(os.path.join(output_path, typename, catname)) - # option to put files in directories with categories names - elif dircat and (len(categories) > 0): - catname = slugify( - categories[0], regex_subs=slug_subs, preserve_case=True) - out_filename = os.path.join(output_path, catname, filename + ext) - if not os.path.isdir(os.path.join(output_path, catname)): - os.mkdir(os.path.join(output_path, catname)) - - return out_filename - - -def get_attachments(xml): - """returns a dictionary of posts that have attachments with a list - of the attachment_urls - """ - soup = xml_to_soup(xml) - items = soup.rss.channel.findAll('item') - names = {} - attachments = [] - - for item in items: - kind = item.find('post_type').string - post_name = item.find('post_name').string - post_id = item.find('post_id').string - - if kind == 'attachment': - attachments.append((item.find('post_parent').string, - item.find('attachment_url').string)) - else: - filename = get_filename(post_name, post_id) - names[post_id] = filename - attachedposts = defaultdict(set) - for parent, url in attachments: - try: - parent_name = names[parent] - except KeyError: - # attachment's parent is not a valid post - parent_name = None - - attachedposts[parent_name].add(url) - return attachedposts - - -def download_attachments(output_path, urls): - """Downloads WordPress attachments and returns a list of paths to - attachments that can be associated with a post (relative path to output - directory). Files that fail to download, will not be added to posts""" - locations = {} - for url in urls: - path = urlparse(url).path - # teardown path and rebuild to negate any errors with - # os.path.join and leading /'s - path = path.split('/') - filename = path.pop(-1) - localpath = '' - for item in path: - if sys.platform != 'win32' or ':' not in item: - localpath = os.path.join(localpath, item) - full_path = os.path.join(output_path, localpath) - - # Generate percent-encoded URL - scheme, netloc, path, query, fragment = urlsplit(url) - if scheme != 'file': - path = quote(path) - url = urlunsplit((scheme, netloc, path, query, fragment)) - - if not os.path.exists(full_path): - os.makedirs(full_path) - print('downloading {}'.format(filename)) - try: - urlretrieve(url, os.path.join(full_path, filename)) - locations[url] = os.path.join(localpath, filename) - except (URLError, OSError) as e: - # Python 2.7 throws an IOError rather Than URLError - logger.warning("No file could be downloaded from %s\n%s", url, e) - return locations - - -def is_pandoc_needed(in_markup): - return in_markup in ('html', 'wp-html') - - -def get_pandoc_version(): - cmd = ['pandoc', '--version'] - try: - output = subprocess.check_output(cmd, universal_newlines=True) - except (subprocess.CalledProcessError, OSError) as e: - logger.warning("Pandoc version unknown: %s", e) - return () - - return tuple(int(i) for i in output.split()[1].split('.')) - - -def update_links_to_attached_files(content, attachments): - for old_url, new_path in attachments.items(): - # url may occur both with http:// and https:// - http_url = old_url.replace('https://', 'http://') - https_url = old_url.replace('http://', 'https://') - for url in [http_url, https_url]: - content = content.replace(url, '{static}' + new_path) - return content - - -def fields2pelican( - fields, out_markup, output_path, - dircat=False, strip_raw=False, disable_slugs=False, - dirpage=False, filename_template=None, filter_author=None, - wp_custpost=False, wp_attach=False, attachments=None): - - pandoc_version = get_pandoc_version() - posts_require_pandoc = [] - - settings = read_settings() - slug_subs = settings['SLUG_REGEX_SUBSTITUTIONS'] - - for (title, content, filename, date, author, categories, tags, status, - kind, in_markup) in fields: - if filter_author and filter_author != author: - continue - if is_pandoc_needed(in_markup) and not pandoc_version: - posts_require_pandoc.append(filename) - - slug = not disable_slugs and filename or None - - if wp_attach and attachments: - try: - urls = attachments[filename] - links = download_attachments(output_path, urls) - except KeyError: - links = None - else: - links = None - - ext = get_ext(out_markup, in_markup) - if ext == '.adoc': - header = build_asciidoc_header(title, date, author, categories, - tags, slug, status, attachments) - elif ext == '.md': - header = build_markdown_header( - title, date, author, categories, tags, slug, - status, links.values() if links else None) - else: - out_markup = 'rst' - header = build_header(title, date, author, categories, - tags, slug, status, links.values() - if links else None) - - out_filename = get_out_filename( - output_path, filename, ext, kind, dirpage, dircat, - categories, wp_custpost, slug_subs) - print(out_filename) - - if in_markup in ('html', 'wp-html'): - html_filename = os.path.join(output_path, filename + '.html') - - with open(html_filename, 'w', encoding='utf-8') as fp: - # Replace newlines with paragraphs wrapped with

so - # HTML is valid before conversion - if in_markup == 'wp-html': - new_content = decode_wp_content(content) - else: - paragraphs = content.splitlines() - paragraphs = ['

{}

'.format(p) for p in paragraphs] - new_content = ''.join(paragraphs) - - fp.write(new_content) - - if pandoc_version < (2,): - parse_raw = '--parse-raw' if not strip_raw else '' - wrap_none = '--wrap=none' \ - if pandoc_version >= (1, 16) else '--no-wrap' - cmd = ('pandoc --normalize {0} --from=html' - ' --to={1} {2} -o "{3}" "{4}"') - cmd = cmd.format(parse_raw, out_markup, wrap_none, - out_filename, html_filename) - else: - from_arg = '-f html+raw_html' if not strip_raw else '-f html' - cmd = ('pandoc {0} --to={1}-smart --wrap=none -o "{2}" "{3}"') - cmd = cmd.format(from_arg, out_markup, - out_filename, html_filename) - - try: - rc = subprocess.call(cmd, shell=True) - if rc < 0: - error = 'Child was terminated by signal %d' % -rc - exit(error) - - elif rc > 0: - error = 'Please, check your Pandoc installation.' - exit(error) - except OSError as e: - error = 'Pandoc execution failed: %s' % e - exit(error) - - os.remove(html_filename) - - with open(out_filename, encoding='utf-8') as fs: - content = fs.read() - if out_markup == 'markdown': - # In markdown, to insert a
, end a line with two - # or more spaces & then a end-of-line - content = content.replace('\\\n ', ' \n') - content = content.replace('\\\n', ' \n') - - if wp_attach and links: - content = update_links_to_attached_files(content, links) - - with open(out_filename, 'w', encoding='utf-8') as fs: - fs.write(header + content) - - if posts_require_pandoc: - logger.error("Pandoc must be installed to import the following posts:" - "\n {}".format("\n ".join(posts_require_pandoc))) - - if wp_attach and attachments and None in attachments: - print("downloading attachments that don't have a parent post") - urls = attachments[None] - download_attachments(output_path, urls) - - -def main(): - parser = argparse.ArgumentParser( - description="Transform feed, Blogger, Dotclear, Posterous, Tumblr, or " - "WordPress files into reST (rst) or Markdown (md) files. " - "Be sure to have pandoc installed.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - parser.add_argument( - dest='input', help='The input file to read') - parser.add_argument( - '--blogger', action='store_true', dest='blogger', - help='Blogger XML export') - parser.add_argument( - '--dotclear', action='store_true', dest='dotclear', - help='Dotclear export') - parser.add_argument( - '--posterous', action='store_true', dest='posterous', - help='Posterous export') - parser.add_argument( - '--tumblr', action='store_true', dest='tumblr', - help='Tumblr export') - parser.add_argument( - '--wpfile', action='store_true', dest='wpfile', - help='Wordpress XML export') - parser.add_argument( - '--feed', action='store_true', dest='feed', - help='Feed to parse') - parser.add_argument( - '-o', '--output', dest='output', default='content', - help='Output path') - parser.add_argument( - '-m', '--markup', dest='markup', default='rst', - help='Output markup format (supports rst & markdown)') - parser.add_argument( - '--dir-cat', action='store_true', dest='dircat', - help='Put files in directories with categories name') - parser.add_argument( - '--dir-page', action='store_true', dest='dirpage', - help=('Put files recognised as pages in "pages/" sub-directory' - ' (blogger and wordpress import only)')) - parser.add_argument( - '--filter-author', dest='author', - help='Import only post from the specified author') - parser.add_argument( - '--strip-raw', action='store_true', dest='strip_raw', - help="Strip raw HTML code that can't be converted to " - "markup such as flash embeds or iframes (wordpress import only)") - parser.add_argument( - '--wp-custpost', action='store_true', - dest='wp_custpost', - help='Put wordpress custom post types in directories. If used with ' - '--dir-cat option directories will be created as ' - '/post_type/category/ (wordpress import only)') - parser.add_argument( - '--wp-attach', action='store_true', dest='wp_attach', - help='(wordpress import only) Download files uploaded to wordpress as ' - 'attachments. Files will be added to posts as a list in the post ' - 'header. All files will be downloaded, even if ' - "they aren't associated with a post. Files will be downloaded " - 'with their original path inside the output directory. ' - 'e.g. output/wp-uploads/date/postname/file.jpg ' - '-- Requires an internet connection --') - parser.add_argument( - '--disable-slugs', action='store_true', - dest='disable_slugs', - help='Disable storing slugs from imported posts within output. ' - 'With this disabled, your Pelican URLs may not be consistent ' - 'with your original posts.') - parser.add_argument( - '-e', '--email', dest='email', - help="Email address (posterous import only)") - parser.add_argument( - '-p', '--password', dest='password', - help="Password (posterous import only)") - parser.add_argument( - '-b', '--blogname', dest='blogname', - help="Blog name (Tumblr import only)") - - args = parser.parse_args() - - input_type = None - if args.blogger: - input_type = 'blogger' - elif args.dotclear: - input_type = 'dotclear' - elif args.posterous: - input_type = 'posterous' - elif args.tumblr: - input_type = 'tumblr' - elif args.wpfile: - input_type = 'wordpress' - elif args.feed: - input_type = 'feed' - else: - error = ('You must provide either --blogger, --dotclear, ' - '--posterous, --tumblr, --wpfile or --feed options') - exit(error) - - if not os.path.exists(args.output): - try: - os.mkdir(args.output) - except OSError: - error = 'Unable to create the output folder: ' + args.output - exit(error) - - if args.wp_attach and input_type != 'wordpress': - error = ('You must be importing a wordpress xml ' - 'to use the --wp-attach option') - exit(error) - - if input_type == 'blogger': - fields = blogger2fields(args.input) - elif input_type == 'dotclear': - fields = dc2fields(args.input) - elif input_type == 'posterous': - fields = posterous2fields(args.input, args.email, args.password) - elif input_type == 'tumblr': - fields = tumblr2fields(args.input, args.blogname) - elif input_type == 'wordpress': - fields = wp2fields(args.input, args.wp_custpost or False) - elif input_type == 'feed': - fields = feed2fields(args.input) - - if args.wp_attach: - attachments = get_attachments(args.input) - else: - attachments = None - - # init logging - init() - fields2pelican(fields, args.markup, args.output, - dircat=args.dircat or False, - dirpage=args.dirpage or False, - strip_raw=args.strip_raw or False, - disable_slugs=args.disable_slugs or False, - filter_author=args.author, - wp_custpost=args.wp_custpost or False, - wp_attach=args.wp_attach or False, - attachments=attachments or None) diff --git a/pelican/tools/pelican_quickstart.py b/pelican/tools/pelican_quickstart.py deleted file mode 100755 index 41ddea3c..00000000 --- a/pelican/tools/pelican_quickstart.py +++ /dev/null @@ -1,354 +0,0 @@ -#!/usr/bin/env python - -import argparse -import locale -import os - -from jinja2 import Environment, FileSystemLoader - -import pytz - -try: - import readline # NOQA -except ImportError: - pass - -try: - import tzlocal - _DEFAULT_TIMEZONE = tzlocal.get_localzone().zone -except ImportError: - _DEFAULT_TIMEZONE = 'Europe/Paris' - -from pelican import __version__ - -locale.setlocale(locale.LC_ALL, '') -try: - _DEFAULT_LANGUAGE = locale.getlocale()[0] -except ValueError: - # Don't fail on macosx: "unknown locale: UTF-8" - _DEFAULT_LANGUAGE = None -if _DEFAULT_LANGUAGE is None: - _DEFAULT_LANGUAGE = 'en' -else: - _DEFAULT_LANGUAGE = _DEFAULT_LANGUAGE.split('_')[0] - -_TEMPLATES_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), - "templates") -_jinja_env = Environment( - loader=FileSystemLoader(_TEMPLATES_DIR), - trim_blocks=True, -) - - -_GITHUB_PAGES_BRANCHES = { - 'personal': 'main', - 'project': 'gh-pages' -} - -CONF = { - 'pelican': 'pelican', - 'pelicanopts': '', - 'basedir': os.curdir, - 'ftp_host': 'localhost', - 'ftp_user': 'anonymous', - 'ftp_target_dir': '/', - 'ssh_host': 'localhost', - 'ssh_port': 22, - 'ssh_user': 'root', - 'ssh_target_dir': '/var/www', - 's3_bucket': 'my_s3_bucket', - 'cloudfiles_username': 'my_rackspace_username', - 'cloudfiles_api_key': 'my_rackspace_api_key', - 'cloudfiles_container': 'my_cloudfiles_container', - 'dropbox_dir': '~/Dropbox/Public/', - 'github_pages_branch': _GITHUB_PAGES_BRANCHES['project'], - 'default_pagination': 10, - 'siteurl': '', - 'lang': _DEFAULT_LANGUAGE, - 'timezone': _DEFAULT_TIMEZONE -} - -# url for list of valid timezones -_TZ_URL = 'https://en.wikipedia.org/wiki/List_of_tz_database_time_zones' - - -# Create a 'marked' default path, to determine if someone has supplied -# a path on the command-line. -class _DEFAULT_PATH_TYPE(str): - is_default_path = True - - -_DEFAULT_PATH = _DEFAULT_PATH_TYPE(os.curdir) - - -def ask(question, answer=str, default=None, length=None): - if answer == str: - r = '' - while True: - if default: - r = input('> {} [{}] '.format(question, default)) - else: - r = input('> {} '.format(question)) - - r = r.strip() - - if len(r) <= 0: - if default: - r = default - break - else: - print('You must enter something') - else: - if length and len(r) != length: - print('Entry must be {} characters long'.format(length)) - else: - break - - return r - - elif answer == bool: - r = None - while True: - if default is True: - r = input('> {} (Y/n) '.format(question)) - elif default is False: - r = input('> {} (y/N) '.format(question)) - else: - r = input('> {} (y/n) '.format(question)) - - r = r.strip().lower() - - if r in ('y', 'yes'): - r = True - break - elif r in ('n', 'no'): - r = False - break - elif not r: - r = default - break - else: - print("You must answer 'yes' or 'no'") - return r - elif answer == int: - r = None - while True: - if default: - r = input('> {} [{}] '.format(question, default)) - else: - r = input('> {} '.format(question)) - - r = r.strip() - - if not r: - r = default - break - - try: - r = int(r) - break - except ValueError: - print('You must enter an integer') - return r - else: - raise NotImplementedError( - 'Argument `answer` must be str, bool, or integer') - - -def ask_timezone(question, default, tzurl): - """Prompt for time zone and validate input""" - lower_tz = [tz.lower() for tz in pytz.all_timezones] - while True: - r = ask(question, str, default) - r = r.strip().replace(' ', '_').lower() - if r in lower_tz: - r = pytz.all_timezones[lower_tz.index(r)] - break - else: - print('Please enter a valid time zone:\n' - ' (check [{}])'.format(tzurl)) - return r - - -def main(): - parser = argparse.ArgumentParser( - description="A kickstarter for Pelican", - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('-p', '--path', default=_DEFAULT_PATH, - help="The path to generate the blog into") - parser.add_argument('-t', '--title', metavar="title", - help='Set the title of the website') - parser.add_argument('-a', '--author', metavar="author", - help='Set the author name of the website') - parser.add_argument('-l', '--lang', metavar="lang", - help='Set the default web site language') - - args = parser.parse_args() - - print('''Welcome to pelican-quickstart v{v}. - -This script will help you create a new Pelican-based website. - -Please answer the following questions so this script can generate the files -needed by Pelican. - - '''.format(v=__version__)) - - project = os.path.join( - os.environ.get('VIRTUAL_ENV', os.curdir), '.project') - no_path_was_specified = hasattr(args.path, 'is_default_path') - if os.path.isfile(project) and no_path_was_specified: - CONF['basedir'] = open(project).read().rstrip("\n") - print('Using project associated with current virtual environment. ' - 'Will save to:\n%s\n' % CONF['basedir']) - else: - CONF['basedir'] = os.path.abspath(os.path.expanduser( - ask('Where do you want to create your new web site?', - answer=str, default=args.path))) - - CONF['sitename'] = ask('What will be the title of this web site?', - answer=str, default=args.title) - CONF['author'] = ask('Who will be the author of this web site?', - answer=str, default=args.author) - CONF['lang'] = ask('What will be the default language of this web site?', - str, args.lang or CONF['lang'], 2) - - if ask('Do you want to specify a URL prefix? e.g., https://example.com ', - answer=bool, default=True): - CONF['siteurl'] = ask('What is your URL prefix? (see ' - 'above example; no trailing slash)', - str, CONF['siteurl']) - - CONF['with_pagination'] = ask('Do you want to enable article pagination?', - bool, bool(CONF['default_pagination'])) - - if CONF['with_pagination']: - CONF['default_pagination'] = ask('How many articles per page ' - 'do you want?', - int, CONF['default_pagination']) - else: - CONF['default_pagination'] = False - - CONF['timezone'] = ask_timezone('What is your time zone?', - CONF['timezone'], _TZ_URL) - - automation = ask('Do you want to generate a tasks.py/Makefile ' - 'to automate generation and publishing?', bool, True) - - if automation: - if ask('Do you want to upload your website using FTP?', - answer=bool, default=False): - CONF['ftp'] = True, - CONF['ftp_host'] = ask('What is the hostname of your FTP server?', - str, CONF['ftp_host']) - CONF['ftp_user'] = ask('What is your username on that server?', - str, CONF['ftp_user']) - CONF['ftp_target_dir'] = ask('Where do you want to put your ' - 'web site on that server?', - str, CONF['ftp_target_dir']) - if ask('Do you want to upload your website using SSH?', - answer=bool, default=False): - CONF['ssh'] = True, - CONF['ssh_host'] = ask('What is the hostname of your SSH server?', - str, CONF['ssh_host']) - CONF['ssh_port'] = ask('What is the port of your SSH server?', - int, CONF['ssh_port']) - CONF['ssh_user'] = ask('What is your username on that server?', - str, CONF['ssh_user']) - CONF['ssh_target_dir'] = ask('Where do you want to put your ' - 'web site on that server?', - str, CONF['ssh_target_dir']) - - if ask('Do you want to upload your website using Dropbox?', - answer=bool, default=False): - CONF['dropbox'] = True, - CONF['dropbox_dir'] = ask('Where is your Dropbox directory?', - str, CONF['dropbox_dir']) - - if ask('Do you want to upload your website using S3?', - answer=bool, default=False): - CONF['s3'] = True, - CONF['s3_bucket'] = ask('What is the name of your S3 bucket?', - str, CONF['s3_bucket']) - - if ask('Do you want to upload your website using ' - 'Rackspace Cloud Files?', answer=bool, default=False): - CONF['cloudfiles'] = True, - CONF['cloudfiles_username'] = ask('What is your Rackspace ' - 'Cloud username?', str, - CONF['cloudfiles_username']) - CONF['cloudfiles_api_key'] = ask('What is your Rackspace ' - 'Cloud API key?', str, - CONF['cloudfiles_api_key']) - CONF['cloudfiles_container'] = ask('What is the name of your ' - 'Cloud Files container?', - str, - CONF['cloudfiles_container']) - - if ask('Do you want to upload your website using GitHub Pages?', - answer=bool, default=False): - CONF['github'] = True, - if ask('Is this your personal page (username.github.io)?', - answer=bool, default=False): - CONF['github_pages_branch'] = \ - _GITHUB_PAGES_BRANCHES['personal'] - else: - CONF['github_pages_branch'] = \ - _GITHUB_PAGES_BRANCHES['project'] - - try: - os.makedirs(os.path.join(CONF['basedir'], 'content')) - except OSError as e: - print('Error: {}'.format(e)) - - try: - os.makedirs(os.path.join(CONF['basedir'], 'output')) - except OSError as e: - print('Error: {}'.format(e)) - - try: - with open(os.path.join(CONF['basedir'], 'pelicanconf.py'), - 'w', encoding='utf-8') as fd: - conf_python = dict() - for key, value in CONF.items(): - conf_python[key] = repr(value) - - _template = _jinja_env.get_template('pelicanconf.py.jinja2') - fd.write(_template.render(**conf_python)) - fd.close() - except OSError as e: - print('Error: {}'.format(e)) - - try: - with open(os.path.join(CONF['basedir'], 'publishconf.py'), - 'w', encoding='utf-8') as fd: - _template = _jinja_env.get_template('publishconf.py.jinja2') - fd.write(_template.render(**CONF)) - fd.close() - except OSError as e: - print('Error: {}'.format(e)) - - if automation: - try: - with open(os.path.join(CONF['basedir'], 'tasks.py'), - 'w', encoding='utf-8') as fd: - _template = _jinja_env.get_template('tasks.py.jinja2') - fd.write(_template.render(**CONF)) - fd.close() - except OSError as e: - print('Error: {}'.format(e)) - try: - with open(os.path.join(CONF['basedir'], 'Makefile'), - 'w', encoding='utf-8') as fd: - py_v = 'python3' - _template = _jinja_env.get_template('Makefile.jinja2') - fd.write(_template.render(py_v=py_v, **CONF)) - fd.close() - except OSError as e: - print('Error: {}'.format(e)) - - print('Done. Your new project is available at %s' % CONF['basedir']) - - -if __name__ == "__main__": - main() diff --git a/pelican/tools/pelican_themes.py b/pelican/tools/pelican_themes.py deleted file mode 100755 index 96d07c1f..00000000 --- a/pelican/tools/pelican_themes.py +++ /dev/null @@ -1,259 +0,0 @@ -#!/usr/bin/env python - -import argparse -import os -import shutil -import sys - - -def err(msg, die=None): - """Print an error message and exits if an exit code is given""" - sys.stderr.write(msg + '\n') - if die: - sys.exit(die if type(die) is int else 1) - - -try: - import pelican -except ImportError: - err('Cannot import pelican.\nYou must ' - 'install Pelican in order to run this script.', - -1) - - -global _THEMES_PATH -_THEMES_PATH = os.path.join( - os.path.dirname( - os.path.abspath(pelican.__file__) - ), - 'themes' -) - -__version__ = '0.2' -_BUILTIN_THEMES = ['simple', 'notmyidea'] - - -def main(): - """Main function""" - - parser = argparse.ArgumentParser( - description="""Install themes for Pelican""") - - excl = parser.add_mutually_exclusive_group() - excl.add_argument( - '-l', '--list', dest='action', action="store_const", const='list', - help="Show the themes already installed and exit") - excl.add_argument( - '-p', '--path', dest='action', action="store_const", const='path', - help="Show the themes path and exit") - excl.add_argument( - '-V', '--version', action='version', - version='pelican-themes v{}'.format(__version__), - help='Print the version of this script') - - parser.add_argument( - '-i', '--install', dest='to_install', nargs='+', metavar="theme path", - help='The themes to install') - parser.add_argument( - '-r', '--remove', dest='to_remove', nargs='+', metavar="theme name", - help='The themes to remove') - parser.add_argument( - '-U', '--upgrade', dest='to_upgrade', nargs='+', - metavar="theme path", help='The themes to upgrade') - parser.add_argument( - '-s', '--symlink', dest='to_symlink', nargs='+', metavar="theme path", - help="Same as `--install', but create a symbolic link instead of " - "copying the theme. Useful for theme development") - parser.add_argument( - '-c', '--clean', dest='clean', action="store_true", - help="Remove the broken symbolic links of the theme path") - - parser.add_argument( - '-v', '--verbose', dest='verbose', - action="store_true", - help="Verbose output") - - args = parser.parse_args() - - to_install = args.to_install or args.to_upgrade - to_sym = args.to_symlink or args.clean - - if args.action: - if args.action == 'list': - list_themes(args.verbose) - elif args.action == 'path': - print(_THEMES_PATH) - elif to_install or args.to_remove or to_sym: - if args.to_remove: - if args.verbose: - print('Removing themes...') - - for i in args.to_remove: - remove(i, v=args.verbose) - - if args.to_install: - if args.verbose: - print('Installing themes...') - - for i in args.to_install: - install(i, v=args.verbose) - - if args.to_upgrade: - if args.verbose: - print('Upgrading themes...') - - for i in args.to_upgrade: - install(i, v=args.verbose, u=True) - - if args.to_symlink: - if args.verbose: - print('Linking themes...') - - for i in args.to_symlink: - symlink(i, v=args.verbose) - - if args.clean: - if args.verbose: - print('Cleaning the themes directory...') - - clean(v=args.verbose) - else: - print('No argument given... exiting.') - - -def themes(): - """Returns the list of the themes""" - for i in os.listdir(_THEMES_PATH): - e = os.path.join(_THEMES_PATH, i) - - if os.path.isdir(e): - if os.path.islink(e): - yield (e, os.readlink(e)) - else: - yield (e, None) - - -def list_themes(v=False): - """Display the list of the themes""" - for t, l in themes(): - if not v: - t = os.path.basename(t) - if l: - if v: - print(t + (" (symbolic link to `" + l + "')")) - else: - print(t + '@') - else: - print(t) - - -def remove(theme_name, v=False): - """Removes a theme""" - - theme_name = theme_name.replace('/', '') - target = os.path.join(_THEMES_PATH, theme_name) - - if theme_name in _BUILTIN_THEMES: - err(theme_name + ' is a builtin theme.\n' - 'You cannot remove a builtin theme with this script, ' - 'remove it by hand if you want.') - elif os.path.islink(target): - if v: - print('Removing link `' + target + "'") - os.remove(target) - elif os.path.isdir(target): - if v: - print('Removing directory `' + target + "'") - shutil.rmtree(target) - elif os.path.exists(target): - err(target + ' : not a valid theme') - else: - err(target + ' : no such file or directory') - - -def install(path, v=False, u=False): - """Installs a theme""" - if not os.path.exists(path): - err(path + ' : no such file or directory') - elif not os.path.isdir(path): - err(path + ' : not a directory') - else: - theme_name = os.path.basename(os.path.normpath(path)) - theme_path = os.path.join(_THEMES_PATH, theme_name) - exists = os.path.exists(theme_path) - if exists and not u: - err(path + ' : already exists') - elif exists and u: - remove(theme_name, v) - install(path, v) - else: - if v: - print("Copying '{p}' to '{t}' ...".format(p=path, - t=theme_path)) - try: - shutil.copytree(path, theme_path) - - try: - if os.name == 'posix': - for root, dirs, files in os.walk(theme_path): - for d in dirs: - dname = os.path.join(root, d) - os.chmod(dname, 493) # 0o755 - for f in files: - fname = os.path.join(root, f) - os.chmod(fname, 420) # 0o644 - except OSError as e: - err("Cannot change permissions of files " - "or directory in `{r}':\n{e}".format(r=theme_path, - e=str(e)), - die=False) - except Exception as e: - err("Cannot copy `{p}' to `{t}':\n{e}".format( - p=path, t=theme_path, e=str(e))) - - -def symlink(path, v=False): - """Symbolically link a theme""" - if not os.path.exists(path): - err(path + ' : no such file or directory') - elif not os.path.isdir(path): - err(path + ' : not a directory') - else: - theme_name = os.path.basename(os.path.normpath(path)) - theme_path = os.path.join(_THEMES_PATH, theme_name) - if os.path.exists(theme_path): - err(path + ' : already exists') - else: - if v: - print("Linking `{p}' to `{t}' ...".format( - p=path, t=theme_path)) - try: - os.symlink(path, theme_path) - except Exception as e: - err("Cannot link `{p}' to `{t}':\n{e}".format( - p=path, t=theme_path, e=str(e))) - - -def is_broken_link(path): - """Returns True if the path given as is a broken symlink""" - path = os.readlink(path) - return not os.path.exists(path) - - -def clean(v=False): - """Removes the broken symbolic links""" - c = 0 - for path in os.listdir(_THEMES_PATH): - path = os.path.join(_THEMES_PATH, path) - if os.path.islink(path): - if is_broken_link(path): - if v: - print('Removing {}'.format(path)) - try: - os.remove(path) - except OSError: - print('Error: cannot remove {}'.format(path)) - else: - c += 1 - - print("\nRemoved {} broken links".format(c)) diff --git a/pelican/tools/templates/Makefile.jinja2 b/pelican/tools/templates/Makefile.jinja2 deleted file mode 100644 index df5b2643..00000000 --- a/pelican/tools/templates/Makefile.jinja2 +++ /dev/null @@ -1,164 +0,0 @@ -PY?={{py_v}} -PELICAN?={{pelican}} -PELICANOPTS={{pelicanopts}} - -BASEDIR=$(CURDIR) -INPUTDIR=$(BASEDIR)/content -OUTPUTDIR=$(BASEDIR)/output -CONFFILE=$(BASEDIR)/pelicanconf.py -PUBLISHCONF=$(BASEDIR)/publishconf.py - -{% if ftp %} -FTP_HOST={{ftp_host}} -FTP_USER={{ftp_user}} -FTP_TARGET_DIR={{ftp_target_dir}} - -{% endif %} -{% if ssh %} -SSH_HOST={{ssh_host}} -SSH_PORT={{ssh_port}} -SSH_USER={{ssh_user}} -SSH_TARGET_DIR={{ssh_target_dir}} - -{% endif %} -{% if s3 %} -S3_BUCKET={{s3_bucket}} - -{% endif %} -{% if cloudfiles %} -CLOUDFILES_USERNAME={{cloudfiles_username}} -CLOUDFILES_API_KEY={{cloudfiles_api_key}} -CLOUDFILES_CONTAINER={{cloudfiles_container}} - -{% endif %} -{% if dropbox %} -DROPBOX_DIR={{dropbox_dir}} - -{% endif %} -{% if github %} -GITHUB_PAGES_BRANCH={{github_pages_branch}} - -{% endif %} - -DEBUG ?= 0 -ifeq ($(DEBUG), 1) - PELICANOPTS += -D -endif - -RELATIVE ?= 0 -ifeq ($(RELATIVE), 1) - PELICANOPTS += --relative-urls -endif - -SERVER ?= "0.0.0.0" - -PORT ?= 0 -ifneq ($(PORT), 0) - PELICANOPTS += -p $(PORT) -endif - - -help: - @echo 'Makefile for a pelican Web site ' - @echo ' ' - @echo 'Usage: ' - @echo ' make html (re)generate the web site ' - @echo ' make clean remove the generated files ' - @echo ' make regenerate regenerate files upon modification ' - @echo ' make publish generate using production settings ' - @echo ' make serve [PORT=8000] serve site at http://localhost:8000' - @echo ' make serve-global [SERVER=0.0.0.0] serve (as root) to $(SERVER):80 ' - @echo ' make devserver [PORT=8000] serve and regenerate together ' - @echo ' make devserver-global regenerate and serve on 0.0.0.0 ' -{% if ssh %} - @echo ' make ssh_upload upload the web site via SSH ' - @echo ' make rsync_upload upload the web site via rsync+ssh ' -{% endif %} -{% if dropbox %} - @echo ' make dropbox_upload upload the web site via Dropbox ' -{% endif %} -{% if ftp %} - @echo ' make ftp_upload upload the web site via FTP ' -{% endif %} -{% if s3 %} - @echo ' make s3_upload upload the web site via S3 ' -{% endif %} -{% if cloudfiles %} - @echo ' make cf_upload upload the web site via Cloud Files' -{% endif %} -{% if github %} - @echo ' make github upload the web site via gh-pages ' -{% endif %} - @echo ' ' - @echo 'Set the DEBUG variable to 1 to enable debugging, e.g. make DEBUG=1 html ' - @echo 'Set the RELATIVE variable to 1 to enable relative urls ' - @echo ' ' - -html: - "$(PELICAN)" "$(INPUTDIR)" -o "$(OUTPUTDIR)" -s "$(CONFFILE)" $(PELICANOPTS) - -clean: - [ ! -d "$(OUTPUTDIR)" ] || rm -rf "$(OUTPUTDIR)" - -regenerate: - "$(PELICAN)" -r "$(INPUTDIR)" -o "$(OUTPUTDIR)" -s "$(CONFFILE)" $(PELICANOPTS) - -serve: - "$(PELICAN)" -l "$(INPUTDIR)" -o "$(OUTPUTDIR)" -s "$(CONFFILE)" $(PELICANOPTS) - -serve-global: - "$(PELICAN)" -l "$(INPUTDIR)" -o "$(OUTPUTDIR)" -s "$(CONFFILE)" $(PELICANOPTS) -b $(SERVER) - -devserver: - "$(PELICAN)" -lr "$(INPUTDIR)" -o "$(OUTPUTDIR)" -s "$(CONFFILE)" $(PELICANOPTS) - -devserver-global: - $(PELICAN) -lr $(INPUTDIR) -o $(OUTPUTDIR) -s $(CONFFILE) $(PELICANOPTS) -b 0.0.0.0 - -publish: - "$(PELICAN)" "$(INPUTDIR)" -o "$(OUTPUTDIR)" -s "$(PUBLISHCONF)" $(PELICANOPTS) - -{% set upload = [] %} -{% if ssh %} -{% set upload = upload + ["ssh_upload"] %} -ssh_upload: publish - scp -P $(SSH_PORT) -r "$(OUTPUTDIR)"/* "$(SSH_USER)@$(SSH_HOST):$(SSH_TARGET_DIR)" - -{% set upload = upload + ["rsync_upload"] %} -rsync_upload: publish - rsync -e "ssh -p $(SSH_PORT)" -P -rvzc --include tags --cvs-exclude --delete "$(OUTPUTDIR)"/ "$(SSH_USER)@$(SSH_HOST):$(SSH_TARGET_DIR)" - -{% endif %} -{% if dropbox %} -{% set upload = upload + ["dropbox_upload"] %} -dropbox_upload: publish - cp -r "$(OUTPUTDIR)"/* "$(DROPBOX_DIR)"/ - -{% endif %} -{% if ftp %} -{% set upload = upload + ["ftp_upload"] %} -ftp_upload: publish - lftp ftp://$(FTP_USER)@$(FTP_HOST) -e "mirror -R $(OUTPUTDIR) $(FTP_TARGET_DIR) ; quit" - -{% endif %} -{% if s3 %} -{% set upload = upload + ["s3_upload"] %} -s3_upload: publish - aws s3 sync "$(OUTPUTDIR)"/ s3://$(S3_BUCKET) --acl public-read --delete - -{% endif %} -{% if cloudfiles %} -{% set upload = upload + ["cf_upload"] %} -cf_upload: publish - cd "$(OUTPUTDIR)" && swift -v -A https://auth.api.rackspacecloud.com/v1.0 -U $(CLOUDFILES_USERNAME) -K $(CLOUDFILES_API_KEY) upload -c $(CLOUDFILES_CONTAINER) . - -{% endif %} -{% if github %} -{% set upload = upload + ["github"] %} -github: publish - ghp-import -m "Generate Pelican site" -b $(GITHUB_PAGES_BRANCH) "$(OUTPUTDIR)" - git push origin $(GITHUB_PAGES_BRANCH) - -{% endif %} - -.PHONY: html help clean regenerate serve serve-global devserver publish {{ upload|join(" ") }} diff --git a/pelican/tools/templates/pelicanconf.py.jinja2 b/pelican/tools/templates/pelicanconf.py.jinja2 deleted file mode 100644 index 4ba0208c..00000000 --- a/pelican/tools/templates/pelicanconf.py.jinja2 +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- # - -AUTHOR = {{author}} -SITENAME = {{sitename}} -SITEURL = '' - -PATH = 'content' - -TIMEZONE = {{timezone}} - -DEFAULT_LANG = {{lang}} - -# Feed generation is usually not desired when developing -FEED_ALL_ATOM = None -CATEGORY_FEED_ATOM = None -TRANSLATION_FEED_ATOM = None -AUTHOR_FEED_ATOM = None -AUTHOR_FEED_RSS = None - -# Blogroll -LINKS = (('Pelican', 'https://getpelican.com/'), - ('Python.org', 'https://www.python.org/'), - ('Jinja2', 'https://palletsprojects.com/p/jinja/'), - ('You can modify those links in your config file', '#'),) - -# Social widget -SOCIAL = (('You can add links in your config file', '#'), - ('Another social link', '#'),) - -DEFAULT_PAGINATION = {{default_pagination}} - -# Uncomment following line if you want document-relative URLs when developing -#RELATIVE_URLS = True diff --git a/pelican/tools/templates/publishconf.py.jinja2 b/pelican/tools/templates/publishconf.py.jinja2 deleted file mode 100755 index bb18966b..00000000 --- a/pelican/tools/templates/publishconf.py.jinja2 +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- # - -# This file is only used if you use `make publish` or -# explicitly specify it as your config file. - -import os -import sys -sys.path.append(os.curdir) -from pelicanconf import * - -# If your site is available via HTTPS, make sure SITEURL begins with https:// -SITEURL = '{{siteurl}}' -RELATIVE_URLS = False - -FEED_ALL_ATOM = 'feeds/all.atom.xml' -CATEGORY_FEED_ATOM = 'feeds/{slug}.atom.xml' - -DELETE_OUTPUT_DIRECTORY = True - -# Following items are often useful when publishing - -#DISQUS_SITENAME = "" -#GOOGLE_ANALYTICS = "" diff --git a/pelican/tools/templates/tasks.py.jinja2 b/pelican/tools/templates/tasks.py.jinja2 deleted file mode 100644 index 861dee37..00000000 --- a/pelican/tools/templates/tasks.py.jinja2 +++ /dev/null @@ -1,176 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import shlex -import shutil -import sys -import datetime - -from invoke import task -from invoke.main import program -from invoke.util import cd -from pelican import main as pelican_main -from pelican.server import ComplexHTTPRequestHandler, RootedHTTPServer -from pelican.settings import DEFAULT_CONFIG, get_settings_from_file - -OPEN_BROWSER_ON_SERVE = True -SETTINGS_FILE_BASE = 'pelicanconf.py' -SETTINGS = {} -SETTINGS.update(DEFAULT_CONFIG) -LOCAL_SETTINGS = get_settings_from_file(SETTINGS_FILE_BASE) -SETTINGS.update(LOCAL_SETTINGS) - -CONFIG = { - 'settings_base': SETTINGS_FILE_BASE, - 'settings_publish': 'publishconf.py', - # Output path. Can be absolute or relative to tasks.py. Default: 'output' - 'deploy_path': SETTINGS['OUTPUT_PATH'], -{% if ssh %} - # Remote server configuration - 'ssh_user': '{{ssh_user}}', - 'ssh_host': '{{ssh_host}}', - 'ssh_port': '{{ssh_port}}', - 'ssh_path': '{{ssh_target_dir}}', -{% endif %} -{% if cloudfiles %} - # Rackspace Cloud Files configuration settings - 'cloudfiles_username': '{{cloudfiles_username}}', - 'cloudfiles_api_key': '{{cloudfiles_api_key}}', - 'cloudfiles_container': '{{cloudfiles_container}}', -{% endif %} -{% if github %} - # Github Pages configuration - 'github_pages_branch': '{{github_pages_branch}}', - 'commit_message': "'Publish site on {}'".format(datetime.date.today().isoformat()), -{% endif %} - # Host and port for `serve` - 'host': 'localhost', - 'port': 8000, -} - -@task -def clean(c): - """Remove generated files""" - if os.path.isdir(CONFIG['deploy_path']): - shutil.rmtree(CONFIG['deploy_path']) - os.makedirs(CONFIG['deploy_path']) - -@task -def build(c): - """Build local version of site""" - pelican_run('-s {settings_base}'.format(**CONFIG)) - -@task -def rebuild(c): - """`build` with the delete switch""" - pelican_run('-d -s {settings_base}'.format(**CONFIG)) - -@task -def regenerate(c): - """Automatically regenerate site upon file modification""" - pelican_run('-r -s {settings_base}'.format(**CONFIG)) - -@task -def serve(c): - """Serve site at http://$HOST:$PORT/ (default is localhost:8000)""" - - class AddressReuseTCPServer(RootedHTTPServer): - allow_reuse_address = True - - server = AddressReuseTCPServer( - CONFIG['deploy_path'], - (CONFIG['host'], CONFIG['port']), - ComplexHTTPRequestHandler) - - if OPEN_BROWSER_ON_SERVE: - # Open site in default browser - import webbrowser - webbrowser.open("http://{host}:{port}".format(**CONFIG)) - - sys.stderr.write('Serving at {host}:{port} ...\n'.format(**CONFIG)) - server.serve_forever() - -@task -def reserve(c): - """`build`, then `serve`""" - build(c) - serve(c) - -@task -def preview(c): - """Build production version of site""" - pelican_run('-s {settings_publish}'.format(**CONFIG)) - -@task -def livereload(c): - """Automatically reload browser tab upon file modification.""" - from livereload import Server - - def cached_build(): - cmd = '-s {settings_base} -e CACHE_CONTENT=True LOAD_CONTENT_CACHE=True' - pelican_run(cmd.format(**CONFIG)) - - cached_build() - server = Server() - theme_path = SETTINGS['THEME'] - watched_globs = [ - CONFIG['settings_base'], - '{}/templates/**/*.html'.format(theme_path), - ] - - content_file_extensions = ['.md', '.rst'] - for extension in content_file_extensions: - content_glob = '{0}/**/*{1}'.format(SETTINGS['PATH'], extension) - watched_globs.append(content_glob) - - static_file_extensions = ['.css', '.js'] - for extension in static_file_extensions: - static_file_glob = '{0}/static/**/*{1}'.format(theme_path, extension) - watched_globs.append(static_file_glob) - - for glob in watched_globs: - server.watch(glob, cached_build) - - if OPEN_BROWSER_ON_SERVE: - # Open site in default browser - import webbrowser - webbrowser.open("http://{host}:{port}".format(**CONFIG)) - - server.serve(host=CONFIG['host'], port=CONFIG['port'], root=CONFIG['deploy_path']) - -{% if cloudfiles %} -@task -def cf_upload(c): - """Publish to Rackspace Cloud Files""" - rebuild(c) - with cd(CONFIG['deploy_path']): - c.run('swift -v -A https://auth.api.rackspacecloud.com/v1.0 ' - '-U {cloudfiles_username} ' - '-K {cloudfiles_api_key} ' - 'upload -c {cloudfiles_container} .'.format(**CONFIG)) -{% endif %} - -@task -def publish(c): - """Publish to production via rsync""" - pelican_run('-s {settings_publish}'.format(**CONFIG)) - c.run( - 'rsync --delete --exclude ".DS_Store" -pthrvz -c ' - '-e "ssh -p {ssh_port}" ' - '{} {ssh_user}@{ssh_host}:{ssh_path}'.format( - CONFIG['deploy_path'].rstrip('/') + '/', - **CONFIG)) - -{% if github %} -@task -def gh_pages(c): - """Publish to GitHub Pages""" - preview(c) - c.run('ghp-import -b {github_pages_branch} ' - '-m {commit_message} ' - '{deploy_path} -p'.format(**CONFIG)) -{% endif %} - -def pelican_run(cmd): - cmd += ' ' + program.core.remainder # allows to pass-through args to pelican - pelican_main(shlex.split(cmd))