move tools in pelican.tools and add the pelican.tools package to setup.py

This commit is contained in:
Andrea Crotti 2012-03-16 18:53:08 +00:00
commit 4e4af9d011
9 changed files with 777 additions and 777 deletions

View file

295
pelican/tools/pelican_import.py Executable file
View file

@ -0,0 +1,295 @@
#!/usr/bin/env python
import argparse
import os
import subprocess
import sys
import time
from codecs import open
from pelican.utils import slugify
def wp2fields(xml):
"""Opens a wordpress XML file, and yield pelican fields"""
from BeautifulSoup import BeautifulStoneSoup
xmlfile = open(xml, encoding='utf-8').read()
soup = BeautifulStoneSoup(xmlfile)
items = soup.rss.channel.findAll('item')
for item in items:
if item.fetch('wp:status')[0].contents[0] == "publish":
title = item.title.contents[0]
content = item.fetch('content:encoded')[0].contents[0]
filename = item.fetch('wp:post_name')[0].contents[0]
raw_date = item.fetch('wp:post_date')[0].contents[0]
date_object = time.strptime(raw_date, "%Y-%m-%d %H:%M:%S")
date = time.strftime("%Y-%m-%d %H:%M", date_object)
author = item.fetch('dc:creator')[0].contents[0].title()
categories = [cat.contents[0] for cat in item.fetch(domain='category')]
# caturl = [cat['nicename'] for cat in item.fetch(domain='category')]
tags = [tag.contents[0].title() for tag in item.fetch(domain='tag', nicename=None)]
yield (title, content, filename, date, author, categories, tags, "html")
def dc2fields(file):
"""Opens a Dotclear export file, and yield pelican fields"""
from BeautifulSoup import BeautifulStoneSoup
in_cat = False
in_post = False
category_list = {}
posts = []
with open(file, 'r', encoding='utf-8') as f:
for line in f:
# remove final \n
line = line[:-1]
if line.startswith('[category'):
in_cat = True
elif line.startswith('[post'):
in_post = True
elif in_cat:
fields = line.split('","')
if not line:
in_cat = False
else:
# remove 1st and last ""
fields[0] = fields[0][1:]
# fields[-1] = fields[-1][:-1]
category_list[fields[0]]=fields[2]
elif in_post:
if not line:
in_post = False
break
else:
posts.append(line)
print("%i posts read." % len(posts))
for post in posts:
fields = post.split('","')
# post_id = fields[0][1:]
# blog_id = fields[1]
# user_id = fields[2]
cat_id = fields[3]
# post_dt = fields[4]
# post_tz = fields[5]
post_creadt = fields[6]
# post_upddt = fields[7]
# post_password = fields[8]
# post_type = fields[9]
post_format = fields[10]
# post_url = fields[11]
# post_lang = fields[12]
post_title = fields[13]
post_excerpt = fields[14]
post_excerpt_xhtml = fields[15]
post_content = fields[16]
post_content_xhtml = fields[17]
# post_notes = fields[18]
# post_words = fields[19]
# post_status = fields[20]
# post_selected = fields[21]
# post_position = fields[22]
# post_open_comment = fields[23]
# post_open_tb = fields[24]
# nb_comment = fields[25]
# nb_trackback = fields[26]
post_meta = fields[27]
# redirect_url = fields[28][:-1]
# remove seconds
post_creadt = ':'.join(post_creadt.split(':')[0:2])
author = ""
categories = []
tags = []
if cat_id:
categories = [category_list[id].strip() for id in cat_id.split(',')]
# Get tags related to a post
tag = post_meta.replace('{', '').replace('}', '').replace('a:1:s:3:\\"tag\\";a:', '').replace('a:0:', '')
if len(tag) > 1:
if int(tag[:1]) == 1:
newtag = tag.split('"')[1]
tags.append(unicode(BeautifulStoneSoup(newtag,convertEntities=BeautifulStoneSoup.HTML_ENTITIES )))
else:
i=1
j=1
while(i <= int(tag[:1])):
newtag = tag.split('"')[j].replace('\\','')
tags.append(unicode(BeautifulStoneSoup(newtag,convertEntities=BeautifulStoneSoup.HTML_ENTITIES )))
i=i+1
if j < int(tag[:1])*2:
j=j+2
"""
dotclear2 does not use markdown by default unless you use the markdown plugin
Ref: http://plugins.dotaddict.org/dc2/details/formatting-markdown
"""
if post_format == "markdown":
content = post_excerpt + post_content
else:
content = post_excerpt_xhtml + post_content_xhtml
content = content.replace('\\n', '')
post_format = "html"
yield (post_title, content, slugify(post_title), post_creadt, author, categories, tags, post_format)
def feed2fields(file):
"""Read a feed and yield pelican fields"""
import feedparser
d = feedparser.parse(file)
for entry in d.entries:
date = (time.strftime("%Y-%m-%d %H:%M", entry.updated_parsed)
if hasattr(entry, "updated_parsed") else None)
author = entry.author if hasattr(entry, "author") else None
tags = [e['term'] for e in entry.tags] if hasattr(entry, "tags") else None
slug = slugify(entry.title)
yield (entry.title, entry.description, slug, date, author, [], tags, "html")
def build_header(title, date, author, categories, tags):
"""Build a header from a list of fields"""
header = '%s\n%s\n' % (title, '#' * len(title))
if date:
header += ':date: %s\n' % date
if categories:
header += ':category: %s\n' % ', '.join(categories)
if tags:
header += ':tags: %s\n' % ', '.join(tags)
header += '\n'
return header
def build_markdown_header(title, date, author, categories, tags):
"""Build a header from a list of fields"""
header = 'Title: %s\n' % title
if date:
header += 'Date: %s\n' % date
if categories:
header += 'Category: %s\n' % ', '.join(categories)
if tags:
header += 'Tags: %s\n' % ', '.join(tags)
header += '\n'
return header
def fields2pelican(fields, out_markup, output_path, dircat=False):
for title, content, filename, date, author, categories, tags, in_markup in fields:
if (in_markup == "markdown") or (out_markup == "markdown") :
ext = '.md'
header = build_markdown_header(title, date, author, categories, tags)
else:
out_markup = "rst"
ext = '.rst'
header = build_header(title, date, author, categories, tags)
filename = os.path.basename(filename)
# option to put files in directories with categories names
if dircat and (len(categories) == 1):
catname = slugify(categories[0])
out_filename = os.path.join(output_path, catname, filename+ext)
if not os.path.isdir(os.path.join(output_path, catname)):
os.mkdir(os.path.join(output_path, catname))
else:
out_filename = os.path.join(output_path, filename+ext)
print(out_filename)
if in_markup == "html":
html_filename = os.path.join(output_path, filename+'.html')
with open(html_filename, 'w', encoding='utf-8') as fp:
# Replace simple newlines with <br />+newline so that the HTML file
# represents the original post more accurately
content = content.replace("\n", "<br />\n")
fp.write(content)
cmd = 'pandoc --normalize --reference-links --from=html --to={0} -o "{1}" "{2}"'.format(
out_markup, out_filename, html_filename)
try:
rc = subprocess.call(cmd, shell=True)
if rc < 0:
print("Child was terminated by signal %d" % -rc)
exit()
elif rc > 0:
print("Please, check your Pandoc installation.")
exit()
except OSError, e:
print("Pandoc execution failed: %s" % e)
exit()
os.remove(html_filename)
with open(out_filename, 'r', encoding='utf-8') as fs:
content = fs.read()
if out_markup == "markdown":
# In markdown, to insert a <br />, end a line with two or more spaces & then a end-of-line
content = content.replace("\\\n ", " \n")
content = content.replace("\\\n", " \n")
with open(out_filename, 'w', encoding='utf-8') as fs:
fs.write(header + content)
def main():
parser = argparse.ArgumentParser(
description="Transform feed, Wordpress or Dotclear files to rst files."
"Be sure to have pandoc installed",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(dest='input', help='The input file to read')
parser.add_argument('--wpfile', action='store_true', dest='wpfile',
help='Wordpress XML export')
parser.add_argument('--dotclear', action='store_true', dest='dotclear',
help='Dotclear export')
parser.add_argument('--feed', action='store_true', dest='feed',
help='Feed to parse')
parser.add_argument('-o', '--output', dest='output', default='output',
help='Output path')
parser.add_argument('-m', '--markup', dest='markup', default='rst',
help='Output markup format (supports rst & markdown)')
parser.add_argument('--dir-cat', action='store_true', dest='dircat',
help='Put files in directories with categories name')
args = parser.parse_args()
input_type = None
if args.wpfile:
input_type = 'wordpress'
elif args.dotclear:
input_type = 'dotclear'
elif args.feed:
input_type = 'feed'
else:
print("You must provide either --wpfile, --dotclear or --feed options")
exit()
if not os.path.exists(args.output):
try:
os.mkdir(args.output)
except OSError:
print("Unable to create the output folder: " + args.output)
exit()
if input_type == 'wordpress':
fields = wp2fields(args.input)
elif input_type == 'dotclear':
fields = dc2fields(args.input)
elif input_type == 'feed':
fields = feed2fields(args.input)
fields2pelican(fields, args.markup, args.output, dircat=args.dircat or False)

View file

@ -0,0 +1,264 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*- #
import os
import string
import argparse
from pelican import __version__
TEMPLATES = {
'Makefile' : '''
PELICAN=$pelican
PELICANOPTS=$pelicanopts
BASEDIR=$$(PWD)
INPUTDIR=$$(BASEDIR)/src
OUTPUTDIR=$$(BASEDIR)/output
CONFFILE=$$(BASEDIR)/pelican.conf.py
FTP_HOST=$ftp_host
FTP_USER=$ftp_user
FTP_TARGET_DIR=$ftp_target_dir
SSH_HOST=$ssh_host
SSH_USER=$ssh_user
SSH_TARGET_DIR=$ssh_target_dir
DROPBOX_DIR=$dropbox_dir
help:
\t@echo 'Makefile for a pelican Web site '
\t@echo ' '
\t@echo 'Usage: '
\t@echo ' make html (re)generate the web site '
\t@echo ' make clean remove the generated files '
\t@echo ' ftp_upload upload the web site using FTP '
\t@echo ' ssh_upload upload the web site using SSH '
\t@echo ' dropbox_upload upload the web site using Dropbox '
\t@echo ' '
html: clean $$(OUTPUTDIR)/index.html
\t@echo 'Done'
$$(OUTPUTDIR)/%.html:
\t$$(PELICAN) $$(INPUTDIR) -o $$(OUTPUTDIR) -s $$(CONFFILE) $$(PELICANOPTS)
clean:
\trm -fr $$(OUTPUTDIR)
\tmkdir $$(OUTPUTDIR)
dropbox_upload: $$(OUTPUTDIR)/index.html
\tcp -r $$(OUTPUTDIR)/* $$(DROPBOX_DIR)
ssh_upload: $$(OUTPUTDIR)/index.html
\tscp -r $$(OUTPUTDIR)/* $$(SSH_USER)@$$(SSH_HOST):$$(SSH_TARGET_DIR)
ftp_upload: $$(OUTPUTDIR)/index.html
\tlftp ftp://$$(FTP_USER)@$$(FTP_HOST) -e "mirror -R $$(OUTPUT_DIR)/* $$(FTP_TARGET_DIR) ; quit"
github: $$(OUTPUTDIR)/index.html
\tghp-import $$(OUTPUTDIR)
\tgit push origin gh-pages
.PHONY: html help clean ftp_upload ssh_upload dropbox_upload github
''',
'pelican.conf.py': '''#!/usr/bin/env python
# -*- coding: utf-8 -*- #
AUTHOR = u"$author"
SITENAME = u"$sitename"
SITEURL = '/'
TIMEZONE = 'Europe/Paris'
DEFAULT_LANG='$lang'
# Blogroll
LINKS = (
('Pelican', 'http://docs.notmyidea.org/alexis/pelican/'),
('Python.org', 'http://python.org'),
('Jinja2', 'http://jinja.pocoo.org'),
('You can modify those links in your config file', '#')
)
# Social widget
SOCIAL = (
('You can add links in your config file', '#'),
)
DEFAULT_PAGINATION = $default_pagination
'''
}
CONF = {
'pelican' : 'pelican',
'pelicanopts' : '',
'basedir': '.',
'ftp_host': 'localhost',
'ftp_user': 'anonymous',
'ftp_target_dir': '/',
'ssh_host': 'locahost',
'ssh_user': 'root',
'ssh_target_dir': '/var/www',
'dropbox_dir' : '~/Dropbox/Public/',
'default_pagination' : 10,
'lang': 'en'
}
def ask(question, answer=str, default=None, l=None):
if answer == str:
r = ''
while True:
if default:
r = raw_input('> {0} [{1}] '.format(question, default))
else:
r = raw_input('> {0} '.format(question, default))
r = r.strip()
if len(r) <= 0:
if default:
r = default
break
else:
print('You must enter something')
else:
if l and len(r) != l:
print('You must enter a {0} letters long string'.format(l))
else:
break
return r
elif answer == bool:
r = None
while True:
if default is True:
r = raw_input('> {0} (Y/n) '.format(question))
elif default is False:
r = raw_input('> {0} (y/N) '.format(question))
else:
r = raw_input('> {0} (y/n) '.format(question))
r = r.strip().lower()
if r in ('y', 'yes'):
r = True
break
elif r in ('n', 'no'):
r = False
break
elif not r:
r = default
break
else:
print("You must answer `yes' or `no'")
return r
elif answer == int:
r = None
while True:
if default:
r = raw_input('> {0} [{1}] '.format(question, default))
else:
r = raw_input('> {0} '.format(question))
r = r.strip()
if not r:
r = default
break
try:
r = int(r)
break
except:
print('You must enter an integer')
return r
else:
raise NotImplemented('Arguent `answer` must be str, bool or integer')
def main():
parser = argparse.ArgumentParser(
description="A kickstarter for pelican",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-p', '--path', default=".",
help="The path to generate the blog into")
parser.add_argument('-t', '--title', metavar="title",
help='Set the title of the website')
parser.add_argument('-a', '--author', metavar="author",
help='Set the author name of the website')
parser.add_argument('-l', '--lang', metavar="lang",
help='Set the default lang of the website')
args = parser.parse_args()
print('''Welcome to pelican-quickstart v{v}.
This script will help you creating a new Pelican based website.
Please answer the following questions so this script can generate the files needed by Pelican.
'''.format(v=__version__))
CONF['basedir'] = os.path.abspath(ask('Where do you want to create your new Web site ?', answer=str, default=args.path))
CONF['sitename'] = ask('How will you call your Web site ?', answer=str, default=args.title)
CONF['author'] = ask('Who will be the author of this Web site ?', answer=str, default=args.author)
CONF['lang'] = ask('What will be the default language of this Web site ?', str, args.lang or CONF['lang'], 2)
CONF['with_pagination'] = ask('Do you want to enable article pagination ?', bool, bool(CONF['default_pagination']))
if CONF['with_pagination']:
CONF['default_pagination'] = ask('So how many articles per page do you want ?', int, CONF['default_pagination'])
else:
CONF['default_pagination'] = False
mkfile = ask('Do you want to generate a Makefile to easily manage your website ?', bool, True)
if mkfile:
if ask('Do you want to upload your website using FTP ?', answer=bool, default=False):
CONF['ftp_host'] = ask('What is the hostname of your FTP server ?', str, CONF['ftp_host'])
CONF['ftp_user'] = ask('What is your username on this server ?', str, CONF['ftp_user'])
CONF['ftp_traget_dir'] = ask('Where do you want to put your website on this server ?', str, CONF['ftp_target_dir'])
if ask('Do you want to upload your website using SSH ?', answer=bool, default=False):
CONF['ssh_host'] = ask('What is the hostname of your SSH server ?', str, CONF['ssh_host'])
CONF['ssh_user'] = ask('What is your username on this server ?', str, CONF['ssh_user'])
CONF['ssh_traget_dir'] = ask('Where do you want to put your website on this server ?', str, CONF['ssh_target_dir'])
if ask('Do you want to upload your website using Dropbox ?', answer=bool, default=False):
CONF['dropbox_dir'] = ask('Where is your Dropbox directory ?', str, CONF['dropbox_dir'])
try:
os.makedirs(os.path.join(CONF['basedir'], 'src'))
except OSError, e:
print('Error: {0}'.format(e))
try:
os.makedirs(os.path.join(CONF['basedir'], 'output'))
except OSError, e:
print('Error: {0}'.format(e))
conf = string.Template(TEMPLATES['pelican.conf.py'])
try:
with open(os.path.join(CONF['basedir'], 'pelican.conf.py'), 'w') as fd:
fd.write(conf.safe_substitute(CONF))
fd.close()
except OSError, e:
print('Error: {0}'.format(e))
if mkfile:
Makefile = string.Template(TEMPLATES['Makefile'])
try:
with open(os.path.join(CONF['basedir'], 'Makefile'), 'w') as fd:
fd.write(Makefile.safe_substitute(CONF))
fd.close()
except OSError, e:
print('Error: {0}'.format(e))
print('Done. Your new project is available at %s' % CONF['basedir'])

214
pelican/tools/pelican_themes.py Executable file
View file

@ -0,0 +1,214 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import argparse
import os
import shutil
import sys
try:
import pelican
except:
err('Cannot import pelican.\nYou must install Pelican in order to run this script.', -1)
global _THEMES_PATH
_THEMES_PATH = os.path.join(
os.path.dirname(
os.path.abspath(
pelican.__file__
)
),
'themes'
)
__version__ = '0.2'
_BUILTIN_THEMES = ['simple', 'notmyidea']
def err(msg, die=None):
"""Print an error message and exits if an exit code is given"""
sys.stderr.write(str(msg) + '\n')
if die:
sys.exit((die if type(die) is int else 1))
def main():
"""Main function"""
parser = argparse.ArgumentParser(description="""Install themes for Pelican""")
excl= parser.add_mutually_exclusive_group()
excl.add_argument('-l', '--list', dest='action', action="store_const", const='list',
help="Show the themes already installed and exit")
excl.add_argument('-p', '--path', dest='action', action="store_const", const='path',
help="Show the themes path and exit")
excl.add_argument('-V', '--version', action='version', version='pelican-themes v{0}'.format(__version__),
help='Print the version of this script')
parser.add_argument('-i', '--install', dest='to_install', nargs='+', metavar="theme path",
help='The themes to install ')
parser.add_argument('-r', '--remove', dest='to_remove', nargs='+', metavar="theme name",
help='The themes to remove')
parser.add_argument('-s', '--symlink', dest='to_symlink', nargs='+', metavar="theme path",
help="Same as `--install', but create a symbolic link instead of copying the theme. Useful for theme development")
parser.add_argument('-c', '--clean', dest='clean', action="store_true",
help="Remove the broken symbolic links of the theme path")
parser.add_argument('-v', '--verbose', dest='verbose', action="store_true",
help="Verbose output")
args = parser.parse_args()
if args.action:
if args.action is 'list':
list_themes(args.verbose)
elif args.action is 'path':
print(_THEMES_PATH)
elif args.to_install or args.to_remove or args.to_symlink or args.clean:
if args.to_remove:
if args.verbose:
print('Removing themes...')
for i in args.to_remove:
remove(i, v=args.verbose)
if args.to_install:
if args.verbose:
print('Installing themes...')
for i in args.to_install:
install(i, v=args.verbose)
if args.to_symlink:
if args.verbose:
print('Linking themes...')
for i in args.to_symlink:
symlink(i, v=args.verbose)
if args.clean:
if args.verbose:
print('Cleaning the themes directory...')
clean(v=args.verbose)
else:
print('No argument given... exiting.')
def themes():
"""Returns the list of the themes"""
for i in os.listdir(_THEMES_PATH):
e = os.path.join(_THEMES_PATH, i)
if os.path.isdir(e):
if os.path.islink(e):
yield (e, os.readlink(e))
else:
yield (e, None)
def list_themes(v=False):
"""Display the list of the themes"""
for t, l in themes():
if not v:
t = os.path.basename(t)
if l:
if v:
print(t + (" (symbolic link to `" + l + "')"))
else:
print(t + '@')
else:
print(t)
def remove(theme_name, v=False):
"""Removes a theme"""
theme_name = theme_name.replace('/','')
target = os.path.join(_THEMES_PATH, theme_name)
if theme_name in _BUILTIN_THEMES:
err(theme_name + ' is a builtin theme.\nYou cannot remove a builtin theme with this script, remove it by hand if you want.')
elif os.path.islink(target):
if v:
print('Removing link `' + target + "'")
os.remove(target)
elif os.path.isdir(target):
if v:
print('Removing directory `' + target + "'")
shutil.rmtree(target)
elif os.path.exists(target):
err(target + ' : not a valid theme')
else:
err(target + ' : no such file or directory')
def install(path, v=False):
"""Installs a theme"""
if not os.path.exists(path):
err(path + ' : no such file or directory')
elif not os.path.isdir(path):
err(path + ' : no a directory')
else:
theme_name = os.path.basename(os.path.normpath(path))
theme_path = os.path.join(_THEMES_PATH, theme_name)
if os.path.exists(theme_path):
err(path + ' : already exists')
else:
if v:
print("Copying `{p}' to `{t}' ...".format(p=path, t=theme_path))
try:
shutil.copytree(path, theme_path)
except Exception, e:
err("Cannot copy `{p}' to `{t}':\n{e}".format(p=path, t=theme_path, e=str(e)))
def symlink(path, v=False):
"""Symbolically link a theme"""
if not os.path.exists(path):
err(path + ' : no such file or directory')
elif not os.path.isdir(path):
err(path + ' : no a directory')
else:
theme_name = os.path.basename(os.path.normpath(path))
theme_path = os.path.join(_THEMES_PATH, theme_name)
if os.path.exists(theme_path):
err(path + ' : already exists')
else:
if v:
print("Linking `{p}' to `{t}' ...".format(p=path, t=theme_path))
try:
os.symlink(path, theme_path)
except Exception, e:
err("Cannot link `{p}' to `{t}':\n{e}".format(p=path, t=theme_path, e=str(e)))
def is_broken_link(path):
"""Returns True if the path given as is a broken symlink"""
path = os.readlink(path)
return not os.path.exists(path)
def clean(v=False):
"""Removes the broken symbolic links"""
c=0
for path in os.listdir(_THEMES_PATH):
path = os.path.join(_THEMES_PATH, path)
if os.path.islink(path):
if is_broken_link(path):
if v:
print('Removing {0}'.format(path))
try:
os.remove(path)
except OSError, e:
print('Error: cannot remove {0}'.format(path))
else:
c+=1
print("\nRemoved {0} broken links".format(c))