import os, sys, re, argparse, subprocess, time
from html.parser import HTMLParser
+VALID_PAGES = 'README INSTALL COPYING rsync.1 rrsync.1 rsync-ssl.1 rsyncd.conf.5'.split()
+
CONSUMES_TXT = set('h1 h2 h3 p li pre'.split())
HTML_START = """\
<html><head>
-<title>%s</title>
+<title>%TITLE%</title>
+<meta charset="UTF-8"/>
<link href="https://fonts.googleapis.com/css2?family=Roboto&family=Roboto+Mono&display=swap" rel="stylesheet">
<style>
body {
body, b, strong, u {
font-family: 'Roboto', sans-serif;
}
+a.tgt { font-face: symbol; font-weight: 400; font-size: 70%; visibility: hidden; text-decoration: none; color: #ddd; padding: 0 4px; border: 0; }
+a.tgt:after { content: '🔗'; }
+a.tgt:hover { color: #444; background-color: #eaeaea; }
+h1:hover > a.tgt, h2:hover > a.tgt, h3:hover > a.tgt, dt:hover > a.tgt { visibility: visible; }
code {
font-family: 'Roboto Mono', monospace;
font-weight: bold;
NBR_DASH = ('\4', r"\-")
NBR_SPACE = ('\xa0', r"\ ")
+FILENAME_RE = re.compile(r'^(?P<fn>(?P<srcdir>.+/)?(?P<name>(?P<prog>[^/]+?)(\.(?P<sect>\d+))?)\.md)$')
+ASSIGNMENT_RE = re.compile(r'^(\w+)=(.+)')
+VER_RE = re.compile(r'^#define\s+RSYNC_VERSION\s+"(\d.+?)"', re.M)
+TZ_RE = re.compile(r'^#define\s+MAINTAINER_TZ_OFFSET\s+(-?\d+(\.\d+)?)', re.M)
+VAR_REF_RE = re.compile(r'\$\{(\w+)\}')
+VERSION_RE = re.compile(r' (\d[.\d]+)[, ]')
+BIN_CHARS_RE = re.compile(r'[\1-\7]+')
+SPACE_DOUBLE_DASH_RE = re.compile(r'\s--(\s)')
+NON_SPACE_SINGLE_DASH_RE = re.compile(r'(^|\W)-')
+WHITESPACE_RE = re.compile(r'\s')
+CODE_BLOCK_RE = re.compile(r'[%s]([^=%s]+)[=%s]' % (BOLD_FONT[0], NORM_FONT[0], NORM_FONT[0]))
+NBR_DASH_RE = re.compile(r'[%s]' % NBR_DASH[0])
+INVALID_TARGET_CHARS_RE = re.compile(r'[^-A-Za-z0-9._]')
+INVALID_START_CHAR_RE = re.compile(r'^([^A-Za-z0-9])')
+MANIFY_LINESTART_RE = re.compile(r"^(['.])", flags=re.M)
+
md_parser = None
env_subs = { }
+warning_count = 0
+
def main():
for mdfn in args.mdfiles:
parse_md_file(mdfn)
def parse_md_file(mdfn):
- fi = re.match(r'^(?P<fn>(?P<srcdir>.+/)?(?P<name>(?P<prog>[^/]+?)(\.(?P<sect>\d+))?)\.md)$', mdfn)
+ fi = FILENAME_RE.match(mdfn)
if not fi:
die('Failed to parse a md input file name:', mdfn)
fi = argparse.Namespace(**fi.groupdict())
fi.want_manpage = not not fi.sect
if fi.want_manpage:
- fi.title = fi.prog + '(' + fi.sect + ') man page'
+ fi.title = fi.prog + '(' + fi.sect + ') manpage'
else:
- fi.title = fi.prog
+ fi.title = fi.prog + ' for rsync'
if fi.want_manpage:
if not env_subs:
if fi.want_manpage:
output_list += [ (fi.name, fi.man_out) ]
for fn, txt in output_list:
+ if args.dest and args.dest != '.':
+ fn = os.path.join(args.dest, fn)
if os.path.lexists(fn):
os.unlink(fn)
print("Wrote:", fn)
env_subs['VERSION'] = '1.0.0'
env_subs['bindir'] = '/usr/bin'
env_subs['libdir'] = '/usr/lib/rsync'
+ tz_offset = 0
else:
for fn in (srcdir + 'version.h', 'Makefile'):
try:
with open(srcdir + 'version.h', 'r', encoding='utf-8') as fh:
txt = fh.read()
- m = re.search(r'"(.+?)"', txt)
+ m = VER_RE.search(txt)
env_subs['VERSION'] = m.group(1)
+ m = TZ_RE.search(txt) # the tzdata lib may not be installed, so we use a simple hour offset
+ tz_offset = float(m.group(1)) * 60 * 60
with open('Makefile', 'r', encoding='utf-8') as fh:
for line in fh:
- m = re.match(r'^(\w+)=(.+)', line)
+ m = ASSIGNMENT_RE.match(line)
if not m:
continue
var, val = (m.group(1), m.group(2))
if var == 'prefix' and env_subs[var] is not None:
continue
- while re.search(r'\$\{', val):
- val = re.sub(r'\$\{(\w+)\}', lambda m: env_subs[m.group(1)], val)
+ while VAR_REF_RE.search(val):
+ val = VAR_REF_RE.sub(lambda m: env_subs[m.group(1)], val)
env_subs[var] = val
if var == 'srcdir':
break
- env_subs['date'] = time.strftime('%d %b %Y', time.localtime(mtime))
+ env_subs['date'] = time.strftime('%d %b %Y', time.gmtime(mtime + tz_offset)).lstrip('0')
def html_via_commonmark(txt):
def __init__(self, fi):
HTMLParser.__init__(self, convert_charrefs=True)
+ self.fn = fi.fn
+
st = self.state = argparse.Namespace(
list_state = [ ],
p_macro = ".P\n",
dt_from = None,
in_pre = False,
in_code = False,
- html_out = [ HTML_START % fi.title ],
+ html_out = [ HTML_START.replace('%TITLE%', fi.title) ],
man_out = [ ],
txt = '',
want_manpage = fi.want_manpage,
+ created_hashtags = set(),
+ derived_hashtags = set(),
+ referenced_hashtags = set(),
+ bad_hashtags = set(),
+ latest_targets = [ ],
+ opt_prefix = 'opt',
+ a_href = None,
+ a_href_external = False,
+ a_txt_start = None,
+ after_a_tag = False,
+ target_suf = '',
)
if st.want_manpage:
fi.man_out = ''.join(st.man_out)
st.man_out = None
+ for tgt, txt in st.derived_hashtags:
+ derived = txt2target(txt, tgt)
+ if derived not in st.created_hashtags:
+ txt = BIN_CHARS_RE.sub('', txt.replace(NBR_DASH[0], '-').replace(NBR_SPACE[0], ' '))
+ warn('Unknown derived hashtag link in', self.fn, 'based on:', (tgt, txt))
+
+ for bad in st.bad_hashtags:
+ if bad in st.created_hashtags:
+ warn('Missing "#" in hashtag link in', self.fn + ':', bad)
+ else:
+ warn('Unknown non-hashtag link in', self.fn + ':', bad)
+
+ for bad in st.referenced_hashtags - st.created_hashtags:
+ warn('Unknown hashtag link in', self.fn + ':', '#' + bad)
+
+ def handle_UE(self):
+ st = self.state
+ if st.txt.startswith(('.', ',', '!', '?', ';', ':')):
+ st.man_out[-1] = ".UE " + st.txt[0] + "\n"
+ st.txt = st.txt[1:]
+ st.after_a_tag = False
def handle_starttag(self, tag, attrs_list):
st = self.state
st.txt += BOLD_FONT[0]
elif tag == 'em' or tag == 'i':
if st.want_manpage:
- tag = 'u' # Change it into underline to be more like the man page
+ tag = 'u' # Change it into underline to be more like the manpage
st.txt += UNDR_FONT[0]
elif tag == 'ol':
start = 1
st.man_out.append(".l\n")
st.html_out.append("<hr />")
return
+ elif tag == 'a':
+ st.a_href = None
+ for var, val in attrs_list:
+ if var == 'href':
+ if val.startswith(('https://', 'http://', 'mailto:', 'ftp:')):
+ if st.after_a_tag:
+ self.handle_UE()
+ st.man_out.append(manify(st.txt.strip()) + "\n")
+ st.man_out.append(".UR " + val + "\n")
+ st.txt = ''
+ st.a_href = val
+ st.a_href_external = True
+ elif '#' in val:
+ pg, tgt = val.split('#', 1)
+ if pg and pg not in VALID_PAGES or '#' in tgt:
+ st.bad_hashtags.add(val)
+ elif tgt in ('', 'opt', 'dopt'):
+ st.a_href = val
+ st.a_href_external = False
+ elif pg == '':
+ st.referenced_hashtags.add(tgt)
+ if tgt in st.latest_targets:
+ warn('Found link to the current section in', self.fn + ':', val)
+ elif val not in VALID_PAGES:
+ st.bad_hashtags.add(val)
+ st.a_txt_start = len(st.txt)
st.html_out.append('<' + tag + ''.join(' ' + var + '="' + htmlify(val) + '"' for var, val in attrs_list) + '>')
st.at_first_tag_in_dd = False
- def add_target(self, txt):
- st = self.state
- txt = re.sub(r'[%s](.+?)[=%s].*' % (BOLD_FONT[0], NORM_FONT[0]), r'\1', txt.strip())
- txt = re.sub(r'[%s]' % NBR_DASH[0], '-', txt)
- txt = re.sub(r'[\1-\7]+', '', txt)
- txt = re.sub(r'[^-A-Za-z0-9._]', '_', txt)
- if txt.startswith('-'):
- txt = 'opt' + txt
- else:
- txt = re.sub(r'^([^A-Za-z])', r't\1', txt)
- if txt:
- st.html_out.append('<span id="' + txt + '"></span>')
-
-
def handle_endtag(self, tag):
st = self.state
if args.debug:
self.output_debug('END', (tag,))
+ if st.after_a_tag:
+ self.handle_UE()
if tag in CONSUMES_TXT or st.dt_from == tag:
txt = st.txt.strip()
st.txt = ''
else:
txt = None
add_to_txt = None
- if tag == 'h1' or tag == 'h2':
+ if tag == 'h1':
+ tgt = txt
+ target_suf = ''
+ if tgt.startswith('NEWS for '):
+ m = VERSION_RE.search(tgt)
+ if m:
+ tgt = m.group(1)
+ st.target_suf = '-' + tgt
+ self.add_targets(tag, tgt)
+ elif tag == 'h2':
st.man_out.append(st.p_macro + '.SH "' + manify(txt) + '"\n')
- self.add_target(txt)
+ self.add_targets(tag, txt, st.target_suf)
+ st.opt_prefix = 'dopt' if txt == 'DAEMON OPTIONS' else 'opt'
elif tag == 'h3':
st.man_out.append(st.p_macro + '.SS "' + manify(txt) + '"\n')
- self.add_target(txt)
+ self.add_targets(tag, txt, st.target_suf)
elif tag == 'p':
if st.dt_from == 'p':
tag = 'dt'
st.man_out.append('.IP "' + manify(txt) + '"\n')
if txt.startswith(BOLD_FONT[0]):
- self.add_target(txt)
+ self.add_targets(tag, txt)
st.dt_from = None
elif txt != '':
st.man_out.append(manify(txt) + "\n")
add_to_txt = NORM_FONT[0]
elif tag == 'em' or tag == 'i':
if st.want_manpage:
- tag = 'u' # Change it into underline to be more like the man page
+ tag = 'u' # Change it into underline to be more like the manpage
add_to_txt = NORM_FONT[0]
elif tag == 'ol' or tag == 'ul':
if st.list_state.pop() == 'dl':
st.at_first_tag_in_dd = False
elif tag == 'hr':
return
+ elif tag == 'a':
+ if st.a_href_external:
+ st.txt = st.txt.strip()
+ if args.force_link_text or st.a_href != st.txt:
+ st.man_out.append(manify(st.txt) + "\n")
+ st.man_out.append(".UE\n") # This might get replaced with a punctuation version in handle_UE()
+ st.after_a_tag = True
+ st.a_href_external = False
+ st.txt = ''
+ elif st.a_href:
+ atxt = st.txt[st.a_txt_start:]
+ find = 'href="' + st.a_href + '"'
+ for j in range(len(st.html_out)-1, 0, -1):
+ if find in st.html_out[j]:
+ pg, tgt = st.a_href.split('#', 1)
+ derived = txt2target(atxt, tgt)
+ if pg == '':
+ if derived in st.latest_targets:
+ warn('Found link to the current section in', self.fn + ':', st.a_href)
+ st.derived_hashtags.add((tgt, atxt))
+ st.html_out[j] = st.html_out[j].replace(find, 'href="' + pg + '#' + derived + '"')
+ break
+ else:
+ die('INTERNAL ERROR: failed to find href in html data:', find)
st.html_out.append('</' + tag + '>')
if add_to_txt:
if txt is None:
def handle_data(self, txt):
st = self.state
+ if '](' in txt:
+ warn('Malformed link in', self.fn + ':', txt)
if args.debug:
self.output_debug('DATA', (txt,))
if st.in_pre:
html = htmlify(txt)
else:
- txt = re.sub(r'\s--(\s)', NBR_SPACE[0] + r'--\1', txt).replace('--', NBR_DASH[0]*2)
- txt = re.sub(r'(^|\W)-', r'\1' + NBR_DASH[0], txt)
+ txt = SPACE_DOUBLE_DASH_RE.sub(NBR_SPACE[0] + r'--\1', txt).replace('--', NBR_DASH[0]*2)
+ txt = NON_SPACE_SINGLE_DASH_RE.sub(r'\1' + NBR_DASH[0], txt)
html = htmlify(txt)
if st.in_code:
- txt = re.sub(r'\s', NBR_SPACE[0], txt)
+ txt = WHITESPACE_RE.sub(NBR_SPACE[0], txt)
html = html.replace(NBR_DASH[0], '-').replace(NBR_SPACE[0], ' ') # <code> is non-breaking in CSS
st.html_out.append(html.replace(NBR_SPACE[0], ' ').replace(NBR_DASH[0], '-⁠'))
st.txt += txt
+ def add_targets(self, tag, txt, suf=None):
+ st = self.state
+ tag = '<' + tag + '>'
+ targets = CODE_BLOCK_RE.findall(txt)
+ if not targets:
+ targets = [ txt ]
+ tag_pos = 0
+ for txt in targets:
+ txt = txt2target(txt, st.opt_prefix)
+ if not txt:
+ continue
+ if suf:
+ txt += suf
+ if txt in st.created_hashtags:
+ for j in range(2, 1000):
+ chk = txt + '-' + str(j)
+ if chk not in st.created_hashtags:
+ print('Made link target unique:', chk)
+ txt = chk
+ break
+ if tag_pos == 0:
+ tag_pos -= 1
+ while st.html_out[tag_pos] != tag:
+ tag_pos -= 1
+ st.html_out[tag_pos] = tag[:-1] + ' id="' + txt + '">'
+ st.html_out.append('<a href="#' + txt + '" class="tgt"></a>')
+ tag_pos -= 1 # take into account the append
+ else:
+ st.html_out[tag_pos] = '<span id="' + txt + '"></span>' + st.html_out[tag_pos]
+ st.created_hashtags.add(txt)
+ st.latest_targets = targets
+
+
def output_debug(self, event, extra):
import pprint
st = self.state
pprint.PrettyPrinter(indent=2).pprint(vars(st))
+def txt2target(txt, opt_prefix):
+ txt = txt.strip().rstrip(':')
+ m = CODE_BLOCK_RE.search(txt)
+ if m:
+ txt = m.group(1)
+ txt = NBR_DASH_RE.sub('-', txt)
+ txt = BIN_CHARS_RE.sub('', txt)
+ txt = INVALID_TARGET_CHARS_RE.sub('_', txt)
+ if opt_prefix and txt.startswith('-'):
+ txt = opt_prefix + txt
+ else:
+ txt = INVALID_START_CHAR_RE.sub(r't\1', txt)
+ return txt
+
+
def manify(txt):
- return re.sub(r"^(['.])", r'\&\1', txt.replace('\\', '\\\\')
+ return MANIFY_LINESTART_RE.sub(r'\&\1', txt.replace('\\', '\\\\')
.replace(NBR_SPACE[0], NBR_SPACE[1])
.replace(NBR_DASH[0], NBR_DASH[1])
.replace(NORM_FONT[0], NORM_FONT[1])
.replace(BOLD_FONT[0], BOLD_FONT[1])
- .replace(UNDR_FONT[0], UNDR_FONT[1]), flags=re.M)
+ .replace(UNDR_FONT[0], UNDR_FONT[1]))
def htmlify(txt):
def warn(*msg):
print(*msg, file=sys.stderr)
+ global warning_count
+ warning_count += 1
def die(*msg):
if __name__ == '__main__':
- parser = argparse.ArgumentParser(description="Output html and (optionally) nroff for markdown pages.", add_help=False)
+ parser = argparse.ArgumentParser(description="Convert markdown into html and (optionally) nroff. Each input filename must have a .md suffix, which is changed to .html for the output filename. If the input filename ends with .num.md (e.g. foo.1.md) then a nroff file is also output with the input filename's .md suffix removed (e.g. foo.1).", add_help=False)
parser.add_argument('--test', action='store_true', help="Just test the parsing without outputting any files.")
+ parser.add_argument('--dest', metavar='DIR', help="Create files in DIR instead of the current directory.")
+ parser.add_argument('--force-link-text', action='store_true', help="Don't remove the link text if it matches the link href. Useful when nroff doesn't understand .UR and .UE.")
parser.add_argument('--debug', '-D', action='count', default=0, help='Output copious info on the html parsing. Repeat for even more.')
parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.")
- parser.add_argument("mdfiles", nargs='+', help="The source .md files to convert.")
+ parser.add_argument("mdfiles", metavar='FILE.md', nargs='+', help="One or more .md files to convert.")
args = parser.parse_args()
try:
gfm_parser = None
main()
+ if warning_count:
+ sys.exit(1)
git.samba.org / rsync.git / blobdiff