git.samba.org - rsync.git/blob

1 #!/usr/bin/env python3

3 # This script transforms markdown files into html and (optionally) nroff. The

4 # output files are written into the current directory named for the input file

5 # without the .md suffix and either the .html suffix or no suffix.

6 #

7 # If the input .md file has a section number at the end of the name (e.g.,

8 # rsync.1.md) a nroff file is also output (PROJ.NUM.md -> PROJ.NUM).

9 #

10 # The markdown input format has one extra extension: if a numbered list starts

11 # at 0, it is turned into a description list. The dl's dt tag is taken from the

12 # contents of the first tag inside the li, which is usually a p, code, or

13 # strong tag.

14 #

15 # The cmarkgfm or commonmark lib is used to transforms the input file into

16 # html. Then, the html.parser is used as a state machine that lets us tweak

17 # the html and (optionally) output nroff data based on the html tags.

18 #

19 # If the string @USE_GFM_PARSER@ exists in the file, the string is removed and

20 # a github-flavored-markup parser is used to parse the file.

21 #

22 # The man-page .md files also get the vars @VERSION@, @BINDIR@, and @LIBDIR@

23 # substituted. Some of these values depend on the Makefile $(prefix) (see the

24 # generated Makefile). If the maintainer wants to build files for /usr/local

25 # while creating release-ready man-page files for /usr, use the environment to

26 # set RSYNC_OVERRIDE_PREFIX=/usr.

29 #

30 # This program is freely redistributable.

32 import os, sys, re, argparse, subprocess, time

33 from html.parser import HTMLParser

35 CONSUMES_TXT = set('h1 h2 h3 p li pre'.split())

37 HTML_START = """\

38 <html><head>

39 <title>%TITLE%</title>

40 <meta charset="UTF-8"/>

41 <link href="https://fonts.googleapis.com/css2?family=Roboto&family=Roboto+Mono&display=swap" rel="stylesheet">

42 <style>

43 body {

44 max-width: 50em;

45 margin: auto;

46 }

47 body, b, strong, u {

48 font-family: 'Roboto', sans-serif;

49 }

50 a.tgt { font-face: symbol; font-weight: 400; font-size: 70%; visibility: hidden; text-decoration: none; color: #ddd; padding: 0 4px; border: 0; vertical-align: top; }

51 a.tgt:after { content: '🔗'; }

52 a.tgt:hover { color: #444; background-color: #eaeaea; }

53 h1:hover > a.tgt, h2:hover > a.tgt, h3:hover > a.tgt, dt:hover > a.tgt { visibility: visible; }

54 code {

55 font-family: 'Roboto Mono', monospace;

56 font-weight: bold;

57 white-space: pre;

58 }

59 pre code {

60 display: block;

61 font-weight: normal;

62 }

63 blockquote pre code {

64 background: #f1f1f1;

65 }

66 dd p:first-of-type {

67 margin-block-start: 0em;

68 }

69 </style>

70 </head><body>

71 """

73 TABLE_STYLE = """\

74 table {

75 border-color: grey;

76 border-spacing: 0;

77 }

78 tr {

79 border-top: 1px solid grey;

80 }

81 tr:nth-child(2n) {

82 background-color: #f6f8fa;

83 }

84 th, td {

85 border: 1px solid #dfe2e5;

86 text-align: center;

87 padding-left: 1em;

88 padding-right: 1em;

89 }

90 """

92 MAN_HTML_END = """\

93 <div style="float: right"><p><i>%s</i></p></div>

94 """

96 HTML_END = """\

97 </body></html>

98 """

100 MAN_START = r"""

101 .TH "%s" "%s" "%s" "%s" "User Commands"

102 .\" prefix=%s

103 """.lstrip()

104

105 MAN_END = """\

106 """

107

108 NORM_FONT = ('\1', r"\fP")

109 BOLD_FONT = ('\2', r"\fB")

110 UNDR_FONT = ('\3', r"\fI")

111 NBR_DASH = ('\4', r"\-")

112 NBR_SPACE = ('\xa0', r"\ ")

113

114 FILENAME_RE = re.compile(r'^(?P<fn>(?P<srcdir>.+/)?(?P<name>(?P<prog>[^/]+?)(\.(?P<sect>\d+))?)\.md)$')

115 ASSIGNMENT_RE = re.compile(r'^(\w+)=(.+)')

116 QUOTED_RE = re.compile(r'"(.+?)"')

117 VAR_REF_RE = re.compile(r'\$\{(\w+)\}')

118 VERSION_RE = re.compile(r' (\d[.\d]+)[, ]')

119 BIN_CHARS_RE = re.compile(r'[\1-\7]+')

120 SPACE_DOUBLE_DASH_RE = re.compile(r'\s--(\s)')

121 NON_SPACE_SINGLE_DASH_RE = re.compile(r'(^|\W)-')

122 WHITESPACE_RE = re.compile(r'\s')

123 CODE_BLOCK_RE = re.compile(r'[%s](.+?)[=%s].*' % (BOLD_FONT[0], NORM_FONT[0]))

124 NBR_DASH_RE = re.compile(r'[%s]' % NBR_DASH[0])

125 INVALID_TARGET_CHARS_RE = re.compile(r'[^-A-Za-z0-9._]')

126 INVALID_START_CHAR_RE = re.compile(r'^([^A-Za-z0-9])')

127 MANIFY_LINESTART_RE = re.compile(r"^(['.])", flags=re.M)

128

129 md_parser = None

130 env_subs = { }

131

132 warning_count = 0

133

134 def main():

135 for mdfn in args.mdfiles:

136 parse_md_file(mdfn)

137

138 if args.test:

139 print("The test was successful.")

140

141

142 def parse_md_file(mdfn):

143 fi = FILENAME_RE.match(mdfn)

144 if not fi:

145 die('Failed to parse a md input file name:', mdfn)

146 fi = argparse.Namespace(**fi.groupdict())

147 fi.want_manpage = not not fi.sect

148 if fi.want_manpage:

149 fi.title = fi.prog + '(' + fi.sect + ') man page'

150 else:

151 fi.title = fi.prog

152

153 if fi.want_manpage:

154 if not env_subs:

155 find_man_substitutions()

156 prog_ver = 'rsync ' + env_subs['VERSION']

157 if fi.prog != 'rsync':

158 prog_ver = fi.prog + ' from ' + prog_ver

159 fi.man_headings = (fi.prog, fi.sect, env_subs['date'], prog_ver, env_subs['prefix'])

160

161 with open(mdfn, 'r', encoding='utf-8') as fh:

162 txt = fh.read()

163

164 use_gfm_parser = '@USE_GFM_PARSER@' in txt

165 if use_gfm_parser:

166 txt = txt.replace('@USE_GFM_PARSER@', '')

167

168 if fi.want_manpage:

169 txt = (txt.replace('@VERSION@', env_subs['VERSION'])

170 .replace('@BINDIR@', env_subs['bindir'])

171 .replace('@LIBDIR@', env_subs['libdir']))

172

173 if use_gfm_parser:

174 if not gfm_parser:

175 die('Input file requires cmarkgfm parser:', mdfn)

176 fi.html_in = gfm_parser(txt)

177 else:

178 fi.html_in = md_parser(txt)

179 txt = None

180

181 TransformHtml(fi)

182

183 if args.test:

184 return

185

186 output_list = [ (fi.name + '.html', fi.html_out) ]

187 if fi.want_manpage:

188 output_list += [ (fi.name, fi.man_out) ]

189 for fn, txt in output_list:

190 if args.dest and args.dest != '.':

191 fn = os.path.join(args.dest, fn)

192 if os.path.lexists(fn):

193 os.unlink(fn)

194 print("Wrote:", fn)

195 with open(fn, 'w', encoding='utf-8') as fh:

196 fh.write(txt)

197

198

199 def find_man_substitutions():

200 srcdir = os.path.dirname(sys.argv[0]) + '/'

201 mtime = 0

202

203 git_dir = srcdir + '.git'

204 if os.path.lexists(git_dir):

205 mtime = int(subprocess.check_output(['git', '--git-dir', git_dir, 'log', '-1', '--format=%at']))

206

207 # Allow "prefix" to be overridden via the environment:

208 env_subs['prefix'] = os.environ.get('RSYNC_OVERRIDE_PREFIX', None)

209

210 if args.test:

211 env_subs['VERSION'] = '1.0.0'

212 env_subs['bindir'] = '/usr/bin'

213 env_subs['libdir'] = '/usr/lib/rsync'

214 else:

215 for fn in (srcdir + 'version.h', 'Makefile'):

216 try:

217 st = os.lstat(fn)

218 except OSError:

219 die('Failed to find', srcdir + fn)

220 if not mtime:

221 mtime = st.st_mtime

222

223 with open(srcdir + 'version.h', 'r', encoding='utf-8') as fh:

224 txt = fh.read()

225 m = QUOTED_RE.search(txt)

226 env_subs['VERSION'] = m.group(1)

227

228 with open('Makefile', 'r', encoding='utf-8') as fh:

229 for line in fh:

230 m = ASSIGNMENT_RE.match(line)

231 if not m:

232 continue

233 var, val = (m.group(1), m.group(2))

234 if var == 'prefix' and env_subs[var] is not None:

235 continue

236 while VAR_REF_RE.search(val):

237 val = VAR_REF_RE.sub(lambda m: env_subs[m.group(1)], val)

238 env_subs[var] = val

239 if var == 'srcdir':

240 break

241

242 env_subs['date'] = time.strftime('%d %b %Y', time.localtime(mtime))

243

244

245 def html_via_commonmark(txt):

246 return commonmark.HtmlRenderer().render(commonmark.Parser().parse(txt))

247

248

249 class TransformHtml(HTMLParser):

250 def __init__(self, fi):

251 HTMLParser.__init__(self, convert_charrefs=True)

252

253 self.fn = fi.fn

254

255 st = self.state = argparse.Namespace(

256 list_state = [ ],

257 p_macro = ".P\n",

258 at_first_tag_in_li = False,

259 at_first_tag_in_dd = False,

260 dt_from = None,

261 in_pre = False,

262 in_code = False,

263 html_out = [ HTML_START.replace('%TITLE%', fi.title) ],

264 man_out = [ ],

265 txt = '',

266 want_manpage = fi.want_manpage,

267 created_hashtags = set(),

268 derived_hashtags = set(),

269 referenced_hashtags = set(),

270 bad_hashtags = set(),

271 prior_target = None,

272 opt_prefix = 'opt',

273 a_txt_start = None,

274 target_suf = '',

275 )

276

277 if st.want_manpage:

278 st.man_out.append(MAN_START % fi.man_headings)

279

280 if '</table>' in fi.html_in:

281 st.html_out[0] = st.html_out[0].replace('</style>', TABLE_STYLE + '</style>')

282

283 self.feed(fi.html_in)

284 fi.html_in = None

285

286 if st.want_manpage:

287 st.html_out.append(MAN_HTML_END % env_subs['date'])

288 st.html_out.append(HTML_END)

289 st.man_out.append(MAN_END)

290

291 fi.html_out = ''.join(st.html_out)

292 st.html_out = None

293

294 fi.man_out = ''.join(st.man_out)

295 st.man_out = None

296

297 for href, txt in st.derived_hashtags:

298 derived = txt2target(txt, href[1:])

299 if derived not in st.created_hashtags:

300 txt = BIN_CHARS_RE.sub('', txt.replace(NBR_DASH[0], '-').replace(NBR_SPACE[0], ' '))

301 warn('Unknown derived hashtag link in', self.fn, 'based on:', (href, txt))

302

303 for bad in st.bad_hashtags:

304 if bad in st.created_hashtags:

305 warn('Missing "#" in hashtag link in', self.fn + ':', bad)

306 else:

307 warn('Unknown non-hashtag link in', self.fn + ':', bad)

308

309 for bad in st.referenced_hashtags - st.created_hashtags:

310 warn('Unknown hashtag link in', self.fn + ':', '#' + bad)

311

312 def handle_starttag(self, tag, attrs_list):

313 st = self.state

314 if args.debug:

315 self.output_debug('START', (tag, attrs_list))

316 if st.at_first_tag_in_li:

317 if st.list_state[-1] == 'dl':

318 st.dt_from = tag

319 if tag == 'p':

320 tag = 'dt'

321 else:

322 st.html_out.append('<dt>')

323 elif tag == 'p':

324 st.at_first_tag_in_dd = True # Kluge to suppress a .P at the start of an li.

325 st.at_first_tag_in_li = False

326 if tag == 'p':

327 if not st.at_first_tag_in_dd:

328 st.man_out.append(st.p_macro)

329 elif tag == 'li':

330 st.at_first_tag_in_li = True

331 lstate = st.list_state[-1]

332 if lstate == 'dl':

333 return

334 if lstate == 'o':

335 st.man_out.append(".IP o\n")

336 else:

337 st.man_out.append(".IP " + str(lstate) + ".\n")

338 st.list_state[-1] += 1

339 elif tag == 'blockquote':

340 st.man_out.append(".RS 4\n")

341 elif tag == 'pre':

342 st.in_pre = True

343 st.man_out.append(st.p_macro + ".nf\n")

344 elif tag == 'code' and not st.in_pre:

345 st.in_code = True

346 st.txt += BOLD_FONT[0]

347 elif tag == 'strong' or tag == 'b':

348 st.txt += BOLD_FONT[0]

349 elif tag == 'em' or tag == 'i':

350 if st.want_manpage:

351 tag = 'u' # Change it into underline to be more like the man page

352 st.txt += UNDR_FONT[0]

353 elif tag == 'ol':

354 start = 1

355 for var, val in attrs_list:

356 if var == 'start':

357 start = int(val) # We only support integers.

358 break

359 if st.list_state:

360 st.man_out.append(".RS\n")

361 if start == 0:

362 tag = 'dl'

363 attrs_list = [ ]

364 st.list_state.append('dl')

365 else:

366 st.list_state.append(start)

367 st.man_out.append(st.p_macro)

368 st.p_macro = ".IP\n"

369 elif tag == 'ul':

370 st.man_out.append(st.p_macro)

371 if st.list_state:

372 st.man_out.append(".RS\n")

373 st.p_macro = ".IP\n"

374 st.list_state.append('o')

375 elif tag == 'hr':

376 st.man_out.append(".l\n")

377 st.html_out.append("<hr />")

378 return

379 elif tag == 'a':

380 st.a_href = None

381 for var, val in attrs_list:

382 if var == 'href':

383 if val in ('#', '#opt', '#daemon-opt'):

384 st.a_href = val

385 elif val.startswith('#'):

386 st.referenced_hashtags.add(val[1:])

387 if val[1:] == st.prior_target:

388 warn('Found link to the current section in', self.fn + ':', val)

389 elif not val.startswith(('https://', 'http://', 'mailto:', 'ftp:', './')):

390 st.bad_hashtags.add(val)

391 st.a_txt_start = len(st.txt)

392 st.html_out.append('<' + tag + ''.join(' ' + var + '="' + htmlify(val) + '"' for var, val in attrs_list) + '>')

393 st.at_first_tag_in_dd = False

394

395

396 def handle_endtag(self, tag):

397 st = self.state

398 if args.debug:

399 self.output_debug('END', (tag,))

400 if tag in CONSUMES_TXT or st.dt_from == tag:

401 txt = st.txt.strip()

402 st.txt = ''

403 else:

404 txt = None

405 add_to_txt = None

406 if tag == 'h1':

407 tgt = txt

408 target_suf = ''

409 if tgt.startswith('NEWS for '):

410 m = VERSION_RE.search(tgt)

411 if m:

412 tgt = m.group(1)

413 st.target_suf = '-' + tgt

414 self.add_target(tgt)

415 elif tag == 'h2':

416 st.man_out.append(st.p_macro + '.SH "' + manify(txt) + '"\n')

417 self.add_target(txt, st.target_suf)

418 st.opt_prefix = 'daemon-opt' if txt == 'DAEMON OPTIONS' else 'opt'

419 elif tag == 'h3':

420 st.man_out.append(st.p_macro + '.SS "' + manify(txt) + '"\n')

421 self.add_target(txt, st.target_suf)

422 elif tag == 'p':

423 if st.dt_from == 'p':

424 tag = 'dt'

425 st.man_out.append('.IP "' + manify(txt) + '"\n')

426 if txt.startswith(BOLD_FONT[0]):

427 self.add_target(txt)

428 st.dt_from = None

429 elif txt != '':

430 st.man_out.append(manify(txt) + "\n")

431 elif tag == 'li':

432 if st.list_state[-1] == 'dl':

433 if st.at_first_tag_in_li:

434 die("Invalid 0. -> td translation")

435 tag = 'dd'

436 if txt != '':

437 st.man_out.append(manify(txt) + "\n")

438 st.at_first_tag_in_li = False

439 elif tag == 'blockquote':

440 st.man_out.append(".RE\n")

441 elif tag == 'pre':

442 st.in_pre = False

443 st.man_out.append(manify(txt) + "\n.fi\n")

444 elif (tag == 'code' and not st.in_pre):

445 st.in_code = False

446 add_to_txt = NORM_FONT[0]

447 elif tag == 'strong' or tag == 'b':

448 add_to_txt = NORM_FONT[0]

449 elif tag == 'em' or tag == 'i':

450 if st.want_manpage:

451 tag = 'u' # Change it into underline to be more like the man page

452 add_to_txt = NORM_FONT[0]

453 elif tag == 'ol' or tag == 'ul':

454 if st.list_state.pop() == 'dl':

455 tag = 'dl'

456 if st.list_state:

457 st.man_out.append(".RE\n")

458 else:

459 st.p_macro = ".P\n"

460 st.at_first_tag_in_dd = False

461 elif tag == 'hr':

462 return

463 elif tag == 'a':

464 if st.a_href:

465 atxt = st.txt[st.a_txt_start:]

466 find = 'href="' + st.a_href + '"'

467 for j in range(len(st.html_out)-1, 0, -1):

468 if find in st.html_out[j]:

469 derived = txt2target(atxt, st.a_href[1:])

470 if derived == st.prior_target:

471 warn('Found link to the current section in', self.fn + ':', derived)

472 st.derived_hashtags.add((st.a_href, atxt))

473 st.html_out[j] = st.html_out[j].replace(find, 'href="#' + derived + '"')

474 break

475 else:

476 die('INTERNAL ERROR: failed to find href in html data:', find)

477 st.html_out.append('</' + tag + '>')

478 if add_to_txt:

479 if txt is None:

480 st.txt += add_to_txt

481 else:

482 txt += add_to_txt

483 if st.dt_from == tag:

484 st.man_out.append('.IP "' + manify(txt) + '"\n')

485 st.html_out.append('</dt><dd>')

486 st.at_first_tag_in_dd = True

487 st.dt_from = None

488 elif tag == 'dt':

489 st.html_out.append('<dd>')

490 st.at_first_tag_in_dd = True

491

492

493 def handle_data(self, txt):

494 st = self.state

495 if '](' in txt:

496 warn('Malformed link in', self.fn + ':', txt)

497 if args.debug:

498 self.output_debug('DATA', (txt,))

499 if st.in_pre:

500 html = htmlify(txt)

501 else:

502 txt = SPACE_DOUBLE_DASH_RE.sub(NBR_SPACE[0] + r'--\1', txt).replace('--', NBR_DASH[0]*2)

503 txt = NON_SPACE_SINGLE_DASH_RE.sub(r'\1' + NBR_DASH[0], txt)

504 html = htmlify(txt)

505 if st.in_code:

506 txt = WHITESPACE_RE.sub(NBR_SPACE[0], txt)

507 html = html.replace(NBR_DASH[0], '-').replace(NBR_SPACE[0], ' ') # <code> is non-breaking in CSS

508 st.html_out.append(html.replace(NBR_SPACE[0], ' ').replace(NBR_DASH[0], '-⁠'))

509 st.txt += txt

510

511

512 def add_target(self, txt, suf=None):

513 st = self.state

514 txt = txt2target(txt, st.opt_prefix)

515 if txt:

516 if suf:

517 txt += suf

518 if txt in st.created_hashtags:

519 for j in range(2, 1000):

520 chk = txt + '-' + str(j)

521 if chk not in st.created_hashtags:

522 print('Made link target unique:', chk)

523 txt = chk

524 break

525 st.html_out.append('<a id="' + txt + '" href="#' + txt + '" class="tgt"></a>')

526 st.created_hashtags.add(txt)

527 st.prior_target = txt

528

529

530 def output_debug(self, event, extra):

531 import pprint

532 st = self.state

533 if args.debug < 2:

534 st = argparse.Namespace(**vars(st))

535 if len(st.html_out) > 2:

536 st.html_out = ['...'] + st.html_out[-2:]

537 if len(st.man_out) > 2:

538 st.man_out = ['...'] + st.man_out[-2:]

539 print(event, extra)

540 pprint.PrettyPrinter(indent=2).pprint(vars(st))

541

542

543 def txt2target(txt, opt_prefix):

544 txt = CODE_BLOCK_RE.sub(r'\1', txt.strip().rstrip(':'))

545 txt = NBR_DASH_RE.sub('-', txt)

546 txt = BIN_CHARS_RE.sub('', txt)

547 txt = INVALID_TARGET_CHARS_RE.sub('_', txt)

548 if opt_prefix and txt.startswith('-'):

549 txt = opt_prefix + txt

550 else:

551 txt = INVALID_START_CHAR_RE.sub(r't\1', txt)

552 return txt

553

554

555 def manify(txt):

556 return MANIFY_LINESTART_RE.sub(r'\&\1', txt.replace('\\', '\\\\')

557 .replace(NBR_SPACE[0], NBR_SPACE[1])

558 .replace(NBR_DASH[0], NBR_DASH[1])

559 .replace(NORM_FONT[0], NORM_FONT[1])

560 .replace(BOLD_FONT[0], BOLD_FONT[1])

561 .replace(UNDR_FONT[0], UNDR_FONT[1]))

562

563

564 def htmlify(txt):

565 return txt.replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"')

566

567

568 def warn(*msg):

569 print(*msg, file=sys.stderr)

570 global warning_count

571 warning_count += 1

572

573

574 def die(*msg):

575 warn(*msg)

576 sys.exit(1)

577

578

579 if __name__ == '__main__':

580 parser = argparse.ArgumentParser(description="Output html and (optionally) nroff for markdown pages.", add_help=False)

581 parser.add_argument('--test', action='store_true', help="Just test the parsing without outputting any files.")

582 parser.add_argument('--dest', metavar='DIR', help="Put files into DIR instead of the current directory.")

583 parser.add_argument('--debug', '-D', action='count', default=0, help='Output copious info on the html parsing. Repeat for even more.')

584 parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.")

585 parser.add_argument("mdfiles", nargs='+', help="The source .md files to convert.")

586 args = parser.parse_args()

587

588 try:

589 import cmarkgfm

590 md_parser = cmarkgfm.markdown_to_html

591 gfm_parser = cmarkgfm.github_flavored_markdown_to_html

592 except:

593 try:

594 import commonmark

595 md_parser = html_via_commonmark

596 except:

597 die("Failed to find cmarkgfm or commonmark for python3.")

598 gfm_parser = None

599

600 main()

601 if warning_count:

602 sys.exit(1)