Coverage for gws-app/gws/lib/vendor/jump/compiler.py: 22%

1import os

4# public API

6def do(cmd, engine, options, text, path):

7 cc = Compiler(engine, options)

9 if text is None:

10 loc = Location(pos=0, path=__file__, line_num=1)

11 text, path = cc.load(None, path or cc.options.path, loc)

12 else:

13 path = path or cc.options.path or '<string>'

15 cc.buf.paste(text, path, cc.buf.pos)

17 node = cc.parse()

18 if cmd == 'parse':

19 return node

21 python = cc.translate(node)

22 if cmd == 'translate':

23 return python

25 fn = cc.compile(python)

26 if cmd == 'compile':

27 return fn

30##

32class T:

33 CONST = 'CONST'

34 EOF = 'EOF'

35 INVALID = 'INVALID'

36 NAME = 'NAME'

37 NEWLINE = 'NEWLINE'

38 NUMBER = 'NUMBER'

39 STRING = 'STRING'

42class C:

43 NL = '\n'

44 WS = ' \r\t\x0b\f'

45 WSNL = ' \r\t\x0b\f\n'

47 IDENTIFIER_START = set('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_')

48 IDENTIFIER_CONTINUE = set('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_0123456789')

50 DIGIT_DEC = set('0123456789')

51 DIGIT_HEX = set('0123456789ABCDEFabcdef')

52 DIGIT_OCT = set('01234567')

53 DIGIT_BIN = set('01')

55 QUOTES = set("\'\"")

57 STRING_ESCAPES = {

58 "'": "'",

59 '"': '"',

60 '0': '\0',

61 '/': '/',

62 '\\': '\\',

63 'b': '\b',

64 'f': '\f',

65 'n': '\n',

66 'r': '\r',

67 't': '\t',

68 }

70 CONST = {

71 'True': True,

72 'true': True,

73 'False': False,

74 'false': False,

75 'None': None,

76 'null': None,

77 }

79 ADD_OPS = {'+', '-'}

80 MUL_OPS = {'*', '/', '//', '%'}

81 CMP_OPS = {'==', '!=', '<=', '<', '>=', '>'}

82 POWER_OP = {'**'}

83 COMPARE_OPS = CMP_OPS | {'in', 'not in'}

85 PUNCT = set('=.:,[]{}()|&?!') | POWER_OP | ADD_OPS | MUL_OPS | CMP_OPS

87 KEYWORD_OPS = {'and', 'or', 'not', 'in', 'is'}

89 KEYWORDS = {

90 'and', 'as', 'assert', 'async', 'await', 'break', 'class', 'continue', 'def', 'del', 'elif',

91 'else', 'except', 'finally', 'for', 'from', 'global', 'if', 'import', 'in', 'is', 'lambda',

92 'nonlocal', 'not', 'or', 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield'

93 }

95 LOOP_KEYWORDS = {'index', 'length'}

97 FORMAT_START_SYMBOLS = ':!'

99 SAFE_FILTER = {'safe', 'none'}

100

101 ELSE_SYMBOL = 'else'

102 ELIF_SYMBOL = 'elif'

103 END_SYMBOL = 'end'

104

105 AUX_COMMANDS = {END_SYMBOL, ELSE_SYMBOL, ELIF_SYMBOL}

106

107 DEF_COMMANDS = {'def', 'box', 'mdef', 'mbox'}

108

109 TOK_TYPE = 0

110 TOK_POS = 1

111 TOK_NEXTPOS = 2

112 TOK_SPACE_BEFORE = 3

113 TOK_SPACE_AFTER = 4

114 TOK_VALUE = 5

115

116 DEFAULT_OPTIONS = dict(

117 name='_RENDER_',

118 filter=None,

119 loader=None,

120 strip=False,

121 escapes='@@ @ {{ { }} }',

122 comment_symbol='@#',

123 command_symbol='@',

124 inline_open_symbol='{@',

125 inline_close_symbol='}',

126 inline_start_whitespace=False,

127 echo_open_symbol='{',

128 echo_close_symbol='}',

129 echo_start_whitespace=False,

130 )

131

132 PY_INDENT = ' ' * 4

133 PY_BEGIN = 1

134 PY_END = -1

135

136 PY_MARKER = '##:'

137

138 PY_TEMPLATE = """\

139def $name$(_ENGINE, _ARGS, _ERRORHANDLER=None):

140 _PATHS = $paths$

141 _ENV = _ENGINE.environment(_PATHS, _ARGS, _ERRORHANDLER)

142 ARGS = _ENV.ARGS

143 print =_ENV.print

144 try:

145 _ENV.pushbuf()

146 $code$

147 return _ENV.popbuf()

148 except Exception as _0:

149 _ENV.error(_0, $loc$)

150"""

151

152

153class Data:

154 def __init__(self, **kwargs):

155 vars(self).update(kwargs)

156

157 def __getattr__(self, item):

158 return None

159

160

161class Location(Data):

162 pos = 0

163 path = ''

164 path_index = 0

165 line_start_pos = 0

166 line_num = 0

167 column = 0

168

169 def __repr__(self):

170 return '[' + repr(self.path) + ',' + repr(self.line_num) + ']'

171

172

173class Node:

174 class And:

175 def __init__(self, subject, pairs):

176 self.subject = subject

177 self.pairs = pairs

178

179 def emit(self, tr: 'Translator'):

180 return tr.emit_left_binary_op(self)

181

182 class Argument:

183 def __init__(self, name, star, expr):

184 self.name = name

185 self.star = star

186 self.expr = expr

187

188 def emit(self, tr: 'Translator'):

189 code = tr.emit(self.expr)

190 if self.name:

191 return self.name + '=' + code

192 if self.star == 1:

193 return '*' + code

194 if self.star == 2:

195 return '**' + code

196 return code

197

198 class Attr:

199 def __init__(self, subject, name):

200 self.subject = subject

201 self.name = name

202

203 def emit(self, tr: 'Translator'):

204 names = []

205 node = self

206

207 while isinstance(node, Node.Attr):

208 names.append(node.name)

209 node = node.subject

210

211 head = tr.emit(node)

212 if len(names) == 1:

213 return f'_ENV.attr({head}, {names[0]!r})'

214

215 names = list(reversed(names))

216 return f'_ENV.attrs({head}, {names!r})'

217

218 class Call:

219 def __init__(self, function, args):

220 self.function = function

221 self.args = args

222

223 def emit(self, tr: 'Translator'):

224 args = _comma(tr.emit(a) for a in self.args)

225 fn = tr.emit(self.function)

226 if not isinstance(self.function, (Node.Name, Node.Attr, Node.Index)):

227 fn = _parens(fn)

228 return f'{fn}({args})'

229

230 class Comparison:

231 def __init__(self, subject, pairs):

232 self.subject = subject

233 self.pairs = pairs

234

235 def emit(self, tr: 'Translator'):

236 return tr.emit_comp_binary_op(self)

237

238 class Const:

239 def __init__(self, value):

240 self.value = value

241

242 def emit(self, tr: 'Translator'):

243 return repr(self.value)

244

245 class Dict:

246 def __init__(self, items):

247 self.items = items

248

249 def emit(self, tr: 'Translator'):

250 return '{' + _comma(tr.emit(k) + ':' + tr.emit(v) for k, v in self.items) + '}'

251

252 class IfExpression:

253 def __init__(self, cond, yes, no):

254 self.cond = cond

255 self.yes = yes

256 self.no = no

257

258 def emit(self, tr: 'Translator'):

259 return (

260 _parens(tr.emit(self.yes))

261 + ' if '

262 + _parens(tr.emit(self.cond))

263 + ' else '

264 + _parens(tr.emit(self.no))

265 )

266

267 class Index:

268 def __init__(self, subject, index):

269 self.subject = subject

270 self.index = index

271

272 def emit(self, tr: 'Translator'):

273 subj = tr.emit(self.subject)

274 index = ':'.join(tr.emit(a) if a else '' for a in self.index)

275 return f'{subj}[{index}]'

276

277 class List:

278 def __init__(self, items):

279 self.items = items

280

281 def emit(self, tr: 'Translator'):

282 return '[' + _comma(tr.emit(v) for v in self.items) + ']'

283

284 class Name:

285 def __init__(self, ident):

286 self.ident = ident

287

288 def emit(self, tr: 'Translator'):

289 s = self.ident

290 if s in tr.locals:

291 return s

292 return f'_ENV.get({s!r})'

293

294 class Not:

295 def __init__(self, subject, ops):

296 self.subject = subject

297 self.ops = ops

298

299 def emit(self, tr: 'Translator'):

300 return tr.emit_unary_op(self)

301

302 class Number:

303 def __init__(self, value):

304 self.value = value

305

306 def emit(self, tr: 'Translator'):

307 return repr(self.value)

308

309 class Or:

310 def __init__(self, subject, pairs):

311 self.subject = subject

312 self.pairs = pairs

313

314 def emit(self, tr: 'Translator'):

315 return tr.emit_left_binary_op(self)

316

317 class Param:

318 def __init__(self, name, star, expr):

319 self.name = name

320 self.star = star

321 self.expr = expr

322

323 def emit(self, tr: 'Translator'):

324 n = self.name

325 if self.star == 1:

326 return '*' + n

327 if self.star == 2:

328 return '**' + n

329 if self.expr:

330 return n + '=' + tr.emit(self.expr)

331 return n

332

333 class PipeList:

334 def __init__(self, subject, pipes):

335 self.subject = subject

336 self.pipes = pipes

337

338 def emit(self, tr: 'Translator'):

339 code = tr.emit(self.subject)

340

341 for p in self.pipes:

342 if isinstance(p, Node.Name):

343 # s | f => f(s)

344 if p.ident in C.SAFE_FILTER:

345 continue

346 fn = tr.emit(p)

347 args = [code]

348 elif isinstance(p, Node.Call):

349 # s | f(a, b) => f(s, a, b)

350 fn = tr.emit(p.function)

351 args = [code] + [tr.emit(a) for a in p.args]

352 else:

353 # s | whatever => (whatever)(s)

354 fn = _parens(tr.emit(p))

355 args = [code]

356

357 code = f'{fn}({_comma(args)})'

358

359 return code

360

361 class Power:

362 def __init__(self, subject, pairs):

363 self.subject = subject

364 self.pairs = pairs

365

366 def emit(self, tr: 'Translator'):

367 return tr.emit_right_binary_op(self)

368

369 class Product:

370 def __init__(self, subject, pairs):

371 self.subject = subject

372 self.pairs = pairs

373

374 def emit(self, tr: 'Translator'):

375 return tr.emit_left_binary_op(self)

376

377 class Sum:

378 def __init__(self, subject, pairs):

379 self.subject = subject

380 self.pairs = pairs

381

382 def emit(self, tr: 'Translator'):

383 return tr.emit_left_binary_op(self)

384

385 class Unary:

386 def __init__(self, subject, ops):

387 self.subject = subject

388 self.ops = ops

389

390 def emit(self, tr: 'Translator'):

391 return tr.emit_unary_op(self)

392

393 class String:

394 def __init__(self, value):

395 self.value = value

396

397 def emit(self, tr: 'Translator'):

398 return repr(self.value)

399

400 ##

401

402 class Template:

403 def __init__(self):

404 self.children = []

405

406 def emit(self, tr: 'Translator'):

407 return [tr.emit(c) for c in self.children]

408

409 class Location:

410 def __init__(self, loc):

411 self.loc = loc

412

413 def emit(self, tr: 'Translator'):

414 return self

415

416 class Echo:

417 def __init__(self, start_pos):

418 self.start_pos = start_pos

419 self.expr = None

420 self.format = None

421 self.filter = None

422

423 def parse(self, tp: 'TemplateParser'):

424 tp.expr.paren_stack.append('{')

425 expr = tp.expr.expect_expression()

426 tp.expr.paren_stack.pop()

427

428 tp.buf.skip_ws(with_nl=True)

429

430 fmt = None

431 if tp.buf.char() in C.FORMAT_START_SYMBOLS:

432 fmt = tp.lex.string_to(tp.cc.options.echo_close_symbol, self.start_pos)

433

434 if not tp.buf.at_string(tp.cc.options.echo_close_symbol):

435 raise CompileError('unterminated echo', tp.buf.loc(), tp.buf.loc(self.start_pos))

436

437 tp.buf.next(len(tp.cc.options.echo_close_symbol))

438

439 return self.parse_with_expression(tp, expr, fmt)

440

441 def parse_with_expression(self, tp, expr, fmt):

442 self.expr = expr

443 self.format = fmt

444

445 has_safe_filter = False

446 if isinstance(self.expr, Node.PipeList):

447 has_safe_filter = any(isinstance(p, Node.Name) and p.ident in C.SAFE_FILTER for p in self.expr.pipes)

448 self.filter = None if has_safe_filter else tp.cc.options.filter

449

450 tp.add_child(self)

451

452 def emit(self, tr: 'Translator'):

453 code = tr.emit(self.expr)

454

455 if self.format:

456 fn = tr.emit(Node.Name('format'))

457 code = f'{fn}({code},{self.format!r})'

458

459 if self.filter:

460 fn = tr.emit(Node.Name(self.filter))

461 code = f'{fn}({code})'

462

463 return tr.emit_echo(code)

464

465 class Text:

466 def __init__(self, text):

467 self.text = text

468

469 def emit(self, tr: 'Translator'):

470 return self

471

472 class Command:

473 def __init__(self, cmd, start_pos):

474 self.cmd = cmd

475 self.start_pos = start_pos

476

477 def parse(self, tp: 'TemplateParser', is_inline):

478 pass

479

480 def parse_elif(self, tp: 'TemplateParser', is_inline):

481 raise ValueError()

482

483 def parse_else(self, tp: 'TemplateParser', is_inline):

484 raise ValueError()

485

486 def parse_end(self, tp: 'TemplateParser', is_inline):

487 raise ValueError()

488

489 class BlockCommand(Command):

490 def __init__(self, cmd, start_pos):

491 super().__init__(cmd, start_pos)

492 self.blocks = []

493 self.children = []

494 self.has_else = False

495

496 def parse_end(self, tp, is_inline):

497 self.blocks.append(self.children)

498 tp.end_command(self, is_inline)

499

500 class DefineFunction(BlockCommand):

501 def __init__(self, cmd, start_pos):

502 super().__init__(cmd, start_pos)

503 self.name = None

504 self.params = None

505 self.expr = None

506 self.from_engine = False

507

508 def parse(self, tp, is_inline):

509 self.name = tp.expr.expect_name().ident

510 self.params = tp.expr.top_level_param_list()

511

512 tp.static_function_bindings[self.name] = [self.cmd, None]

513

514 tok = tp.lex.token(ignore_nl=is_inline)

515 if tok[C.TOK_TYPE] == '=':

516 self.expr = tp.expr.expect_expression()

517 else:

518 tp.lex.back(tok)

519

520 tp.expect_end_of_command(is_inline)

521 if self.expr:

522 tp.add_child(self)

523 else:

524 tp.begin_command(self)

525

526 def emit(self, tr: 'Translator'):

527 tr.locals.add(self.name)

528 tr.enter_frame()

529 tr.locals.update(p.name for p in self.params)

530 params = _comma(tr.emit(p) for p in self.params)

531

532 if self.blocks:

533 # 'big' functions are wrapped in push/popbuf to allow explicit returns

534 #

535 # def userfunc(args):

536 # nul = object()

537 # def fun()

538 # [@return ...]

539 # return nul

540 #

541 # _ENV.pushbuf

542 # res = fun()

543 # buf = _ENV.popbuf

544 # return buf if res == nul else res

545 #

546

547 nul = tr.var()

548 fun = tr.var()

549 res = tr.var()

550 buf = tr.var()

551

552 code = [

553 f'def {self.name}({params}):',

554 C.PY_BEGIN,

555 f'{nul} = object()',

556 f'def {fun}():',

557 C.PY_BEGIN,

558 [tr.emit(c) for c in self.blocks[0]],

559 f'return {nul}',

560 C.PY_END,

561 f'_ENV.pushbuf()',

562 f'{res} = {fun}()',

563 f'{buf} = _ENV.popbuf()',

564 f'return {buf} if {res} == {nul} else {res}',

565 C.PY_END,

566 ]

567

568 else:

569 # small function

570

571 res = tr.var()

572 code = [

573 f'def {self.name}({params}):',

574 C.PY_BEGIN,

575 tr.emit_assign(res, tr.emit(self.expr)),

576 f'return {res}',

577 C.PY_END,

578 ]

579

580 tr.leave_frame()

581 return code

582

583 class CallFunctionAsCommand(BlockCommand):

584 def __init__(self, cmd, start_pos):

585 super().__init__(cmd, start_pos)

586 self.args = None

587 self.static_emit_name = ''

588 self.block_required = False

589

590 def parse(self, tp, is_inline):

591 kind, engine_name = tp.static_function_bindings.get(self.cmd)

592 self.static_emit_name = f'_ENGINE.{engine_name}' if engine_name else self.cmd

593

594 if kind == 'box':

595 self.block_required = True

596 self.args = tp.expr.top_level_arg_list()

597 tp.expect_end_of_command(is_inline)

598 tp.begin_command(self)

599 return

600

601 if kind == 'mbox':

602 self.args = tp.expr.top_level_arg_list()

603 tp.expect_end_of_command(is_inline)

604 text = tp.quoted_content(self.cmd, is_inline)

605 self.args.insert(0, Node.String(text))

606 tp.add_child(self)

607 return

608

609 if kind == 'def':

610 self.args = tp.expr.top_level_arg_list()

611 tp.expect_end_of_command(is_inline)

612 tp.add_child(self)

613 return

614

615 if kind == 'mdef':

616 tail = tp.command_tail(is_inline)

617 self.args = [Node.String(tail)]

618 tp.add_child(self)

619 return

620

621 def emit(self, tr: 'Translator'):

622 fn = self.static_emit_name

623

624 if self.block_required:

625 buf = tr.var()

626 args = _comma([buf] + [tr.emit(a) for a in self.args])

627 return [

628 '_ENV.pushbuf()',

629 [tr.emit(c) for c in self.blocks[0]],

630 f'{buf} = _ENV.popbuf()',

631 tr.emit_echo(f'{fn}({args})')

632 ]

633 else:

634 args = _comma(tr.emit(a) for a in self.args)

635 return tr.emit_echo(f'{fn}({args})')

636

637 ##

638

639 class CommandBreakContinue(Command):

640 def parse(self, tp, is_inline):

641 start_pos = tp.buf.pos

642 tp.expect_end_of_command(is_inline)

643 if tp.in_loop_context():

644 return tp.add_child(self)

645 raise CompileError(f"unexpected '{self.cmd}'", tp.buf.loc(start_pos))

646

647 def emit(self, tr: 'Translator'):

648 return self.cmd

649

650 class CommandCode(Command):

651 def __init__(self, cmd, start_pos):

652 super().__init__(cmd, start_pos)

653 self.lines = []

654

655 def parse(self, tp, is_inline):

656 start_pos = tp.buf.pos

657 label = tp.command_label(is_inline)

658 tp.expect_end_of_command(is_inline)

659 text = tp.quoted_content(label, is_inline)

660 self.lines = _dedent(text.split(C.NL))

661

662 check = self.lines

663 check = ['def _():'] + _indent(check)

664

665 try:

666 compile(C.NL.join(check), '', 'exec')

667 except SyntaxError as exc:

668 loc = tp.buf.loc(start_pos)

669 loc.line_num += exc.lineno - 1

670 raise CompileError(exc.msg, loc)

671

672 tp.add_child(self)

673

674 def emit(self, tr: 'Translator'):

675 nul = tr.var()

676 res = tr.var()

677 fun = tr.var()

678 return [

679 f'def {fun}():',

680 C.PY_BEGIN,

681 self.lines,

682 f'return {nul}',

683 C.PY_END,

684 f'{nul} = object()',

685 tr.emit_assign(res, f'{fun}()'),

686 f'if {res} != {nul}:',

687 C.PY_BEGIN,

688 f'return {res}',

689 C.PY_END,

690 ]

691

692 class CommandDo(Command):

693 def __init__(self, cmd, start_pos):

694 super().__init__(cmd, start_pos)

695 self.exprs = []

696

697 def parse(self, tp, is_inline):

698 self.exprs = tp.expr.expression_list()

699 tp.expect_end_of_command(is_inline)

700 tp.add_child(self)

701

702 def emit(self, tr: 'Translator'):

703 return tr.emit_try(f'{_comma(tr.emit(a) for a in self.exprs)}')

704

705 class CommandFor(BlockCommand):

706 def __init__(self, cmd, start_pos):

707 super().__init__(cmd, start_pos)

708 self.subject = None

709 self.names = []

710 self.extras = {}

711

712 def parse(self, tp, is_inline):

713 self.extras = {w: None for w in C.LOOP_KEYWORDS}

714

715 if self.cmd == 'for':

716 self.names = [n.ident for n in tp.expr.name_list()]

717 tp.expr.expect_token('in')

718 self.subject = tp.expr.expect_expression()

719 while True:

720 tok = tp.lex.token(ignore_nl=is_inline)

721 if tok[C.TOK_TYPE] == T.NAME and tok[C.TOK_VALUE] in C.LOOP_KEYWORDS:

722 self.extras[tok[C.TOK_VALUE]] = tp.expr.expect_name().ident

723 else:

724 tp.lex.back(tok)

725 break

726

727 if self.cmd == 'each':

728 self.subject = tp.expr.expect_expression()

729 tp.expr.expect_token('as')

730 self.names = [n.ident for n in tp.expr.name_list()]

731 for _ in C.LOOP_KEYWORDS:

732 if len(self.names) > 2 and self.names[-2] in C.LOOP_KEYWORDS:

733 self.extras[self.names[-2]] = self.names[-1]

734 self.names = self.names[:-2]

735

736 tp.expect_end_of_command(is_inline)

737 tp.begin_command(self)

738

739 def parse_else(self, tp, is_inline):

740 if self.has_else:

741 raise ValueError()

742 self.has_else = True

743 self.blocks.append(self.children)

744 self.children = []

745 tp.expect_end_of_command(is_inline)

746

747 def emit(self, tr: 'Translator'):

748 it = tr.var()

749 expr = f'_ENV.iter({tr.emit(self.subject)}, {len(self.names)})'

750 code = tr.emit_assign(it, expr, fallback='""')

751

752 tr.locals.update(self.names)

753

754 v = self.extras.get('length')

755 if v:

756 tr.locals.add(v)

757 code.append(tr.emit_try(

758 f'{v} = len({it})',

759 f'{v} = 0',

760 ))

761

762 cnt = tr.var()

763 v = self.extras.get('index')

764 if v:

765 tr.locals.add(v)

766 cnt = v

767

768 code.extend([

769 f'{cnt} = 0',

770 f'for {_comma(self.names)} in {it}:',

771 C.PY_BEGIN,

772 f'{cnt} += 1',

773 [tr.emit(c) for c in self.blocks[0]],

774 C.PY_END

775 ])

776

777 if len(self.blocks) > 1:

778 code.extend([

779 f'if {cnt} == 0:',

780 C.PY_BEGIN,

781 [tr.emit(c) for c in self.blocks[1]],

782 C.PY_END

783 ])

784

785 return code

786

787 class CommandIf(BlockCommand):

788 def __init__(self, cmd, start_pos):

789 super().__init__(cmd, start_pos)

790 self.conds = []

791

792 def parse(self, tp, is_inline):

793 self.conds.append(tp.expr.expect_expression())

794 tp.expect_end_of_command(is_inline)

795 tp.begin_command(self)

796

797 def parse_elif(self, tp, is_inline):

798 self.blocks.append(self.children)

799 self.children = []

800 self.conds.append(tp.expr.expect_expression())

801 tp.expect_end_of_command(is_inline)

802

803 def parse_else(self, tp, is_inline):

804 if self.has_else:

805 raise ValueError()

806 self.has_else = True

807 self.blocks.append(self.children)

808 self.children = []

809 self.conds.append(Node.Const(True))

810 tp.expect_end_of_command(is_inline)

811

812 def emit(self, tr: 'Translator'):

813 """

814

815 if A

816 aaa

817 elif B

818 bbb

819 else

820 ccc

821

822 is compiled to

823

824 done = False

825 if not done:

826 x = A

827 if x:

828 done = True

829 aaa

830 if not done:

831 x = B

832 if x:

833 done = True

834 bbb

835 if not done:

836 x = True

837 if x:

838 done = True

839 ccc

840

841

842 """

843

844 done = tr.var()

845 code = [f'{done} = False']

846 for cond, block in zip(self.conds, self.blocks):

847 cnd = tr.var()

848 code.extend([

849 f'if not {done}:',

850 C.PY_BEGIN,

851 tr.emit_assign(cnd, tr.emit(cond)),

852 f'if {cnd}:',

853 C.PY_BEGIN,

854 f'{done} = True',

855 [tr.emit(c) for c in block],

856 C.PY_END,

857 C.PY_END,

858 ])

859

860 return code

861

862 class CommandImport(Command):

863 def __init__(self, cmd, start_pos):

864 super().__init__(cmd, start_pos)

865 self.names = []

866

867 def parse(self, tp, is_inline):

868 self.names = tp.expect_dotted_names()

869 tp.expect_end_of_command(is_inline)

870 tp.add_child(self)

871

872 def emit(self, tr: 'Translator'):

873 tr.locals.add(self.names[0])

874 return tr.emit_try('import ' + '.'.join(self.names))

875

876 class CommandInclude(Command):

877 def parse(self, tp, is_inline):

878 loc = tp.buf.loc()

879 tail = _unquote(tp.command_tail(is_inline))

880 text, path = tp.cc.load(loc.path, tail, loc)

881 tp.buf.paste(text, path, tp.buf.pos)

882 tp.reset()

883

884 class CommandLet(BlockCommand):

885 def __init__(self, cmd, start_pos):

886 super().__init__(cmd, start_pos)

887 self.names = []

888 self.exprs = None

889

890 def parse(self, tp, is_inline):

891 self.names = [n.ident for n in tp.expr.name_list()]

892

893 tok = tp.lex.token(ignore_nl=is_inline)

894 if tok[C.TOK_TYPE] == '=':

895 self.exprs = tp.expr.expression_list()

896 else:

897 tp.lex.back(tok)

898

899 tp.expect_end_of_command(is_inline)

900 if self.exprs:

901 tp.add_child(self)

902 else:

903 tp.begin_command(self)

904

905 def emit(self, tr: 'Translator'):

906 if self.exprs:

907 code = tr.emit_try(

908 f'{_comma(self.names)} = {_comma(tr.emit(a) for a in self.exprs)}',

909 f'{"=".join(self.names)} = None',

910 )

911 else:

912 code = [

913 f'_ENV.pushbuf()',

914 [tr.emit(c) for c in self.blocks[0]],

915 f'{_comma(self.names)} = _ENV.popbuf()',

916 ]

917

918 tr.locals.update(self.names)

919 return code

920

921 class CommandOption(Command):

922 def parse(self, tp, is_inline):

923 name = tp.expr.expect_name().ident

924 tok = tp.lex.token(ignore_nl=is_inline)

925 if tok[C.TOK_TYPE] != '=':

926 tp.lex.back(tok)

927 pos = tp.buf.pos

928 val = tp.expr.expect_expression()

929 tp.expect_end_of_command(is_inline)

930 if not isinstance(val, (Node.String, Node.Number, Node.Const)):

931 raise CompileError(f'invalid option value', tp.buf.loc(pos))

932 setattr(tp.cc.options, name, val.value)

933 tp.reset()

934

935 class CommandPrint(Command):

936 def __init__(self, cmd, start_pos):

937 super().__init__(cmd, start_pos)

938 self.exprs = []

939

940 def parse(self, tp, is_inline):

941 args = tp.expr.expression_list()

942 tp.expect_end_of_command(is_inline)

943

944 start = True

945 for arg in args:

946 if not start:

947 tp.add_raw_text(' ')

948 start = False

949 Node.Echo(self.start_pos).parse_with_expression(tp, arg, None)

950 if not is_inline:

951 tp.add_raw_text('\n')

952

953 class CommandComment(Command):

954 def parse(self, tp, is_inline):

955 label = tp.command_label(is_inline)

956 tp.expect_end_of_command(is_inline)

957 text = tp.quoted_content(label, is_inline)

958

959 class CommandQuote(Command):

960 def parse(self, tp, is_inline):

961 label = tp.command_label(is_inline)

962 tp.expect_end_of_command(is_inline)

963 text = tp.quoted_content(label, is_inline)

964 tp.add_raw_text(text)

965

966 class CommandReturn(Command):

967 def __init__(self, cmd, start_pos):

968 super().__init__(cmd, start_pos)

969 self.expr = None

970

971 def parse(self, tp, is_inline):

972 self.expr = tp.expr.expression()

973 tp.expect_end_of_command(is_inline)

974 return tp.add_child(self)

975

976 def emit(self, tr: 'Translator'):

977 code = 'return'

978 if self.expr:

979 code += ' ' + tr.emit(self.expr)

980 return code

981

982 class CommandWith(BlockCommand):

983 def __init__(self, cmd, start_pos):

984 super().__init__(cmd, start_pos)

985 self.subject = None

986 self.alias = None

987 self.inverted = False

988

989 def parse(self, tp, is_inline):

990 self.inverted = self.cmd == 'without'

991 self.subject = tp.expr.expect_expression()

992

993 tok = tp.lex.token(ignore_nl=is_inline)

994 if tok[C.TOK_TYPE] == 'as':

995 self.alias = tp.expr.expect_name().ident

996 else:

997 tp.lex.back(tok)

998

999 tp.expect_end_of_command(is_inline)

1000 tp.begin_command(self)

1001

1002 def parse_else(self, tp, is_inline):

1003 if self.has_else:

1004 raise ValueError()

1005 self.has_else = True

1006 self.blocks.append(self.children)

1007 self.children = []

1008 tp.expect_end_of_command(is_inline)

1009

1010 def emit(self, tr: 'Translator'):

1011 if self.alias:

1012 tr.locals.add(self.alias)

1013 var = self.alias

1014 else:

1015 var = tr.var()

1016

1017 op = "" if self.inverted else "not"

1018

1019 code = [

1020 tr.emit_assign(var, tr.emit(self.subject), mute=True),

1021 f'if {op} _ENV.isempty({var}):',

1022 C.PY_BEGIN,

1023 [tr.emit(c) for c in self.blocks[0]],

1024 C.PY_END

1025 ]

1026

1027 if len(self.blocks) > 1:

1028 code.extend([

1029 'else:',

1030 C.PY_BEGIN,

1031 [tr.emit(c) for c in self.blocks[1]],

1032 C.PY_END

1033 ])

1034

1035 return code

1036

1037 COMMANDS = {

1038 'def': DefineFunction,

1039 'box': DefineFunction,

1040 'mdef': DefineFunction,

1041 'mbox': DefineFunction,

1042

1043 'break': CommandBreakContinue,

1044 'code': CommandCode,

1045 'comment': CommandComment,

1046 'continue': CommandBreakContinue,

1047 'do': CommandDo,

1048 'each': CommandFor,

1049 'for': CommandFor,

1050 'if': CommandIf,

1051 'import': CommandImport,

1052 'include': CommandInclude,

1053 'let': CommandLet,

1054 'option': CommandOption,

1055 'print': CommandPrint,

1056 'quote': CommandQuote,

1057 'return': CommandReturn,

1058 'skip': CommandComment,

1059 'with': CommandWith,

1060 'without': CommandWith,

1061 }

1062

1063

1064class CompileError(ValueError):

1065 def __init__(self, message, loc: Location, start_loc: Location = None):

1066 self.path = loc.path if loc else '<unknown>'

1067 self.line = loc.line_num if loc else '?'

1068 if loc:

1069 message += ' in ' + _path_line(loc.path, loc.line_num)

1070 if start_loc:

1071 message += ' (started in ' + _path_line(start_loc.path, start_loc.line_num) + ')'

1072 super().__init__(message)

1073 self.message = message

1074

1075

1076class Buffer:

1077 def __init__(self):

1078 self.length = 0

1079 self.line_pointers = []

1080 self.location_cache = {}

1081 self.paths = []

1082 self.pos = 0

1083 self.text = ''

1084

1085 def paste(self, text, path, pos):

1086 try:

1087 path_index = self.paths.index(path)

1088 except ValueError:

1089 self.paths.append(path)

1090 path_index = len(self.paths) - 1

1091

1092 if not text:

1093 if not self.line_pointers:

1094 self.line_pointers = [[0, path_index, 1]]

1095 return

1096

1097 text_len = len(text)

1098 lps = self.line_pointers

1099

1100 for lp in lps:

1101 if lp[0] >= pos:

1102 lp[0] += text_len

1103

1104 line_num = 1

1105 p = 0

1106 while p < text_len:

1107 lps.append([pos + p, path_index, line_num])

1108 p = text.find(C.NL, p)

1109 if p < 0:

1110 break

1111 p += 1

1112 line_num += 1

1113

1114 lps.sort()

1115 self.text = self.text[:pos] + text + self.text[pos:]

1116 self.length = len(self.text)

1117 self.location_cache = {}

1118

1119 def loc(self, pos=None):

1120 if pos is None:

1121 pos = self.pos

1122

1123 if pos in self.location_cache:

1124 return self.location_cache[pos]

1125

1126 lptr = self.line_pointers[self.lptr_index(pos)]

1127 line_start_pos, path_index, line_num = lptr

1128 loc = Location(

1129 line_start_pos=line_start_pos,

1130 path_index=path_index,

1131 path=self.paths[path_index],

1132 line_num=line_num,

1133 column=pos - line_start_pos,

1134 pos=pos)

1135

1136 self.location_cache[pos] = loc

1137 return loc

1138

1139 def lptr_index(self, pos):

1140 lps = self.line_pointers

1141 le = 0

1142 ri = len(lps) - 1

1143 while le <= ri:

1144 m = (le + ri) >> 1

1145 if lps[m][0] < pos:

1146 le = m + 1

1147 elif lps[m][0] > pos:

1148 ri = m - 1

1149 else:

1150 return m

1151 return le - 1

1152

1153 def eof(self):

1154 return self.pos >= self.length

1155

1156 def at_line_start(self, pos=None):

1157 if pos is None:

1158 pos = self.pos

1159 lptr = self.line_pointers[self.lptr_index(pos)]

1160 p = lptr[0]

1161 while p < self.pos:

1162 if self.text[p] not in C.WS:

1163 return False

1164 p += 1

1165 return True

1166

1167 def at_space(self):

1168 return self.char() in C.WSNL

1169

1170 def at_string(self, s):

1171 if not s:

1172 return False

1173 return self.text.startswith(s, self.pos)

1174

1175 def char(self):

1176 try:

1177 return self.text[self.pos]

1178 except IndexError:

1179 return ''

1180

1181 def next(self, n=1):

1182 self.pos += n

1183

1184 def to(self, p):

1185 self.pos = p

1186

1187 def line_tail(self):

1188 li = self.lptr_index(self.pos)

1189 if li < len(self.line_pointers) - 1:

1190 p = self.line_pointers[li + 1][0] - 1

1191 s = self.text[self.pos:p]

1192 self.pos = p + 1

1193 else:

1194 s = self.text[self.pos:]

1195 self.pos = self.length

1196 return s.strip()

1197

1198 def find(self, s, pos):

1199 return self.text.find(s, pos)

1200

1201 def skip_ws(self, with_nl: bool):

1202 p = self.pos

1203 chars = C.WSNL if with_nl else C.WS

1204 while p < self.length:

1205 if self.text[p] not in chars:

1206 break

1207 p += 1

1208 if p == self.pos:

1209 return False

1210 self.pos = p

1211 return True

1212

1213 def strpbrk(self, chars):

1214 p = self.pos

1215 while p < self.length:

1216 if self.text[p] in chars:

1217 break

1218 p += 1

1219 return p

1220

1221 def slice(self, a, b):

1222 return self.text[a:b]

1223

1224

1225class Lexer:

1226 def __init__(self, buf: Buffer):

1227 self.buf = buf

1228 self.token_cache = {}

1229

1230 def token(self, ignore_nl: bool):

1231 pos = self.buf.pos

1232 if pos not in self.token_cache:

1233 self.token_cache[pos] = self.token2(ignore_nl)

1234 self.buf.to(self.token_cache[pos][C.TOK_NEXTPOS])

1235 return self.token_cache[pos]

1236

1237 def back(self, tok):

1238 self.buf.to(tok[C.TOK_POS])

1239

1240 def token2(self, ignore_nl):

1241 value = None

1242 pos = self.buf.pos

1243

1244 space_before = self.buf.skip_ws(with_nl=ignore_nl)

1245

1246 if self.buf.eof():

1247 typ = T.EOF

1248 elif self.buf.char() == C.NL:

1249 typ = T.NEWLINE

1250 else:

1251 tv = self.token3()

1252 if tv:

1253 typ = tv[0]

1254 value = tv[1]

1255 else:

1256 self.buf.to(pos)

1257 typ = T.INVALID

1258

1259 return [

1260 typ,

1261 pos,

1262 self.buf.pos,

1263 space_before,

1264 self.buf.at_space(),

1265 value

1266 ]

1267

1268 def token3(self):

1269 ch = self.buf.char()

1270

1271 if ch in C.IDENTIFIER_START:

1272 s = self.identifier()

1273 if s in C.KEYWORDS:

1274 if s == 'not':

1275 pos2 = self.buf.pos

1276 self.buf.skip_ws(with_nl=False)

1277 s2 = self.identifier()

1278 if s2 == 'in':

1279 return 'not in', None

1280 self.buf.to(pos2)

1281 return s, None

1282 if s in C.CONST:

1283 return T.CONST, C.CONST[s]

1284 return T.NAME, s

1285

1286 if ch in C.QUOTES:

1287 return T.STRING, self.string()

1288

1289 if ch in C.DIGIT_DEC:

1290 return T.NUMBER, self.number()

1291

1292 if ch in C.PUNCT:

1293 return self.punctuation(), None

1294

1295 def identifier(self):

1296 chars = ''

1297

1298 while not self.buf.eof():

1299 ch = self.buf.char()

1300 if ch not in C.IDENTIFIER_CONTINUE:

1301 break

1302 chars += ch

1303 self.buf.next()

1304

1305 return chars

1306

1307 def punctuation(self):

1308 c1 = self.buf.char()

1309 if c1 in C.PUNCT:

1310 self.buf.next()

1311 c2 = self.buf.char()

1312 if c1 + c2 in C.PUNCT:

1313 self.buf.next()

1314 return c1 + c2

1315 return c1

1316

1317 def string(self):

1318 quot = self.buf.char()

1319 pos = self.buf.pos

1320 self.buf.next()

1321 chars = self.string_to(quot, pos)

1322 self.buf.next()

1323 return chars

1324

1325 def string_to(self, end_sym, start_pos):

1326 chars = ''

1327

1328 while True:

1329 if self.buf.at_string(end_sym):

1330 break

1331

1332 ch = self.buf.char()

1333 if not ch or ch == C.NL:

1334 raise CompileError('unterminated string', self.buf.loc(self.buf.pos), self.buf.loc(start_pos))

1335 if ch == '\\':

1336 self.buf.next()

1337 chars += self.string_escape()

1338 else:

1339 self.buf.next()

1340 chars += ch

1341

1342 return chars

1343

1344 def string_escape(self):

1345 ch = self.buf.char()

1346

1347 if ch in C.STRING_ESCAPES:

1348 self.buf.next()

1349 return C.STRING_ESCAPES[ch]

1350

1351 if ch == 'x' or ch == 'X' or ch == 'u' or ch == 'U':

1352 self.buf.next()

1353 code = self.unicode_escape(ch)

1354 if 0 <= code < 0x110000:

1355 return chr(code)

1356 raise CompileError('invalid escape sequence', self.buf.loc(self.buf.pos))

1357

1358 self.buf.next()

1359 return ch

1360

1361 def unicode_escape(self, prefix):

1362 # \xXX

1363 if prefix == 'x' or prefix == 'X':

1364 return self.hexcode(2, 2)

1365

1366 # \UXXXXXXXX

1367 if prefix == 'U':

1368 return self.hexcode(8, 8)

1369

1370 # \u{XX...}

1371 if prefix == 'u' and self.buf.char() == '{':

1372 self.buf.next()

1373 code = self.hexcode(1, 8)

1374 if self.buf.char() == '}':

1375 self.buf.next()

1376 return code

1377 return -1

1378

1379 # \uXXXX

1380 if prefix == 'u':

1381 return self.hexcode(4, 4)

1382

1383 return -1

1384

1385 def hexcode(self, minlen, maxlen):

1386 chars = ''

1387

1388 while len(chars) < maxlen:

1389 ch = self.buf.char()

1390 if ch in C.DIGIT_HEX:

1391 self.buf.next()

1392 chars += ch

1393 else:

1394 break

1395

1396 if minlen <= len(chars) <= maxlen:

1397 return int(chars, 16)

1398

1399 return -1

1400

1401 def number(self):

1402 ch = self.buf.char()

1403

1404 if ch == '0':

1405 pos = self.buf.pos

1406 self.buf.next()

1407 ch = self.buf.char()

1408 if ch == 'x' or ch == 'X':

1409 self.buf.next()

1410 return self.nondec_number(C.DIGIT_HEX, 16)

1411 if ch == 'o' or ch == 'O':

1412 self.buf.next()

1413 return self.nondec_number(C.DIGIT_OCT, 8)

1414 if ch == 'b' or ch == 'B':

1415 self.buf.next()

1416 return self.nondec_number(C.DIGIT_BIN, 2)

1417 self.buf.to(pos)

1418

1419 return self.dec_number()

1420

1421 def dec_number(self):

1422 i = self.digit_seq(C.DIGIT_DEC)

1423

1424 f = ''

1425 if self.buf.char() == '.':

1426 self.buf.next()

1427 f = self.digit_seq(C.DIGIT_DEC)

1428 if not f:

1429 raise self.number_error()

1430

1431 if not i and not f:

1432 raise self.number_error()

1433

1434 e = esign = ''

1435 ch = self.buf.char()

1436 if ch == 'e' or ch == 'E':

1437 self.buf.next()

1438 ch = self.buf.char()

1439 if ch == '+' or ch == '-':

1440 esign = ch

1441 self.buf.next()

1442 e = self.digit_seq(C.DIGIT_DEC)

1443 if not e:

1444 raise self.number_error()

1445

1446 if f or e:

1447 return float((i or '0') + '.' + (f or '0') + 'E' + (esign or '') + (e or '0'))

1448

1449 return int(i, 10)

1450

1451 def nondec_number(self, digits, base):

1452 n = self.digit_seq(digits)

1453 if not n:

1454 raise self.number_error()

1455 return int(n, base)

1456

1457 def digit_seq(self, digits):

1458 chars = ''

1459

1460 while True:

1461 ch = self.buf.char()

1462 if ch in digits:

1463 chars += ch

1464 self.buf.next()

1465 elif ch == '_':

1466 self.buf.next()

1467 else:

1468 break

1469

1470 return chars

1471

1472 def number_error(self):

1473 return CompileError('invalid numeric literal', self.buf.loc())

1474

1475

1476class ExpressionParser:

1477 def __init__(self, lex: Lexer):

1478 self.lex = lex

1479 self.paren_stack = []

1480

1481 def expression(self):

1482 subject = self.ifexpr()

1483 if not subject:

1484 return

1485

1486 pipes = []

1487

1488 while True:

1489 tok = self.token()

1490 if tok[C.TOK_TYPE] == '|' and tok[C.TOK_SPACE_BEFORE] == tok[C.TOK_SPACE_AFTER]:

1491 pipes.append(self.expect(self.primary))

1492 else:

1493 self.lex.back(tok)

1494 break

1495

1496 if not pipes:

1497 return subject

1498

1499 return Node.PipeList(subject, pipes)

1500

1501 def ifexpr(self):

1502 yes = self.orexpr()

1503 if not yes:

1504 return

1505

1506 tok = self.token()

1507 if tok[C.TOK_TYPE] == 'if':

1508 cond = self.expect(self.orexpr)

1509 self.expect_token('else')

1510 return Node.IfExpression(cond, yes, self.expect(self.ifexpr))

1511 self.lex.back(tok)

1512

1513 return yes

1514

1515 def orexpr(self):

1516 return self.binary_op(Node.Or, {'or'}, self.andexpr)

1517

1518 def andexpr(self):

1519 return self.binary_op(Node.And, {'and'}, self.notexpr)

1520

1521 def notexpr(self):

1522 return self.unary_op(Node.Not, {'not'}, self.comparison)

1523

1524 def comparison(self):

1525 return self.binary_op(Node.Comparison, C.COMPARE_OPS, self.sum)

1526

1527 def sum(self):

1528 return self.binary_op(Node.Sum, C.ADD_OPS, self.product)

1529

1530 def product(self):

1531 return self.binary_op(Node.Product, C.MUL_OPS, self.power)

1532

1533 def power(self):

1534 return self.binary_op(Node.Power, C.POWER_OP, self.unary)

1535

1536 def unary(self):

1537 return self.unary_op(Node.Unary, C.ADD_OPS, self.primary)

1538

1539 def primary(self):

1540 node = self.atom()

1541 if not node:

1542 return

1543

1544 while True:

1545 tok = self.token()

1546

1547 if tok[C.TOK_TYPE] == '.' and not tok[C.TOK_SPACE_BEFORE]:

1548 node = Node.Attr(node, self.expect_name().ident)

1549 continue

1550

1551 if tok[C.TOK_TYPE] == '(' and not tok[C.TOK_SPACE_BEFORE]:

1552 self.push_paren('(')

1553 node = Node.Call(node, self.generic_list(self.arg_list_item))

1554 self.pop_paren()

1555 continue

1556

1557 if tok[C.TOK_TYPE] == '[' and not tok[C.TOK_SPACE_BEFORE]:

1558 self.push_paren('[')

1559 node = Node.Index(node, self.index_list())

1560 self.pop_paren()

1561 continue

1562

1563 self.lex.back(tok)

1564 break

1565

1566 return node

1567

1568 def atom(self):

1569 tok = self.token()

1570

1571 if tok[C.TOK_TYPE] == T.STRING:

1572 return Node.String(tok[C.TOK_VALUE])

1573

1574 if tok[C.TOK_TYPE] == T.NUMBER:

1575 return Node.Number(tok[C.TOK_VALUE])

1576

1577 if tok[C.TOK_TYPE] == T.CONST:

1578 return Node.Const(tok[C.TOK_VALUE])

1579

1580 if tok[C.TOK_TYPE] == T.NAME:

1581 return Node.Name(tok[C.TOK_VALUE])

1582

1583 if tok[C.TOK_TYPE] == '(':

1584 self.push_paren('(')

1585 e = self.expression()

1586 self.pop_paren()

1587 return e

1588

1589 if tok[C.TOK_TYPE] == '[':

1590 self.push_paren('[')

1591 items = self.generic_list(self.expression)

1592 self.pop_paren()

1593 return Node.List(items)

1594

1595 if tok[C.TOK_TYPE] == '{':

1596 self.push_paren('{')

1597 items = self.generic_list(self.dict_item)

1598 self.pop_paren()

1599 return Node.Dict(items)

1600

1601 self.lex.back(tok)

1602

1603 # lists

1604

1605 def generic_list_with_opt_parens(self, item_fn):

1606 tok = self.token()

1607

1608 if tok[C.TOK_TYPE] == '(':

1609 self.push_paren('(')

1610 res = self.generic_list(item_fn)

1611 self.pop_paren()

1612 return res

1613

1614 self.lex.back(tok)

1615 if self.lex.buf.at_space():

1616 return self.generic_list(item_fn)

1617

1618 return []

1619

1620 def generic_list(self, fn):

1621 items = []

1622 nl = bool(self.paren_stack)

1623

1624 while True:

1625 a = fn()

1626 if not a:

1627 break

1628 items.append(a)

1629

1630 pos = self.lex.buf.pos

1631 has_delim = self.lex.buf.skip_ws(with_nl=nl)

1632 if self.lex.buf.char() == ',':

1633 has_delim = True

1634 self.lex.buf.next()

1635 self.lex.buf.skip_ws(with_nl=nl)

1636 if not has_delim:

1637 self.lex.buf.to(pos)

1638 break

1639

1640 return items

1641

1642 def top_level_arg_list(self):

1643 return self.generic_list_with_opt_parens(self.arg_list_item)

1644

1645 def arg_list_item(self):

1646 tok = self.token()

1647

1648 if tok[C.TOK_TYPE] == '*' and not tok[C.TOK_SPACE_AFTER]:

1649 return Node.Argument(None, 1, self.expect_expression())

1650

1651 if tok[C.TOK_TYPE] == '**' and not tok[C.TOK_SPACE_AFTER]:

1652 return Node.Argument(None, 2, self.expect_expression())

1653

1654 if tok[C.TOK_TYPE] == T.NAME:

1655 if self.arg_equal():

1656 return Node.Argument(tok[C.TOK_VALUE], 0, self.expect_expression())

1657

1658 self.lex.back(tok)

1659

1660 expr = self.expression()

1661 if expr:

1662 return Node.Argument(None, 0, expr)

1663

1664 def top_level_param_list(self):

1665 return self.generic_list_with_opt_parens(self.param_list_item)

1666

1667 def param_list_item(self):

1668 tok = self.token()

1669

1670 if tok[C.TOK_TYPE] == T.NAME:

1671 if self.arg_equal():

1672 return Node.Param(tok[C.TOK_VALUE], 0, self.expect_expression())

1673 return Node.Param(tok[C.TOK_VALUE], 0, None)

1674

1675 if tok[C.TOK_TYPE] == '*' and not tok[C.TOK_SPACE_AFTER]:

1676 n = self.expect_name()

1677 return Node.Param(n.ident, 1, None)

1678

1679 if tok[C.TOK_TYPE] == '**' and not tok[C.TOK_SPACE_AFTER]:

1680 n = self.expect_name()

1681 return Node.Param(n.ident, 2, None)

1682

1683 self.lex.back(tok)

1684

1685 def name_list(self):

1686 ls = self.generic_list(self.name_list_item)

1687 if not ls:

1688 raise CompileError('name expected', self.lex.buf.loc())

1689 return ls

1690

1691 def name_list_item(self):

1692 tok = self.token()

1693 if tok[C.TOK_TYPE] == T.NAME:

1694 return Node.Name(tok[C.TOK_VALUE])

1695 self.lex.back(tok)

1696

1697 def expression_list(self):

1698 ls = self.generic_list(self.expression_list_item)

1699 if not ls:

1700 raise CompileError('expression expected', self.lex.buf.loc())

1701 return ls

1702

1703 def expression_list_item(self):

1704 pos = self.lex.buf.pos

1705 e = self.expression()

1706 if e:

1707 return e

1708 self.lex.buf.to(pos)

1709

1710 def index_list(self):

1711 args = [self.expression()]

1712 if self.slice_colon():

1713 args.append(self.expression())

1714 if self.slice_colon():

1715 args.append(self.expression())

1716

1717 return args

1718

1719 def arg_equal(self):

1720 tok = self.token()

1721 if tok[C.TOK_TYPE] == '=' and not tok[C.TOK_SPACE_BEFORE] and not tok[C.TOK_SPACE_AFTER]:

1722 return True

1723 self.lex.back(tok)

1724 return False

1725

1726 def slice_colon(self):

1727 tok = self.token()

1728 if tok[C.TOK_TYPE] == ':':

1729 return True

1730 self.lex.back(tok)

1731 return False

1732

1733 def dict_item(self):

1734 k = self.expression()

1735 if k:

1736 self.expect_token(':')

1737 return [k, self.expect_expression()]

1738

1739 # expression utils

1740

1741 def unary_op(self, typ, operators, arg_fn):

1742 pos = self.lex.buf.pos

1743 ops = []

1744

1745 while True:

1746 tok = self.token()

1747 if tok[C.TOK_TYPE] in operators and (tok[C.TOK_TYPE] in C.KEYWORD_OPS or not tok[C.TOK_SPACE_AFTER]):

1748 ops.append(tok[C.TOK_TYPE])

1749 else:

1750 self.lex.back(tok)

1751 break

1752

1753 subject = arg_fn()

1754 if not subject:

1755 self.lex.buf.to(pos)

1756 return

1757

1758 if not ops:

1759 return subject

1760

1761 return typ(subject, ops)

1762

1763 def binary_op(self, typ, operators, arg_fn):

1764 subject = arg_fn()

1765 if not subject:

1766 return

1767

1768 pairs = []

1769

1770 while True:

1771 tok = self.token()

1772 if tok[C.TOK_TYPE] in operators and tok[C.TOK_SPACE_BEFORE] == tok[C.TOK_SPACE_AFTER]:

1773 pairs.append([tok[C.TOK_TYPE], self.expect(arg_fn)])

1774 else:

1775 self.lex.back(tok)

1776 break

1777

1778 if not pairs:

1779 return subject

1780

1781 return typ(subject, pairs)

1782

1783 def expect(self, fn):

1784 node = fn()

1785 if node:

1786 return node

1787 raise CompileError(f'{fn.__name__!r} expected', self.lex.buf.loc())

1788

1789 def expect_token(self, typ):

1790 tok = self.token()

1791 if tok[C.TOK_TYPE] == typ:

1792 return tok

1793 raise CompileError(f'expected {typ!r}, found {tok[C.TOK_TYPE]!r}', self.lex.buf.loc())

1794

1795 def expect_expression(self):

1796 return self.expect(self.expression)

1797

1798 def expect_name(self):

1799 tok = self.expect_token(T.NAME)

1800 return Node.Name(tok[C.TOK_VALUE])

1801

1802 def push_paren(self, s):

1803 self.paren_stack.append(s)

1804

1805 def pop_paren(self):

1806 if not self.paren_stack:

1807 raise CompileError('unbalanced parentheses', self.lex.buf.loc())

1808 p = self.paren_stack.pop()

1809 if p == '(':

1810 self.expect_token(')')

1811 if p == '[':

1812 self.expect_token(']')

1813 if p == '{':

1814 self.expect_token('}')

1815

1816 def token(self):

1817 return self.lex.token(bool(self.paren_stack))

1818

1819

1820class TemplateParser:

1821 def __init__(self, compiler: 'Compiler'):

1822 self.cc = compiler

1823 self.buf = self.cc.buf

1824 self.lex = Lexer(self.buf)

1825 self.expr = ExpressionParser(self.lex)

1826 self.stack = []

1827 self.special_chars = ''

1828 self.elements = []

1829 self.escape_dct = {}

1830

1831 self.static_function_bindings = {}

1832 for a in dir(self.cc.engine):

1833 if '_' in a:

1834 cmd, _, name = a.partition('_')

1835 if cmd in C.DEF_COMMANDS:

1836 self.static_function_bindings[name] = [cmd, a]

1837

1838 self.reset()

1839

1840 def reset(self):

1841 self.buf.location_cache = {}

1842 self.lex.token_cache = {}

1843

1844 self.escape_dct = dict(zip(

1845 self.cc.options.escapes.split()[::2],

1846 self.cc.options.escapes.split()[1::2]

1847 ))

1848

1849 self.special_chars = ''.join(set(

1850 self.cc.options.comment_symbol

1851 + self.cc.options.command_symbol

1852 + self.cc.options.inline_open_symbol

1853 + self.cc.options.inline_close_symbol

1854 + self.cc.options.echo_open_symbol

1855 + self.cc.options.echo_close_symbol

1856 + ''.join(self.cc.options.escapes.split())

1857 ))

1858

1859 # make sure longest parsers come first

1860 by_len = [

1861 [self.cc.options.inline_open_symbol, self.inline_element],

1862 [self.cc.options.echo_open_symbol, self.echo_element],

1863 [self.cc.options.comment_symbol, self.comment_element],

1864 [self.cc.options.command_symbol, self.command_element],

1865 ]

1866 by_len.sort(key=lambda h: len(h[0]), reverse=True)

1867

1868 self.elements = [self.text_element, self.escape_element] + [h[1] for h in by_len]

1869

1870 def parse(self):

1871 self.stack = [Node.Template()]

1872 self.add_child(Node.Location(self.buf.loc(0)))

1873

1874 while not self.buf.eof():

1875 if not any(f() for f in self.elements):

1876 self.add_raw_text(self.buf.char())

1877 self.buf.next()

1878

1879 if len(self.stack) > 1:

1880 raise CompileError(

1881 f'missing {C.END_SYMBOL!r}',

1882 self.buf.loc(),

1883 self.buf.loc(self.top().start_pos))

1884

1885 return self.stack[0]

1886

1887 def text_element(self):

1888 p = self.buf.strpbrk(self.special_chars)

1889 if p > self.buf.pos:

1890 self.add_text(self.buf.slice(self.buf.pos, p), self.buf.pos)

1891 self.buf.to(p)

1892 return True

1893

1894 def escape_element(self):

1895 for esc, repl in self.escape_dct.items():

1896 if self.buf.at_string(esc):

1897 self.add_raw_text(repl)

1898 self.buf.next(len(esc))

1899 return True

1900

1901 def echo_element(self):

1902 if not self.buf.at_string(self.cc.options.echo_open_symbol):

1903 return False

1904 return self.inline_or_echo_element(

1905 self.cc.options.echo_open_symbol,

1906 self.cc.options.echo_close_symbol,

1907 self.cc.options.echo_start_whitespace,

1908 self.parse_echo)

1909

1910 def inline_element(self):

1911 if not self.buf.at_string(self.cc.options.inline_open_symbol):

1912 return False

1913 return self.inline_or_echo_element(

1914 self.cc.options.inline_open_symbol,

1915 self.cc.options.inline_close_symbol,

1916 self.cc.options.inline_start_whitespace,

1917 self.parse_inline_command)

1918

1919 def inline_or_echo_element(self, open_sym, close_sym, whitespace_flag, parse_func):

1920 start_pos = self.buf.pos

1921

1922 self.buf.next(len(open_sym))

1923 has_ws = self.buf.skip_ws(with_nl=True)

1924

1925 # { at EOF or followed by whitespace

1926 if self.buf.eof() or (has_ws and not whitespace_flag):

1927 self.add_text(self.buf.slice(start_pos, self.buf.pos), start_pos)

1928 return True

1929

1930 # { whitespace... }

1931 if self.buf.at_string(close_sym):

1932 self.buf.next(len(close_sym))

1933 self.add_text(self.buf.slice(start_pos, self.buf.pos), start_pos)

1934 return True

1935

1936 return parse_func(start_pos)

1937

1938 def command_element(self):

1939 if not (self.buf.at_line_start() and self.buf.at_string(self.cc.options.command_symbol)):

1940 return False

1941 start_pos = self.buf.pos

1942 self.buf.next(len(self.cc.options.command_symbol))

1943 self.parse_command(start_pos, is_inline=False)

1944 return True

1945

1946 def comment_element(self):

1947 if not (self.buf.at_line_start() and self.buf.at_string(self.cc.options.comment_symbol)):

1948 return False

1949 self.strip_last_indent()

1950 _ = self.buf.line_tail()

1951 return True

1952

1953 def parse_echo(self, start_pos):

1954 self.add_child(Node.Location(self.buf.loc(start_pos)))

1955 Node.Echo(start_pos).parse(self)

1956 return True

1957

1958 def parse_inline_command(self, start_pos):

1959 return self.parse_command(start_pos, is_inline=True)

1960

1961 def parse_command(self, start_pos, is_inline):

1962 cmd = self.lex.identifier()

1963 if not cmd:

1964 self.add_text(self.buf.slice(start_pos, self.buf.pos), start_pos)

1965 return True

1966

1967 if not is_inline:

1968 self.strip_last_indent()

1969

1970 self.add_child(Node.Location(self.buf.loc(start_pos)))

1971

1972 # custom command?

1973 p = self.static_function_bindings.get(cmd)

1974 if p:

1975 Node.CallFunctionAsCommand(cmd, start_pos).parse(self, is_inline)

1976 return True

1977

1978 # else, elif, end?

1979 if cmd in C.AUX_COMMANDS:

1980 fn = getattr(self.top(), 'parse_' + cmd, None)

1981 try:

1982 fn(self, is_inline)

1983 return True

1984 except (TypeError, ValueError):

1985 raise CompileError(f'unexpected {cmd!r}', self.buf.loc(start_pos))

1986

1987 # built-in command?

1988 cls = Node.COMMANDS.get(cmd)

1989 if cls:

1990 cls(cmd, start_pos).parse(self, is_inline)

1991 return True

1992

1993 raise CompileError(f'unknown command {cmd!r}', self.buf.loc(start_pos))

1994

1995 def add_text(self, text, pos):

1996 if self.cc.options.strip:

1997 lines = text.split(C.NL)

1998 n = 0 if self.buf.at_line_start(pos) else 1

1999 while n < len(lines):

2000 lines[n] = lines[n].lstrip()

2001 n += 1

2002 text = C.NL.join(lines)

2003 return self.add_child(Node.Text(text))

2004

2005 def add_raw_text(self, text):

2006 return self.add_child(Node.Text(text))

2007

2008 def strip_last_indent(self):

2009 last = self.top().children[-1] if self.top().children else None

2010 if isinstance(last, Node.Text):

2011 last.text = last.text.rstrip(C.WS)

2012 if not last.text:

2013 self.top().children.pop()

2014

2015 def command_tail(self, is_inline):

2016 if is_inline:

2017 p = self.buf.find(self.cc.options.inline_close_symbol, self.buf.pos)

2018 if p < 0:

2019 raise CompileError('unterminated command', self.buf.loc())

2020 s = self.buf.slice(self.buf.pos, p)

2021 self.buf.to(p + len(self.cc.options.inline_close_symbol))

2022 else:

2023 p = self.buf.find(C.NL, self.buf.pos)

2024 if p < 0:

2025 s = self.buf.slice(self.buf.pos, self.buf.length)

2026 self.buf.to(self.buf.length)

2027 else:

2028 s = self.buf.slice(self.buf.pos, p)

2029 self.buf.to(p + 1)

2030

2031 return s.strip()

2032

2033 def expect_end_of_command(self, is_inline):

2034 tail = self.command_tail(is_inline)

2035 if tail:

2036 tail = _cut(tail, 10)

2037 raise CompileError(f'expected end of command, found {tail!r}', self.buf.loc())

2038

2039 def expect_dotted_names(self):

2040 start_pos = self.buf.pos

2041 names = []

2042 expr = self.expr.expect_expression()

2043 while True:

2044 if isinstance(expr, Node.Attr):

2045 names.insert(0, expr.name)

2046 expr = expr.subject

2047 elif isinstance(expr, Node.Name):

2048 names.insert(0, expr.ident)

2049 break

2050 else:

2051 raise CompileError('qualified name expected', self.buf.loc(start_pos))

2052 return names

2053

2054 def command_label(self, is_inline):

2055 s = ''

2056 if self.buf.at_space():

2057 self.buf.skip_ws(with_nl=is_inline)

2058 s = self.lex.identifier()

2059 return s

2060

2061 def quoted_content(self, label, is_inline):

2062 start_pos = self.buf.pos

2063

2064 end_sym = (self.cc.options.inline_open_symbol if is_inline else self.cc.options.command_symbol) + C.END_SYMBOL

2065

2066 while not self.buf.eof():

2067 end_pos = self.buf.find(end_sym, self.buf.pos)

2068 if end_pos < 0:

2069 break

2070 at_line_start = self.buf.at_line_start()

2071 self.buf.to(end_pos + len(end_sym))

2072 if not is_inline and not at_line_start:

2073 continue

2074 end_label = self.command_label(is_inline)

2075 tail = self.command_tail(is_inline)

2076 if end_label == label and not tail:

2077 s = self.buf.slice(start_pos, end_pos)

2078 if not is_inline:

2079 s = s.rstrip(C.WS)

2080 return s

2081

2082 raise CompileError(f'missing {C.END_SYMBOL!r}', self.buf.loc(), self.buf.loc(start_pos))

2083

2084 def top(self):

2085 return self.stack[-1]

2086

2087 def add_child(self, node):

2088 self.top().children.append(node)

2089 return node

2090

2091 def begin_command(self, node):

2092 node = self.add_child(node)

2093 self.stack.append(node)

2094 return node

2095

2096 def end_command(self, node, is_inline):

2097 label = self.command_label(is_inline)

2098 if label and label != node.cmd:

2099 a = C.END_SYMBOL + ' ' + node.cmd

2100 b = C.END_SYMBOL + ' ' + label

2101 raise CompileError(f'expected {a!r}, found {b!r}', self.buf.loc())

2102 self.expect_end_of_command(is_inline)

2103 self.stack.pop()

2104

2105 def in_loop_context(self):

2106 for node in reversed(self.stack):

2107 if isinstance(node, Node.CommandFor):

2108 return True

2109 if isinstance(node, Node.DefineFunction):

2110 break

2111 return False

2112

2113 def in_def_context(self):

2114 for node in reversed(self.stack):

2115 if isinstance(node, Node.DefineFunction):

2116 return True

2117 return False

2118

2119

2120class Translator:

2121 def __init__(self, cc: 'Compiler'):

2122 self.cc = cc

2123 self.num_vars = 0

2124 self.locals = {'_', 'ARGS'}

2125 self.frames = []

2126

2127 def translate(self, node):

2128 code = []

2129 indent = 2 # see py_template

2130 cur_loc = '(0,0)'

2131 text_buf = []

2132

2133 for elem in _flatten(self.emit(node)):

2134 if isinstance(elem, Node.Text):

2135 text_buf.append(elem.text or '')

2136 continue

2137

2138 if text_buf:

2139 s = ''.join(text_buf)

2140 if s:

2141 code.append((C.PY_INDENT * indent) + f'_ENV.echo({s!r})')

2142 text_buf = []

2143

2144 if isinstance(elem, Node.Location):

2145 new_loc = _parens(f'{elem.loc.path_index},{elem.loc.line_num}')

2146 if new_loc != cur_loc:

2147 code.append(C.PY_MARKER + _path_line(elem.loc.path, elem.loc.line_num))

2148 cur_loc = new_loc

2149 continue

2150

2151 if elem == C.PY_BEGIN:

2152 indent += 1

2153 continue

2154

2155 if elem == C.PY_END:

2156 indent -= 1

2157 continue

2158

2159 if elem.strip():

2160 code.append((C.PY_INDENT * indent) + elem.replace('$loc$', cur_loc))

2161

2162 s = ''.join(text_buf)

2163 if s:

2164 code.append((C.PY_INDENT * indent) + f'_ENV.echo({s!r})')

2165

2166 py = C.PY_TEMPLATE

2167 py = py.replace('$name$', self.cc.options.name)

2168 py = py.replace('$paths$', repr(self.cc.buf.paths))

2169 py = py.replace('$code$', C.NL + C.NL.join(code) + C.NL)

2170 py = py.replace('$loc$', cur_loc)

2171

2172 return py

2173

2174 def var(self):

2175 self.num_vars += 1

2176 return '_' + str(self.num_vars)

2177

2178 def enter_frame(self):

2179 self.frames.append(self.locals)

2180 self.locals = set(self.locals)

2181

2182 def leave_frame(self):

2183 self.locals = self.frames.pop()

2184

2185 def emit(self, node):

2186 return node.emit(self)

2187

2188 def emit_try(self, block, fallback=None, mute=False):

2189 exc = self.var()

2190

2191 if mute:

2192 err = fallback or 'pass'

2193 else:

2194 err = [f'_ENV.error({exc}, $loc$)', fallback or '']

2195

2196 return [

2197 'try:',

2198 C.PY_BEGIN,

2199 block,

2200 C.PY_END,

2201 f'except Exception as {exc}:',

2202 C.PY_BEGIN,

2203 err,

2204 C.PY_END

2205 ]

2206

2207 def emit_assign(self, var, value, fallback='None', mute=False):

2208 return self.emit_try(

2209 f'{var} = {value}',

2210 f'{var} = {fallback}',

2211 mute=mute

2212 )

2213

2214 def emit_echo(self, value, mute=False):

2215 res = self.var()

2216 code = [

2217 f'{res} = {value}',

2218 f'_ENV.echo({res})'

2219 ]

2220 return self.emit_try(code, mute=mute)

2221

2222 def emit_left_binary_op(self, node):

2223 # a + b + c => ((a + b) + c)

2224 code = self.emit(node.subject)

2225 for op, other in node.pairs:

2226 code = _parens(f'{code} {op} {self.emit(other)}')

2227 return code

2228

2229 def emit_right_binary_op(self, node):

2230 # a ** b ** c => (a ** (b ** c))

2231 code = self.emit(node.subject)

2232 for op, other in node.pairs:

2233 code = f'{code} {op} ({self.emit(other)}'

2234 code += ')' * len(node.pairs)

2235 return _parens(code)

2236

2237 def emit_comp_binary_op(self, node):

2238 # a < b < c => (a) < (b) < (c)

2239 codes = [self.emit(node.subject)]

2240 for op, other in node.pairs:

2241 codes.append(op)

2242 codes.append(_parens(self.emit(other)))

2243 return ' '.join(codes)

2244

2245 def emit_unary_op(self, node):

2246 code = self.emit(node.subject)

2247 for op in node.ops:

2248 space = ' ' if op.isalnum() else ''

2249 code = _parens(f'{op}{space}{code}')

2250 return code

2251

2252

2253class Compiler:

2254 def __init__(self, engine, options):

2255 self.engine = engine

2256 self.buf = Buffer()

2257

2258 opts = dict(C.DEFAULT_OPTIONS)

2259 opts.update(options or {})

2260 self.options = Data(**opts)

2261

2262 def load(self, basepath, path, loc):

2263 try:

2264 if callable(self.options.loader):

2265 return self.options.loader(basepath, path)

2266 if not os.path.isabs(path) and basepath:

2267 path = os.path.abspath(os.path.join(os.path.dirname(basepath), path))

2268 with open(path, 'rt', encoding='utf8') as fp:

2269 text = fp.read()

2270 return text, path

2271 except OSError as exc:

2272 raise CompileError(f'cannot load {path!r}: {exc.args[1]}', loc)

2273

2274 def parse(self):

2275 return TemplateParser(self).parse()

2276

2277 def translate(self, node):

2278 return Translator(self).translate(node)

2279

2280 def compile(self, python):

2281 local_vars = {}

2282 exec(python, {}, local_vars)

2283 return local_vars[self.options.name]

2284

2285

2286def _dedent(lines):

2287 ind = 100_000

2288 for ln in lines:

2289 n = len(ln.lstrip())

2290 if n > 0:

2291 ind = min(ind, len(ln) - n)

2292 return [ln[ind:] for ln in lines]

2293

2294

2295def _indent(lines, size=4):

2296 sp = ' ' * size

2297 return [sp + ln for ln in lines]

2298

2299

2300def _flatten(ls):

2301 if not isinstance(ls, (list, tuple)):

2302 yield ls

2303 return

2304

2305 for item in ls:

2306 if not isinstance(item, (list, tuple)):

2307 yield item

2308 else:

2309 yield from _flatten(item)

2310

2311

2312def _path_line(path, line):

2313 s = repr(path)[1:-1]

2314 return s + ':' + str(line)

2315

2316

2317def _parens(s):

2318 return '(' + s + ')'

2319

2320

2321def _unquote(s):

2322 return s.strip('\'\" ')

2323

2324

2325def _cut(s, maxlen):

2326 if len(s) <= maxlen:

2327 return s

2328 return s[:maxlen] + '...'

2329

2330

2331_comma = ','.join