| from collections import namedtuple |
| import shlex |
| import os |
| import re |
| |
| from ..common import util, info |
| |
| |
| CONTINUATION = '\\' + os.linesep |
| |
| IDENTIFIER = r'(?:\w*[a-zA-Z]\w*)' |
| IDENTIFIER_RE = re.compile('^' + IDENTIFIER + '$') |
| |
| |
| def _coerce_str(value): |
| if not value: |
| return '' |
| return str(value).strip() |
| |
| |
| ############################# |
| # directives |
| |
| DIRECTIVE_START = r''' |
| (?: |
| ^ \s* |
| [#] \s* |
| )''' |
| DIRECTIVE_TEXT = r''' |
| (?: |
| (?: \s+ ( .*\S ) )? |
| \s* $ |
| )''' |
| DIRECTIVE = rf''' |
| (?: |
| {DIRECTIVE_START} |
| ( |
| include | |
| error | warning | |
| pragma | |
| define | undef | |
| if | ifdef | ifndef | elseif | else | endif | |
| __FILE__ | __LINE__ | __DATE __ | __TIME__ | __TIMESTAMP__ |
| ) |
| {DIRECTIVE_TEXT} |
| )''' |
| # (?: |
| # [^\\\n] | |
| # \\ [^\n] | |
| # \\ \n |
| # )+ |
| # ) \n |
| # )''' |
| DIRECTIVE_RE = re.compile(DIRECTIVE, re.VERBOSE) |
| |
| DEFINE = rf''' |
| (?: |
| {DIRECTIVE_START} define \s+ |
| (?: |
| ( \w*[a-zA-Z]\w* ) |
| (?: \s* [(] ([^)]*) [)] )? |
| ) |
| {DIRECTIVE_TEXT} |
| )''' |
| DEFINE_RE = re.compile(DEFINE, re.VERBOSE) |
| |
| |
| def parse_directive(line): |
| """Return the appropriate directive for the given line.""" |
| line = line.strip() |
| if line.startswith('#'): |
| line = line[1:].lstrip() |
| line = '#' + line |
| directive = line |
| #directive = '#' + line |
| while ' ' in directive: |
| directive = directive.replace(' ', ' ') |
| return _parse_directive(directive) |
| |
| |
| def _parse_directive(line): |
| m = DEFINE_RE.match(line) |
| if m: |
| name, args, text = m.groups() |
| if args: |
| args = [a.strip() for a in args.split(',')] |
| return Macro(name, args, text) |
| else: |
| return Constant(name, text) |
| |
| m = DIRECTIVE_RE.match(line) |
| if not m: |
| raise ValueError(f'unsupported directive {line!r}') |
| kind, text = m.groups() |
| if not text: |
| if kind not in ('else', 'endif'): |
| raise ValueError(f'missing text in directive {line!r}') |
| elif kind in ('else', 'endif', 'define'): |
| raise ValueError(f'unexpected text in directive {line!r}') |
| if kind == 'include': |
| directive = Include(text) |
| elif kind in IfDirective.KINDS: |
| directive = IfDirective(kind, text) |
| else: |
| directive = OtherDirective(kind, text) |
| directive.validate() |
| return directive |
| |
| |
| class PreprocessorDirective(util._NTBase): |
| """The base class for directives.""" |
| |
| __slots__ = () |
| |
| KINDS = frozenset([ |
| 'include', |
| 'pragma', |
| 'error', 'warning', |
| 'define', 'undef', |
| 'if', 'ifdef', 'ifndef', 'elseif', 'else', 'endif', |
| '__FILE__', '__DATE__', '__LINE__', '__TIME__', '__TIMESTAMP__', |
| ]) |
| |
| @property |
| def text(self): |
| return ' '.join(v for v in self[1:] if v and v.strip()) or None |
| |
| def validate(self): |
| """Fail if the object is invalid (i.e. init with bad data).""" |
| super().validate() |
| |
| if not self.kind: |
| raise TypeError('missing kind') |
| elif self.kind not in self.KINDS: |
| raise ValueError |
| |
| # text can be anything, including None. |
| |
| |
| class Constant(PreprocessorDirective, |
| namedtuple('Constant', 'kind name value')): |
| """A single "constant" directive ("define").""" |
| |
| __slots__ = () |
| |
| def __new__(cls, name, value=None): |
| self = super().__new__( |
| cls, |
| 'define', |
| name=_coerce_str(name) or None, |
| value=_coerce_str(value) or None, |
| ) |
| return self |
| |
| def validate(self): |
| """Fail if the object is invalid (i.e. init with bad data).""" |
| super().validate() |
| |
| if not self.name: |
| raise TypeError('missing name') |
| elif not IDENTIFIER_RE.match(self.name): |
| raise ValueError(f'name must be identifier, got {self.name!r}') |
| |
| # value can be anything, including None |
| |
| |
| class Macro(PreprocessorDirective, |
| namedtuple('Macro', 'kind name args body')): |
| """A single "macro" directive ("define").""" |
| |
| __slots__ = () |
| |
| def __new__(cls, name, args, body=None): |
| # "args" must be a string or an iterable of strings (or "empty"). |
| if isinstance(args, str): |
| args = [v.strip() for v in args.split(',')] |
| if args: |
| args = tuple(_coerce_str(a) or None for a in args) |
| self = super().__new__( |
| cls, |
| kind='define', |
| name=_coerce_str(name) or None, |
| args=args if args else (), |
| body=_coerce_str(body) or None, |
| ) |
| return self |
| |
| @property |
| def text(self): |
| if self.body: |
| return f'{self.name}({", ".join(self.args)}) {self.body}' |
| else: |
| return f'{self.name}({", ".join(self.args)})' |
| |
| def validate(self): |
| """Fail if the object is invalid (i.e. init with bad data).""" |
| super().validate() |
| |
| if not self.name: |
| raise TypeError('missing name') |
| elif not IDENTIFIER_RE.match(self.name): |
| raise ValueError(f'name must be identifier, got {self.name!r}') |
| |
| for arg in self.args: |
| if not arg: |
| raise ValueError(f'missing arg in {self.args}') |
| elif not IDENTIFIER_RE.match(arg): |
| raise ValueError(f'arg must be identifier, got {arg!r}') |
| |
| # body can be anything, including None |
| |
| |
| class IfDirective(PreprocessorDirective, |
| namedtuple('IfDirective', 'kind condition')): |
| """A single conditional directive (e.g. "if", "ifdef"). |
| |
| This only includes directives that actually provide conditions. The |
| related directives "else" and "endif" are covered by OtherDirective |
| instead. |
| """ |
| |
| __slots__ = () |
| |
| KINDS = frozenset([ |
| 'if', |
| 'ifdef', |
| 'ifndef', |
| 'elseif', |
| ]) |
| |
| @classmethod |
| def _condition_from_raw(cls, raw, kind): |
| #return Condition.from_raw(raw, _kind=kind) |
| condition = _coerce_str(raw) |
| if not condition: |
| return None |
| |
| if kind == 'ifdef': |
| condition = f'defined({condition})' |
| elif kind == 'ifndef': |
| condition = f'! defined({condition})' |
| |
| return condition |
| |
| def __new__(cls, kind, condition): |
| kind = _coerce_str(kind) |
| self = super().__new__( |
| cls, |
| kind=kind or None, |
| condition=cls._condition_from_raw(condition, kind), |
| ) |
| return self |
| |
| @property |
| def text(self): |
| if self.kind == 'ifdef': |
| return self.condition[8:-1] # strip "defined(" |
| elif self.kind == 'ifndef': |
| return self.condition[10:-1] # strip "! defined(" |
| else: |
| return self.condition |
| #return str(self.condition) |
| |
| def validate(self): |
| """Fail if the object is invalid (i.e. init with bad data).""" |
| super().validate() |
| |
| if not self.condition: |
| raise TypeError('missing condition') |
| #else: |
| # for cond in self.condition: |
| # if not cond: |
| # raise ValueError(f'missing condition in {self.condition}') |
| # cond.validate() |
| # if self.kind in ('ifdef', 'ifndef'): |
| # if len(self.condition) != 1: |
| # raise ValueError('too many condition') |
| # if self.kind == 'ifdef': |
| # if not self.condition[0].startswith('defined '): |
| # raise ValueError('bad condition') |
| # else: |
| # if not self.condition[0].startswith('! defined '): |
| # raise ValueError('bad condition') |
| |
| |
| class Include(PreprocessorDirective, |
| namedtuple('Include', 'kind file')): |
| """A single "include" directive. |
| |
| Supported "file" values are either follow the bracket style |
| (<stdio>) or double quotes ("spam.h"). |
| """ |
| |
| __slots__ = () |
| |
| def __new__(cls, file): |
| self = super().__new__( |
| cls, |
| kind='include', |
| file=_coerce_str(file) or None, |
| ) |
| return self |
| |
| def validate(self): |
| """Fail if the object is invalid (i.e. init with bad data).""" |
| super().validate() |
| |
| if not self.file: |
| raise TypeError('missing file') |
| |
| |
| class OtherDirective(PreprocessorDirective, |
| namedtuple('OtherDirective', 'kind text')): |
| """A single directive not covered by another class. |
| |
| This includes the "else", "endif", and "undef" directives, which are |
| otherwise inherently related to the directives covered by the |
| Constant, Macro, and IfCondition classes. |
| |
| Note that all directives must have a text value, except for "else" |
| and "endif" (which must have no text). |
| """ |
| |
| __slots__ = () |
| |
| KINDS = PreprocessorDirective.KINDS - {'include', 'define'} - IfDirective.KINDS |
| |
| def __new__(cls, kind, text): |
| self = super().__new__( |
| cls, |
| kind=_coerce_str(kind) or None, |
| text=_coerce_str(text) or None, |
| ) |
| return self |
| |
| def validate(self): |
| """Fail if the object is invalid (i.e. init with bad data).""" |
| super().validate() |
| |
| if self.text: |
| if self.kind in ('else', 'endif'): |
| raise ValueError('unexpected text in directive') |
| elif self.kind not in ('else', 'endif'): |
| raise TypeError('missing text') |
| |
| |
| ############################# |
| # iterating lines |
| |
| def _recompute_conditions(directive, ifstack): |
| if directive.kind in ('if', 'ifdef', 'ifndef'): |
| ifstack.append( |
| ([], directive.condition)) |
| elif directive.kind == 'elseif': |
| if ifstack: |
| negated, active = ifstack.pop() |
| if active: |
| negated.append(active) |
| else: |
| negated = [] |
| ifstack.append( |
| (negated, directive.condition)) |
| elif directive.kind == 'else': |
| if ifstack: |
| negated, active = ifstack.pop() |
| if active: |
| negated.append(active) |
| ifstack.append( |
| (negated, None)) |
| elif directive.kind == 'endif': |
| if ifstack: |
| ifstack.pop() |
| |
| conditions = [] |
| for negated, active in ifstack: |
| for condition in negated: |
| conditions.append(f'! ({condition})') |
| if active: |
| conditions.append(active) |
| return tuple(conditions) |
| |
| |
| def _iter_clean_lines(lines): |
| lines = iter(enumerate(lines, 1)) |
| for lno, line in lines: |
| # Handle line continuations. |
| while line.endswith(CONTINUATION): |
| try: |
| lno, _line = next(lines) |
| except StopIteration: |
| break |
| line = line[:-len(CONTINUATION)] + ' ' + _line |
| |
| # Deal with comments. |
| after = line |
| line = '' |
| while True: |
| # Look for a comment. |
| before, begin, remainder = after.partition('/*') |
| if '//' in before: |
| before, _, _ = before.partition('//') |
| line += before + ' ' # per the C99 spec |
| break |
| line += before |
| if not begin: |
| break |
| line += ' ' # per the C99 spec |
| |
| # Go until we find the end of the comment. |
| _, end, after = remainder.partition('*/') |
| while not end: |
| try: |
| lno, remainder = next(lines) |
| except StopIteration: |
| raise Exception('unterminated comment') |
| _, end, after = remainder.partition('*/') |
| |
| yield lno, line |
| |
| |
| def iter_lines(lines, *, |
| _iter_clean_lines=_iter_clean_lines, |
| _parse_directive=_parse_directive, |
| _recompute_conditions=_recompute_conditions, |
| ): |
| """Yield (lno, line, directive, active conditions) for each given line. |
| |
| This is effectively a subset of the operations taking place in |
| translation phases 2-4 from the C99 spec (ISO/IEC 9899:TC2); see |
| section 5.1.1.2. Line continuations are removed and comments |
| replaced with a single space. (In both cases "lno" will be the last |
| line involved.) Otherwise each line is returned as-is. |
| |
| "lno" is the (1-indexed) line number for the line. |
| |
| "directive" will be a PreprocessorDirective or None, depending on |
| whether or not there is a directive on the line. |
| |
| "active conditions" is the set of preprocessor conditions (e.g. |
| "defined()") under which the current line of code will be included |
| in compilation. That set is derived from every conditional |
| directive block (e.g. "if defined()", "ifdef", "else") containing |
| that line. That includes nested directives. Note that the |
| current line does not affect the active conditions for iteself. |
| It only impacts subsequent lines. That applies to directives |
| that close blocks (e.g. "endif") just as much as conditional |
| directvies. Also note that "else" and "elseif" directives |
| update the active conditions (for later lines), rather than |
| adding to them. |
| """ |
| ifstack = [] |
| conditions = () |
| for lno, line in _iter_clean_lines(lines): |
| stripped = line.strip() |
| if not stripped.startswith('#'): |
| yield lno, line, None, conditions |
| continue |
| |
| directive = '#' + stripped[1:].lstrip() |
| while ' ' in directive: |
| directive = directive.replace(' ', ' ') |
| directive = _parse_directive(directive) |
| yield lno, line, directive, conditions |
| |
| if directive.kind in ('else', 'endif'): |
| conditions = _recompute_conditions(directive, ifstack) |
| elif isinstance(directive, IfDirective): |
| conditions = _recompute_conditions(directive, ifstack) |
| |
| |
| ############################# |
| # running (platform-specific?) |
| |
| def _gcc(filename, *, |
| _get_argv=(lambda: _get_gcc_argv()), |
| _run=util.run_cmd, |
| ): |
| argv = _get_argv() |
| argv.extend([ |
| '-E', filename, |
| ]) |
| output = _run(argv) |
| return output |
| |
| |
| def _get_gcc_argv(*, |
| _open=open, |
| _run=util.run_cmd, |
| ): |
| with _open('/tmp/print.mk', 'w') as tmpfile: |
| tmpfile.write('print-%:\n') |
| #tmpfile.write('\t@echo $* = $($*)\n') |
| tmpfile.write('\t@echo $($*)\n') |
| argv = ['/usr/bin/make', |
| '-f', 'Makefile', |
| '-f', '/tmp/print.mk', |
| 'print-CC', |
| 'print-PY_CORE_CFLAGS', |
| ] |
| output = _run(argv) |
| gcc, cflags = output.strip().splitlines() |
| argv = shlex.split(gcc.strip()) |
| cflags = shlex.split(cflags.strip()) |
| return argv + cflags |
| |
| |
| def run(filename, *, |
| _gcc=_gcc, |
| ): |
| """Return the text of the given file after running the preprocessor.""" |
| return _gcc(filename) |