diff --git a/src/devicetree/__init__.py b/src/devicetree/__init__.py index f8215df..e9a5683 100644 --- a/src/devicetree/__init__.py +++ b/src/devicetree/__init__.py @@ -1,2 +1,4 @@ -__all__ = ['edtlib', 'dtlib'] +# Copyright (c) 2021 Nordic Semiconductor ASA +# SPDX-License-Identifier: Apache-2.0 +__all__ = ['edtlib', 'dtlib'] diff --git a/src/devicetree/dtlib.py b/src/devicetree/dtlib.py index 37f7ade..dc337a6 100644 --- a/src/devicetree/dtlib.py +++ b/src/devicetree/dtlib.py @@ -1,11 +1,7 @@ # Copyright (c) 2019, Nordic Semiconductor # SPDX-License-Identifier: BSD-3-Clause -# Tip: You can view just the documentation with 'pydoc3 dtlib' - -# _init_tokens() builds names dynamically. -# -# pylint: disable=undefined-variable +# Tip: You can view just the documentation with 'pydoc3 devicetree.dtlib' """ A library for extracting information from .dts (devicetree) files. See the @@ -17,13 +13,660 @@ files. """ import collections +import enum import errno import os import re +import string import sys import textwrap +from typing import Any, Dict, Iterable, List, \ + NamedTuple, NoReturn, Optional, Tuple, Union -# NOTE: testdtlib.py is the test suite for this library. +# NOTE: tests/test_dtlib.py is the test suite for this library. + +class DTError(Exception): + "Exception raised for devicetree-related errors" + +class Node: + r""" + Represents a node in the devicetree ('node-name { ... };'). + + These attributes are available on Node instances: + + name: + The name of the node (a string). + + unit_addr: + The portion after the '@' in the node's name, or the empty string if the + name has no '@' in it. + + Note that this is a string. Run int(node.unit_addr, 16) to get an + integer. + + props: + A collections.OrderedDict that maps the properties defined on the node to + their values. 'props' is indexed by property name (a string), and values + are Property objects. + + To convert property values to Python numbers or strings, use + dtlib.to_num(), dtlib.to_nums(), or dtlib.to_string(). + + Property values are represented as 'bytes' arrays to support the full + generality of DTS, which allows assignments like + + x = "foo", < 0x12345678 >, [ 9A ]; + + This gives x the value b"foo\0\x12\x34\x56\x78\x9A". Numbers in DTS are + stored in big-endian format. + + nodes: + A collections.OrderedDict containing the subnodes of the node, indexed by + name. + + labels: + A list with all labels pointing to the node, in the same order as the + labels appear, but with duplicates removed. + + 'label_1: label_2: node { ... };' gives 'labels' the value + ["label_1", "label_2"]. + + parent: + The parent Node of the node. 'None' for the root node. + + path: + The path to the node as a string, e.g. "/foo/bar". + + dt: + The DT instance this node belongs to. + """ + + # + # Public interface + # + + def __init__(self, name: str, parent: Optional['Node'], dt: 'DT'): + """ + Node constructor. Not meant to be called directly by clients. + """ + self.name = name + self.parent = parent + self.dt = dt + + if name.count("@") > 1: + dt._parse_error("multiple '@' in node name") + if not name == "/": + for char in name: + if char not in _nodename_chars: + dt._parse_error(f"{self.path}: bad character '{char}' " + "in node name") + + self.props: Dict[str, 'Property'] = collections.OrderedDict() + self.nodes: Dict[str, 'Node'] = collections.OrderedDict() + self.labels: List[str] = [] + self._omit_if_no_ref = False + self._is_referenced = False + + @property + def unit_addr(self) -> str: + """ + See the class documentation. + """ + return self.name.partition("@")[2] + + @property + def path(self) -> str: + """ + See the class documentation. + """ + node_names = [] + + cur = self + while cur.parent: + node_names.append(cur.name) + cur = cur.parent + + return "/" + "/".join(reversed(node_names)) + + def node_iter(self) -> Iterable['Node']: + """ + Returns a generator for iterating over the node and its children, + recursively. + + For example, this will iterate over all nodes in the tree (like + dt.node_iter()). + + for node in dt.root.node_iter(): + ... + """ + yield self + for node in self.nodes.values(): + yield from node.node_iter() + + def _get_prop(self, name: str) -> 'Property': + # Returns the property named 'name' on the node, creating it if it + # doesn't already exist + + prop = self.props.get(name) + if not prop: + prop = Property(self, name) + self.props[name] = prop + return prop + + def _del(self) -> None: + # Removes the node from the tree + self.parent.nodes.pop(self.name) # type: ignore + + def __str__(self): + """ + Returns a DTS representation of the node. Called automatically if the + node is print()ed. + """ + s = "".join(label + ": " for label in self.labels) + + s += f"{self.name} {{\n" + + for prop in self.props.values(): + s += "\t" + str(prop) + "\n" + + for child in self.nodes.values(): + s += textwrap.indent(child.__str__(), "\t") + "\n" + + s += "};" + + return s + + def __repr__(self): + """ + Returns some information about the Node instance. Called automatically + if the Node instance is evaluated. + """ + return f"" + +# See Property.type +class Type(enum.IntEnum): + EMPTY = 0 + BYTES = 1 + NUM = 2 + NUMS = 3 + STRING = 4 + STRINGS = 5 + PATH = 6 + PHANDLE = 7 + PHANDLES = 8 + PHANDLES_AND_NUMS = 9 + COMPOUND = 10 + +class _MarkerType(enum.IntEnum): + # Types of markers in property values + + # References + PATH = 0 # &foo + PHANDLE = 1 # <&foo> + LABEL = 2 # foo: <1 2 3> + + # Start of data blocks of specific type + UINT8 = 3 # [00 01 02] (and also used for /incbin/) + UINT16 = 4 # /bits/ 16 <1 2 3> + UINT32 = 5 # <1 2 3> + UINT64 = 6 # /bits/ 64 <1 2 3> + STRING = 7 # "foo" + +class Property: + """ + Represents a property ('x = ...'). + + These attributes are available on Property instances: + + name: + The name of the property (a string). + + value: + The value of the property, as a 'bytes' string. Numbers are stored in + big-endian format, and strings are null-terminated. Putting multiple + comma-separated values in an assignment (e.g., 'x = < 1 >, "foo"') will + concatenate the values. + + See the to_*() methods for converting the value to other types. + + type: + The type of the property, inferred from the syntax used in the + assignment. This is one of the following constants (with example + assignments): + + Assignment | Property.type + ----------------------------+------------------------ + foo; | dtlib.Type.EMPTY + foo = []; | dtlib.Type.BYTES + foo = [01 02]; | dtlib.Type.BYTES + foo = /bits/ 8 <1>; | dtlib.Type.BYTES + foo = <1>; | dtlib.Type.NUM + foo = <>; | dtlib.Type.NUMS + foo = <1 2 3>; | dtlib.Type.NUMS + foo = <1 2>, <3>; | dtlib.Type.NUMS + foo = "foo"; | dtlib.Type.STRING + foo = "foo", "bar"; | dtlib.Type.STRINGS + foo = <&l>; | dtlib.Type.PHANDLE + foo = <&l1 &l2 &l3>; | dtlib.Type.PHANDLES + foo = <&l1 &l2>, <&l3>; | dtlib.Type.PHANDLES + foo = <&l1 1 2 &l2 3 4>; | dtlib.Type.PHANDLES_AND_NUMS + foo = <&l1 1 2>, <&l2 3 4>; | dtlib.Type.PHANDLES_AND_NUMS + foo = &l; | dtlib.Type.PATH + *Anything else* | dtlib.Type.COMPOUND + + *Anything else* includes properties mixing phandle (<&label>) and node + path (&label) references with other data. + + Data labels in the property value do not influence the type. + + labels: + A list with all labels pointing to the property, in the same order as the + labels appear, but with duplicates removed. + + 'label_1: label2: x = ...' gives 'labels' the value + {"label_1", "label_2"}. + + offset_labels: + A dictionary that maps any labels within the property's value to their + offset, in bytes. For example, 'x = < 0 label_1: 1 label_2: >' gives + 'offset_labels' the value {"label_1": 4, "label_2": 8}. + + Iteration order will match the order of the labels on Python versions + that preserve dict insertion order. + + node: + The Node the property is on. + """ + + # + # Public interface + # + + def __init__(self, node: Node, name: str): + if "@" in name: + node.dt._parse_error("'@' is only allowed in node names") + + self.name = name + self.node = node + self.value = b"" + self.labels: List[str] = [] + self._label_offset_lst: List[Tuple[str, int]] = [] + # We have to wait to set this until later, when we've got + # the entire tree. + self.offset_labels: Dict[str, int] = {} + + # A list of [offset, label, type] lists (sorted by offset), + # giving the locations of references within the value. 'type' + # is either _MarkerType.PATH, for a node path reference, + # _MarkerType.PHANDLE, for a phandle reference, or + # _MarkerType.LABEL, for a label on/within data. Node paths + # and phandles need to be patched in after parsing. + self._markers: List[List] = [] + + def to_num(self, signed=False) -> int: + """ + Returns the value of the property as a number. + + Raises DTError if the property was not assigned with this syntax (has + Property.type Type.NUM): + + foo = < 1 >; + + signed (default: False): + If True, the value will be interpreted as signed rather than + unsigned. + """ + if self.type is not Type.NUM: + _err("expected property '{0}' on {1} in {2} to be assigned with " + "'{0} = < (number) >;', not '{3}'" + .format(self.name, self.node.path, self.node.dt.filename, + self)) + + return int.from_bytes(self.value, "big", signed=signed) + + def to_nums(self, signed=False) -> List[int]: + """ + Returns the value of the property as a list of numbers. + + Raises DTError if the property was not assigned with this syntax (has + Property.type Type.NUM or Type.NUMS): + + foo = < 1 2 ... >; + + signed (default: False): + If True, the values will be interpreted as signed rather than + unsigned. + """ + if self.type not in (Type.NUM, Type.NUMS): + _err("expected property '{0}' on {1} in {2} to be assigned with " + "'{0} = < (number) (number) ... >;', not '{3}'" + .format(self.name, self.node.path, self.node.dt.filename, + self)) + + return [int.from_bytes(self.value[i:i + 4], "big", signed=signed) + for i in range(0, len(self.value), 4)] + + def to_bytes(self) -> bytes: + """ + Returns the value of the property as a raw 'bytes', like + Property.value, except with added type checking. + + Raises DTError if the property was not assigned with this syntax (has + Property.type Type.BYTES): + + foo = [ 01 ... ]; + """ + if self.type is not Type.BYTES: + _err("expected property '{0}' on {1} in {2} to be assigned with " + "'{0} = [ (byte) (byte) ... ];', not '{3}'" + .format(self.name, self.node.path, self.node.dt.filename, + self)) + + return self.value + + def to_string(self) -> str: + """ + Returns the value of the property as a string. + + Raises DTError if the property was not assigned with this syntax (has + Property.type Type.STRING): + + foo = "string"; + + This function might also raise UnicodeDecodeError if the string is + not valid UTF-8. + """ + if self.type is not Type.STRING: + _err("expected property '{0}' on {1} in {2} to be assigned with " + "'{0} = \"string\";', not '{3}'" + .format(self.name, self.node.path, self.node.dt.filename, + self)) + + try: + ret = self.value.decode("utf-8")[:-1] # Strip null + except UnicodeDecodeError: + _err(f"value of property '{self.name}' ({self.value!r}) " + f"on {self.node.path} in {self.node.dt.filename} " + "is not valid UTF-8") + + return ret # The separate 'return' appeases the type checker. + + def to_strings(self) -> List[str]: + """ + Returns the value of the property as a list of strings. + + Raises DTError if the property was not assigned with this syntax (has + Property.type Type.STRING or Type.STRINGS): + + foo = "string", "string", ... ; + + Also raises DTError if any of the strings are not valid UTF-8. + """ + if self.type not in (Type.STRING, Type.STRINGS): + _err("expected property '{0}' on {1} in {2} to be assigned with " + "'{0} = \"string\", \"string\", ... ;', not '{3}'" + .format(self.name, self.node.path, self.node.dt.filename, + self)) + + try: + ret = self.value.decode("utf-8").split("\0")[:-1] + except UnicodeDecodeError: + _err(f"value of property '{self.name}' ({self.value!r}) " + f"on {self.node.path} in {self.node.dt.filename} " + "is not valid UTF-8") + + return ret # The separate 'return' appeases the type checker. + + def to_node(self) -> Node: + """ + Returns the Node the phandle in the property points to. + + Raises DTError if the property was not assigned with this syntax (has + Property.type Type.PHANDLE). + + foo = < &bar >; + """ + if self.type is not Type.PHANDLE: + _err("expected property '{0}' on {1} in {2} to be assigned with " + "'{0} = < &foo >;', not '{3}'" + .format(self.name, self.node.path, self.node.dt.filename, + self)) + + return self.node.dt.phandle2node[int.from_bytes(self.value, "big")] + + def to_nodes(self) -> List[Node]: + """ + Returns a list with the Nodes the phandles in the property point to. + + Raises DTError if the property value contains anything other than + phandles. All of the following are accepted: + + foo = < > + foo = < &bar >; + foo = < &bar &baz ... >; + foo = < &bar ... >, < &baz ... >; + """ + def type_ok(): + if self.type in (Type.PHANDLE, Type.PHANDLES): + return True + # Also accept 'foo = < >;' + return self.type is Type.NUMS and not self.value + + if not type_ok(): + _err("expected property '{0}' on {1} in {2} to be assigned with " + "'{0} = < &foo &bar ... >;', not '{3}'" + .format(self.name, self.node.path, + self.node.dt.filename, self)) + + return [self.node.dt.phandle2node[int.from_bytes(self.value[i:i + 4], + "big")] + for i in range(0, len(self.value), 4)] + + def to_path(self) -> Node: + """ + Returns the Node referenced by the path stored in the property. + + Raises DTError if the property was not assigned with either of these + syntaxes (has Property.type Type.PATH or Type.STRING): + + foo = &bar; + foo = "/bar"; + + For the second case, DTError is raised if the path does not exist. + """ + if self.type not in (Type.PATH, Type.STRING): + _err("expected property '{0}' on {1} in {2} to be assigned with " + "either '{0} = &foo' or '{0} = \"/path/to/node\"', not '{3}'" + .format(self.name, self.node.path, self.node.dt.filename, + self)) + + try: + path = self.value.decode("utf-8")[:-1] + except UnicodeDecodeError: + _err(f"value of property '{self.name}' ({self.value!r}) " + f"on {self.node.path} in {self.node.dt.filename} " + "is not valid UTF-8") + + try: + ret = self.node.dt.get_node(path) + except DTError: + _err(f"property '{self.name}' on {self.node.path} in " + f"{self.node.dt.filename} points to the non-existent node " + f'"{path}"') + + return ret # The separate 'return' appeases the type checker. + + @property + def type(self) -> int: + """ + See the class docstring. + """ + # Data labels (e.g. 'foo = label: <3>') are irrelevant, so filter them + # out + types = [marker[1] for marker in self._markers + if marker[1] != _MarkerType.LABEL] + + if not types: + return Type.EMPTY + + if types == [_MarkerType.UINT8]: + return Type.BYTES + + if types == [_MarkerType.UINT32]: + return Type.NUM if len(self.value) == 4 else Type.NUMS + + # Treat 'foo = <1 2 3>, <4 5>, ...' as Type.NUMS too + if set(types) == {_MarkerType.UINT32}: + return Type.NUMS + + if set(types) == {_MarkerType.STRING}: + return Type.STRING if len(types) == 1 else Type.STRINGS + + if types == [_MarkerType.PATH]: + return Type.PATH + + if types == [_MarkerType.UINT32, _MarkerType.PHANDLE] and \ + len(self.value) == 4: + return Type.PHANDLE + + if set(types) == {_MarkerType.UINT32, _MarkerType.PHANDLE}: + if len(self.value) == 4*types.count(_MarkerType.PHANDLE): + # Array with just phandles in it + return Type.PHANDLES + # Array with both phandles and numbers + return Type.PHANDLES_AND_NUMS + + return Type.COMPOUND + + def __str__(self): + s = "".join(label + ": " for label in self.labels) + self.name + if not self.value: + return s + ";" + + s += " =" + + for i, (pos, marker_type, ref) in enumerate(self._markers): + if i < len(self._markers) - 1: + next_marker = self._markers[i + 1] + else: + next_marker = None + + # End of current marker + end = next_marker[0] if next_marker else len(self.value) + + if marker_type is _MarkerType.STRING: + # end - 1 to strip off the null terminator + s += f' "{_decode_and_escape(self.value[pos:end - 1])}"' + if end != len(self.value): + s += "," + elif marker_type is _MarkerType.PATH: + s += " &" + ref + if end != len(self.value): + s += "," + else: + # <> or [] + + if marker_type is _MarkerType.LABEL: + s += f" {ref}:" + elif marker_type is _MarkerType.PHANDLE: + s += " &" + ref + pos += 4 + # Subtle: There might be more data between the phandle and + # the next marker, so we can't 'continue' here + else: # marker_type is _MarkerType.UINT* + elm_size = _TYPE_TO_N_BYTES[marker_type] + s += _N_BYTES_TO_START_STR[elm_size] + + while pos != end: + num = int.from_bytes(self.value[pos:pos + elm_size], + "big") + if elm_size == 1: + s += f" {num:02X}" + else: + s += f" {hex(num)}" + + pos += elm_size + + if pos != 0 and \ + (not next_marker or + next_marker[1] not in (_MarkerType.PHANDLE, _MarkerType.LABEL)): + + s += _N_BYTES_TO_END_STR[elm_size] + if pos != len(self.value): + s += "," + + return s + ";" + + + def __repr__(self): + return f"" + + # + # Internal functions + # + + def _add_marker(self, marker_type: _MarkerType, data: Any = None): + # Helper for registering markers in the value that are processed after + # parsing. See _fixup_props(). 'marker_type' identifies the type of + # marker, and 'data' has any optional data associated with the marker. + + # len(self.value) gives the current offset. This function is called + # while the value is built. We use a list instead of a tuple to be able + # to fix up offsets later (they might increase if the value includes + # path references, e.g. 'foo = &bar, <3>;', which are expanded later). + self._markers.append([len(self.value), marker_type, data]) + + # For phandle references, add a dummy value with the same length as a + # phandle. This is handy for the length check in _register_phandles(). + if marker_type is _MarkerType.PHANDLE: + self.value += b"\0\0\0\0" + +class _T(enum.IntEnum): + # Token IDs used by the DT lexer. + + # These values must be contiguous and start from 1. + INCLUDE = 1 + LINE = 2 + STRING = 3 + DTS_V1 = 4 + PLUGIN = 5 + MEMRESERVE = 6 + BITS = 7 + DEL_PROP = 8 + DEL_NODE = 9 + OMIT_IF_NO_REF = 10 + LABEL = 11 + CHAR_LITERAL = 12 + REF = 13 + INCBIN = 14 + SKIP = 15 + EOF = 16 + + # These values must be larger than the above contiguous range. + NUM = 17 + PROPNODENAME = 18 + MISC = 19 + BYTE = 20 + BAD = 21 + +class _FileStackElt(NamedTuple): + # Used for maintaining the /include/ stack. + + filename: str + lineno: int + contents: str + pos: int + +_TokVal = Union[int, str] + +class _Token(NamedTuple): + id: int + val: _TokVal + + def __repr__(self): + id_repr = _T(self.id).name + return f'Token(id=_T.{id_repr}, val={repr(self.val)})' class DT: """ @@ -73,7 +716,8 @@ class DT: # Public interface # - def __init__(self, filename, include_path=()): + def __init__(self, filename: str, include_path: Iterable[str] = (), + force: bool = False): """ Parses a DTS file to create a DT instance. Raises OSError if 'filename' can't be opened, and DTError for any parse errors. @@ -85,22 +729,29 @@ class DT: An iterable (e.g. list or tuple) containing paths to search for /include/d and /incbin/'d files. By default, files are only looked up relative to the .dts file that contains the /include/ or /incbin/. + + force: + Try not to raise DTError even if the input tree has errors. + For experimental use; results not guaranteed. """ self.filename = filename - self._include_path = include_path + self._include_path = list(include_path) + self._force = force with open(filename, encoding="utf-8") as f: self._file_contents = f.read() self._tok_i = self._tok_end_i = 0 - self._filestack = [] + self._filestack: List[_FileStackElt] = [] - self.alias2node = {} + self.alias2node: Dict[str, Node] = {} - self._lexer_state = _DEFAULT - self._saved_token = None + self._lexer_state: int = _DEFAULT + self._saved_token: Optional[_Token] = None - self._lineno = 1 + self._lineno: int = 1 + + self._root: Optional[Node] = None self._parse_dt() @@ -110,7 +761,17 @@ class DT: self._remove_unreferenced() self._register_labels() - def get_node(self, path): + @property + def root(self) -> Node: + """ + See the class documentation. + """ + # This is necessary because mypy can't tell that we never + # treat self._root as a non-None value until it's initialized + # properly in _parse_dt(). + return self._root # type: ignore + + def get_node(self, path: str) -> Node: """ Returns the Node instance for the node with path or alias 'path' (a string). Raises DTError if the path or alias doesn't exist. @@ -142,12 +803,12 @@ class DT: # Path does not start with '/'. First component must be an alias. alias, _, rest = path.partition("/") if alias not in self.alias2node: - _err("no alias '{}' found -- did you forget the leading '/' in " - "the node path?".format(alias)) + _err(f"no alias '{alias}' found -- did you forget the leading " + "'/' in the node path?") return _root_and_path_to_node(self.alias2node[alias], rest, path) - def has_node(self, path): + def has_node(self, path: str) -> bool: """ Returns True if the path or alias 'path' exists. See Node.get_node(). """ @@ -157,7 +818,7 @@ class DT: except DTError: return False - def node_iter(self): + def node_iter(self) -> Iterable[Node]: """ Returns a generator for iterating over all nodes in the devicetree. @@ -181,9 +842,8 @@ class DT: for labels, address, offset in self.memreserves: # List the labels in a consistent order to help with testing for label in labels: - s += label + ": " - s += "/memreserve/ {:#018x} {:#018x};\n" \ - .format(address, offset) + s += f"{label}: " + s += f"/memreserve/ {address:#018x} {offset:#018x};\n" s += "\n" return s + str(self.root) @@ -193,8 +853,8 @@ class DT: Returns some information about the DT instance. Called automatically if the DT instance is evaluated. """ - return "DT(filename='{}', include_path={})" \ - .format(self.filename, self._include_path) + return f"DT(filename='{self.filename}', " \ + f"include_path={self._include_path})" # # Parsing @@ -206,25 +866,23 @@ class DT: self._parse_header() self._parse_memreserves() - self.root = None - while True: tok = self._next_token() if tok.val == "/": # '/ { ... };', the root node - if not self.root: - self.root = Node(name="/", parent=None, dt=self) + if not self._root: + self._root = Node(name="/", parent=None, dt=self) self._parse_node(self.root) - elif tok.id in (_T_LABEL, _T_REF): + elif tok.id in (_T.LABEL, _T.REF): # '&foo { ... };' or 'label: &foo { ... };'. The C tools only # support a single label here too. - if tok.id is _T_LABEL: + if tok.id == _T.LABEL: label = tok.val tok = self._next_token() - if tok.id is not _T_REF: + if tok.id != _T.REF: self._parse_error("expected label reference (&foo)") else: label = None @@ -238,16 +896,16 @@ class DT: if label: _append_no_dup(node.labels, label) - elif tok.id is _T_DEL_NODE: + elif tok.id == _T.DEL_NODE: self._next_ref2node()._del() self._expect_token(";") - elif tok.id is _T_OMIT_IF_NO_REF: + elif tok.id == _T.OMIT_IF_NO_REF: self._next_ref2node()._omit_if_no_ref = True self._expect_token(";") - elif tok.id is _T_EOF: - if not self.root: + elif tok.id == _T.EOF: + if not self._root: self._parse_error("no root node defined") return @@ -260,12 +918,12 @@ class DT: has_dts_v1 = False - while self._peek_token().id is _T_DTS_V1: + while self._peek_token().id == _T.DTS_V1: has_dts_v1 = True self._next_token() self._expect_token(";") # /plugin/ always comes after /dts-v1/ - if self._peek_token().id is _T_PLUGIN: + if self._peek_token().id == _T.PLUGIN: self._parse_error("/plugin/ is not supported") if not has_dts_v1: @@ -278,10 +936,10 @@ class DT: while True: # Labels before /memreserve/ labels = [] - while self._peek_token().id is _T_LABEL: + while self._peek_token().id == _T.LABEL: _append_no_dup(labels, self._next_token().val) - if self._peek_token().id is _T_MEMRESERVE: + if self._peek_token().id == _T.MEMRESERVE: self._next_token() self.memreserves.append( (labels, self._eval_prim(), self._eval_prim())) @@ -301,13 +959,10 @@ class DT: labels, omit_if_no_ref = self._parse_propnode_labels() tok = self._next_token() - if tok.id is _T_PROPNODENAME: + if tok.id == _T.PROPNODENAME: if self._peek_token().val == "{": # ' { ...', expect node - if tok.val.count("@") > 1: - self._parse_error("multiple '@' in node name") - # Fetch the existing node if it already exists. This # happens when overriding nodes. child = node.nodes.get(tok.val) or \ @@ -340,17 +995,17 @@ class DT: for label in labels: _append_no_dup(prop.labels, label) - elif tok.id is _T_DEL_NODE: + elif tok.id == _T.DEL_NODE: tok2 = self._next_token() - if tok2.id is not _T_PROPNODENAME: + if tok2.id != _T.PROPNODENAME: self._parse_error("expected node name") if tok2.val in node.nodes: node.nodes[tok2.val]._del() self._expect_token(";") - elif tok.id is _T_DEL_PROP: + elif tok.id == _T.DEL_PROP: tok2 = self._next_token() - if tok2.id is not _T_PROPNODENAME: + if tok2.id != _T.PROPNODENAME: self._parse_error("expected property name") node.props.pop(tok2.val, None) self._expect_token(";") @@ -371,11 +1026,11 @@ class DT: omit_if_no_ref = False while True: tok = self._peek_token() - if tok.id is _T_LABEL: + if tok.id == _T.LABEL: _append_no_dup(labels, tok.val) - elif tok.id is _T_OMIT_IF_NO_REF: + elif tok.id == _T.OMIT_IF_NO_REF: omit_if_no_ref = True - elif (labels or omit_if_no_ref) and tok.id is not _T_PROPNODENAME: + elif (labels or omit_if_no_ref) and tok.id != _T.PROPNODENAME: # Got something like 'foo: bar: }' self._parse_error("expected node or property name") else: @@ -403,7 +1058,7 @@ class DT: if tok.val == "<": self._parse_cells(prop, 4) - elif tok.id is _T_BITS: + elif tok.id == _T.BITS: n_bits = self._expect_num() if n_bits not in {8, 16, 32, 64}: self._parse_error("expected 8, 16, 32, or 64") @@ -413,14 +1068,14 @@ class DT: elif tok.val == "[": self._parse_bytes(prop) - elif tok.id is _T_STRING: - prop._add_marker(_TYPE_STRING) + elif tok.id == _T.STRING: + prop._add_marker(_MarkerType.STRING) prop.value += self._unescape(tok.val.encode("utf-8")) + b"\0" - elif tok.id is _T_REF: - prop._add_marker(_REF_PATH, tok.val) + elif tok.id == _T.REF: + prop._add_marker(_MarkerType.PATH, tok.val) - elif tok.id is _T_INCBIN: + elif tok.id == _T.INCBIN: self._parse_incbin(prop) else: @@ -443,15 +1098,15 @@ class DT: while True: tok = self._peek_token() - if tok.id is _T_REF: + if tok.id == _T.REF: self._next_token() if n_bytes != 4: self._parse_error("phandle references are only allowed in " "arrays with 32-bit elements") - prop._add_marker(_REF_PHANDLE, tok.val) + prop._add_marker(_MarkerType.PHANDLE, tok.val) - elif tok.id is _T_LABEL: - prop._add_marker(_REF_LABEL, tok.val) + elif tok.id == _T.LABEL: + prop._add_marker(_MarkerType.LABEL, tok.val) self._next_token() elif self._check_token(">"): @@ -467,21 +1122,21 @@ class DT: # Try again as a signed number, in case it's negative prop.value += num.to_bytes(n_bytes, "big", signed=True) except OverflowError: - self._parse_error("{} does not fit in {} bits" - .format(num, 8*n_bytes)) + self._parse_error( + f"{num} does not fit in {8*n_bytes} bits") def _parse_bytes(self, prop): # Parses '[ ... ]' - prop._add_marker(_TYPE_UINT8) + prop._add_marker(_MarkerType.UINT8) while True: tok = self._next_token() - if tok.id is _T_BYTE: + if tok.id == _T.BYTE: prop.value += tok.val.to_bytes(1, "big") - elif tok.id is _T_LABEL: - prop._add_marker(_REF_LABEL, tok.val) + elif tok.id == _T.LABEL: + prop._add_marker(_MarkerType.LABEL, tok.val) elif tok.val == "]": return @@ -498,12 +1153,12 @@ class DT: # # /incbin/ ("filename", , ) - prop._add_marker(_TYPE_UINT8) + prop._add_marker(_MarkerType.UINT8) self._expect_token("(") tok = self._next_token() - if tok.id is not _T_STRING: + if tok.id != _T.STRING: self._parse_error("expected quoted filename") filename = tok.val @@ -526,8 +1181,7 @@ class DT: f.seek(offset) prop.value += f.read(size) except OSError as e: - self._parse_error("could not read '{}': {}" - .format(filename, e)) + self._parse_error(f"could not read '{filename}': {e}") def _parse_value_labels(self, prop): # _parse_assignment() helper for parsing labels before/after each @@ -535,9 +1189,9 @@ class DT: while True: tok = self._peek_token() - if tok.id is not _T_LABEL: + if tok.id != _T.LABEL: return - prop._add_marker(_REF_LABEL, tok.val) + prop._add_marker(_MarkerType.LABEL, tok.val) self._next_token() def _node_phandle(self, node): @@ -551,7 +1205,7 @@ class DT: phandle_prop = node.props["phandle"] else: phandle_prop = Property(node, "phandle") - phandle_prop._add_marker(_TYPE_UINT32) # For displaying + phandle_prop._add_marker(_MarkerType.UINT32) # For displaying phandle_prop.value = b'\0\0\0\0' if phandle_prop.value == b'\0\0\0\0': @@ -569,7 +1223,7 @@ class DT: def _eval_prim(self): tok = self._peek_token() - if tok.id in (_T_NUM, _T_CHAR_LITERAL): + if tok.id in (_T.NUM, _T.CHAR_LITERAL): return self._next_token().val tok = self._next_token() @@ -716,7 +1370,7 @@ class DT: match = _token_re.match(self._file_contents, self._tok_end_i) if match: tok_id = match.lastindex - if tok_id is _T_CHAR_LITERAL: + if tok_id == _T.CHAR_LITERAL: val = self._unescape(match.group(tok_id).encode("utf-8")) if len(val) != 1: self._parse_error("character literals must be length 1") @@ -727,7 +1381,7 @@ class DT: elif self._lexer_state is _DEFAULT: match = _num_re.match(self._file_contents, self._tok_end_i) if match: - tok_id = _T_NUM + tok_id = _T.NUM num_s = match.group(1) tok_val = int(num_s, 16 if num_s.startswith(("0x", "0X")) else @@ -738,21 +1392,20 @@ class DT: match = _propnodename_re.match(self._file_contents, self._tok_end_i) if match: - tok_id = _T_PROPNODENAME + tok_id = _T.PROPNODENAME tok_val = match.group(1) self._lexer_state = _DEFAULT else: # self._lexer_state is _EXPECT_BYTE match = _byte_re.match(self._file_contents, self._tok_end_i) if match: - tok_id = _T_BYTE + tok_id = _T.BYTE tok_val = int(match.group(), 16) - if not tok_id: match = _misc_re.match(self._file_contents, self._tok_end_i) if match: - tok_id = _T_MISC + tok_id = _T.MISC tok_val = match.group() else: self._tok_i = self._tok_end_i @@ -761,18 +1414,18 @@ class DT: # files. Generate a token for it so that the error can # trickle up to some context where we can give a more # helpful error message. - return _Token(_T_BAD, "") + return _Token(_T.BAD, "") self._tok_i = match.start() self._tok_end_i = match.end() - if tok_id is _T_SKIP: + if tok_id == _T.SKIP: self._lineno += tok_val.count("\n") continue # /include/ is handled in the lexer in the C tools as well, and can # appear anywhere - if tok_id is _T_INCLUDE: + if tok_id == _T.INCLUDE: # Can have newlines between /include/ and the filename self._lineno += tok_val.count("\n") # Do this manual extraction instead of doing it in the regex so @@ -781,21 +1434,21 @@ class DT: self._enter_file(filename) continue - if tok_id is _T_LINE: + if tok_id == _T.LINE: # #line directive self._lineno = int(tok_val.split()[0]) - 1 self.filename = tok_val[tok_val.find('"') + 1:-1] continue - if tok_id is _T_EOF: + if tok_id == _T.EOF: if self._filestack: self._leave_file() continue - return _Token(_T_EOF, "") + return _Token(_T.EOF, "") # State handling - if tok_id in (_T_DEL_PROP, _T_DEL_NODE, _T_OMIT_IF_NO_REF) or \ + if tok_id in (_T.DEL_PROP, _T.DEL_NODE, _T.OMIT_IF_NO_REF) or \ tok_val in ("{", ";"): self._lexer_state = _EXPECT_PROPNODENAME @@ -803,7 +1456,7 @@ class DT: elif tok_val == "[": self._lexer_state = _EXPECT_BYTE - elif tok_id in (_T_MEMRESERVE, _T_BITS) or tok_val == "]": + elif tok_id in (_T.MEMRESERVE, _T.BITS) or tok_val == "]": self._lexer_state = _DEFAULT return _Token(tok_id, tok_val) @@ -814,8 +1467,7 @@ class DT: tok = self._next_token() if tok.val != tok_val: - self._parse_error("expected '{}', not '{}'" - .format(tok_val, tok.val)) + self._parse_error(f"expected '{tok_val}', not '{tok.val}'") return tok @@ -823,24 +1475,25 @@ class DT: # Raises an error if the next token is not a number. Returns the token. tok = self._next_token() - if tok.id is not _T_NUM: + if tok.id != _T.NUM: self._parse_error("expected number") return tok.val def _parse_error(self, s): - _err("{}:{} (column {}): parse error: {}".format( - self.filename, self._lineno, - # This works out for the first line of the file too, where rfind() - # returns -1 - self._tok_i - self._file_contents.rfind("\n", 0, self._tok_i + 1), - s)) + # This works out for the first line of the file too, where rfind() + # returns -1 + column = self._tok_i - self._file_contents.rfind("\n", 0, + self._tok_i + 1) + _err(f"{self.filename}:{self._lineno} (column {column}): " + f"parse error: {s}") def _enter_file(self, filename): # Enters the /include/d file 'filename', remembering the position in # the /include/ing file for later - self._filestack.append((self.filename, self._lineno, - self._file_contents, self._tok_end_i)) + self._filestack.append( + _FileStackElt(self.filename, self._lineno, + self._file_contents, self._tok_end_i)) # Handle escapes in filenames, just for completeness filename = self._unescape(filename.encode("utf-8")) @@ -859,8 +1512,8 @@ class DT: for i, parent in enumerate(self._filestack): if filename == parent[0]: self._parse_error("recursive /include/:\n" + " ->\n".join( - ["{}:{}".format(parent[0], parent[1]) - for parent in self._filestack[i:]] + + [f"{parent[0]}:{parent[1]}" + for parent in self._filestack[i:]] + [filename])) self.filename = f.name @@ -879,7 +1532,7 @@ class DT: # on errors to save some code in callers. label = self._next_token() - if label.id is not _T_REF: + if label.id != _T.REF: self._parse_error( "expected label (&foo) or path (&{/foo/bar}) reference") try: @@ -894,7 +1547,7 @@ class DT: # Path reference (&{/foo/bar}) path = s[1:-1] if not path.startswith("/"): - _err("node path '{}' does not start with '/'".format(path)) + _err(f"node path '{path}' does not start with '/'") # Will raise DTError if the path doesn't exist return _root_and_path_to_node(self.root, path, path) @@ -906,7 +1559,7 @@ class DT: if s in node.labels: return node - _err("undefined node label '{}'".format(s)) + _err(f"undefined node label '{s}'") # # Post-processing @@ -923,13 +1576,13 @@ class DT: phandle = node.props.get("phandle") if phandle: if len(phandle.value) != 4: - _err("{}: bad phandle length ({}), expected 4 bytes" - .format(node.path, len(phandle.value))) + _err(f"{node.path}: bad phandle length " + f"({len(phandle.value)}), expected 4 bytes") is_self_referential = False for marker in phandle._markers: _, marker_type, ref = marker - if marker_type is _REF_PHANDLE: + if marker_type is _MarkerType.PHANDLE: # The phandle's value is itself a phandle reference if self._ref2node(ref) is node: # Alright to set a node's phandle equal to its own @@ -939,8 +1592,8 @@ class DT: is_self_referential = True break - _err("{}: {} refers to another node" - .format(node.path, phandle.name)) + _err(f"{node.path}: {phandle.name} " + "refers to another node") # Could put on else on the 'for' above too, but keep it # somewhat readable @@ -948,13 +1601,13 @@ class DT: phandle_val = int.from_bytes(phandle.value, "big") if phandle_val in {0, 0xFFFFFFFF}: - _err("{}: bad value {:#010x} for {}" - .format(node.path, phandle_val, phandle.name)) + _err(f"{node.path}: bad value {phandle_val:#010x} " + f"for {phandle.name}") if phandle_val in self.phandle2node: - _err("{}: duplicated phandle {:#x} (seen before at {})" - .format(node.path, phandle_val, - self.phandle2node[phandle_val].path)) + _err(f"{node.path}: duplicated phandle {phandle_val:#x} " + "(seen before at " + f"{self.phandle2node[phandle_val].path})") self.phandle2node[phandle_val] = node @@ -986,23 +1639,23 @@ class DT: # references, which expand to something like "/foo/bar". marker[0] = len(res) - if marker_type is _REF_LABEL: + if marker_type is _MarkerType.LABEL: # This is a temporary format so that we can catch - # duplicate references. prop.offset_labels is changed + # duplicate references. prop._label_offset_lst is changed # to a dictionary that maps labels to offsets in # _register_labels(). - _append_no_dup(prop.offset_labels, (ref, len(res))) - elif marker_type in (_REF_PATH, _REF_PHANDLE): + _append_no_dup(prop._label_offset_lst, (ref, len(res))) + elif marker_type in (_MarkerType.PATH, _MarkerType.PHANDLE): # Path or phandle reference try: ref_node = self._ref2node(ref) except DTError as e: - _err("{}: {}".format(prop.node.path, e)) + _err(f"{prop.node.path}: {e}") # For /omit-if-no-ref/ ref_node._is_referenced = True - if marker_type is _REF_PATH: + if marker_type is _MarkerType.PATH: res += ref_node.path.encode("utf-8") + b'\0' else: # marker_type is PHANDLE res += self._node_phandle(ref_node) @@ -1029,11 +1682,18 @@ class DT: if aliases: for prop in aliases.props.values(): if not alias_re.match(prop.name): - _err("/aliases: alias property name '{}' should include " - "only characters from [0-9a-z-]".format(prop.name)) + _err(f"/aliases: alias property name '{prop.name}' " + "should include only characters from [0-9a-z-]") - # Property.to_path() already checks that the node exists - alias2node[prop.name] = prop.to_path() + # Property.to_path() checks that the node exists, has + # the right type, etc. Swallow errors for invalid + # aliases with self._force. + try: + alias2node[prop.name] = prop.to_path() + except DTError: + if self._force: + continue + raise self.alias2node = alias2node @@ -1068,34 +1728,32 @@ class DT: label2things[label].add(prop) self.label2prop[label] = prop - for label, offset in prop.offset_labels: + for label, offset in prop._label_offset_lst: label2things[label].add((prop, offset)) self.label2prop_offset[label] = (prop, offset) # See _fixup_props() - prop.offset_labels = {label: offset for label, offset in - prop.offset_labels} + prop.offset_labels = dict(prop._label_offset_lst) for label, things in label2things.items(): if len(things) > 1: strings = [] for thing in things: if isinstance(thing, Node): - strings.append("on " + thing.path) + strings.append(f"on {thing.path}") elif isinstance(thing, Property): - strings.append("on property '{}' of node {}" - .format(thing.name, thing.node.path)) + strings.append(f"on property '{thing.name}' " + f"of node {thing.node.path}") else: # Label within property value - strings.append("in the value of property '{}' of node {}" - .format(thing[0].name, - thing[0].node.path)) + strings.append("in the value of property " + f"'{thing[0].name}' of node " + f"{thing[0].node.path}") # Give consistent error messages to help with testing strings.sort() - _err("Label '{}' appears ".format(label) + - " and ".join(strings)) + _err(f"Label '{label}' appears " + " and ".join(strings)) # @@ -1159,566 +1817,14 @@ class DT: self._parse_error(e) continue - self._parse_error("'{}' could not be found".format(filename)) - - -class Node: - r""" - Represents a node in the devicetree ('node-name { ... };'). - - These attributes are available on Node instances: - - name: - The name of the node (a string). - - unit_addr: - The portion after the '@' in the node's name, or the empty string if the - name has no '@' in it. - - Note that this is a string. Run int(node.unit_addr, 16) to get an - integer. - - props: - A collections.OrderedDict that maps the properties defined on the node to - their values. 'props' is indexed by property name (a string), and values - are represented as 'bytes' arrays. - - To convert property values to Python numbers or strings, use - dtlib.to_num(), dtlib.to_nums(), or dtlib.to_string(). - - Property values are represented as 'bytes' arrays to support the full - generality of DTS, which allows assignments like - - x = "foo", < 0x12345678 >, [ 9A ]; - - This gives x the value b"foo\0\x12\x34\x56\x78\x9A". Numbers in DTS are - stored in big-endian format. - - nodes: - A collections.OrderedDict containing the subnodes of the node, indexed by - name. - - labels: - A list with all labels pointing to the node, in the same order as the - labels appear, but with duplicates removed. - - 'label_1: label_2: node { ... };' gives 'labels' the value - ["label_1", "label_2"]. - - parent: - The parent Node of the node. 'None' for the root node. - - path: - The path to the node as a string, e.g. "/foo/bar". - - dt: - The DT instance this node belongs to. - """ - - # - # Public interface - # - - def __init__(self, name, parent, dt): - """ - Node constructor. Not meant to be called directly by clients. - """ - self.name = name - self.parent = parent - self.dt = dt - - self.props = collections.OrderedDict() - self.nodes = collections.OrderedDict() - self.labels = [] - self._omit_if_no_ref = False - self._is_referenced = False - - @property - def unit_addr(self): - """ - See the class documentation. - """ - return self.name.partition("@")[2] - - @property - def path(self): - """ - See the class documentation. - """ - node_names = [] - - cur = self - while cur.parent: - node_names.append(cur.name) - cur = cur.parent - - return "/" + "/".join(reversed(node_names)) - - def node_iter(self): - """ - Returns a generator for iterating over the node and its children, - recursively. - - For example, this will iterate over all nodes in the tree (like - dt.node_iter()). - - for node in dt.root.node_iter(): - ... - """ - yield self - for node in self.nodes.values(): - yield from node.node_iter() - - def _get_prop(self, name): - # Returns the property named 'name' on the node, creating it if it - # doesn't already exist - - prop = self.props.get(name) - if not prop: - prop = Property(self, name) - self.props[name] = prop - return prop - - def _del(self): - # Removes the node from the tree - - self.parent.nodes.pop(self.name) - - def __str__(self): - """ - Returns a DTS representation of the node. Called automatically if the - node is print()ed. - """ - s = "".join(label + ": " for label in self.labels) - - s += "{} {{\n".format(self.name) - - for prop in self.props.values(): - s += "\t" + str(prop) + "\n" - - for child in self.nodes.values(): - s += textwrap.indent(child.__str__(), "\t") + "\n" - - s += "};" - - return s - - def __repr__(self): - """ - Returns some information about the Node instance. Called automatically - if the Node instance is evaluated. - """ - return "" \ - .format(self.path, self.dt.filename) - - -class Property: - """ - Represents a property ('x = ...'). - - These attributes are available on Property instances: - - name: - The name of the property (a string). - - value: - The value of the property, as a 'bytes' string. Numbers are stored in - big-endian format, and strings are null-terminated. Putting multiple - comma-separated values in an assignment (e.g., 'x = < 1 >, "foo"') will - concatenate the values. - - See the to_*() methods for converting the value to other types. - - type: - The type of the property, inferred from the syntax used in the - assignment. This is one of the following constants (with example - assignments): - - Assignment | Property.type - ----------------------------+------------------------ - foo; | dtlib.TYPE_EMPTY - foo = []; | dtlib.TYPE_BYTES - foo = [01 02]; | dtlib.TYPE_BYTES - foo = /bits/ 8 <1>; | dtlib.TYPE_BYTES - foo = <1>; | dtlib.TYPE_NUM - foo = <>; | dtlib.TYPE_NUMS - foo = <1 2 3>; | dtlib.TYPE_NUMS - foo = <1 2>, <3>; | dtlib.TYPE_NUMS - foo = "foo"; | dtlib.TYPE_STRING - foo = "foo", "bar"; | dtlib.TYPE_STRINGS - foo = <&l>; | dtlib.TYPE_PHANDLE - foo = <&l1 &l2 &l3>; | dtlib.TYPE_PHANDLES - foo = <&l1 &l2>, <&l3>; | dtlib.TYPE_PHANDLES - foo = <&l1 1 2 &l2 3 4>; | dtlib.TYPE_PHANDLES_AND_NUMS - foo = <&l1 1 2>, <&l2 3 4>; | dtlib.TYPE_PHANDLES_AND_NUMS - foo = &l; | dtlib.TYPE_PATH - *Anything else* | dtlib.TYPE_COMPOUND - - *Anything else* includes properties mixing phandle (<&label>) and node - path (&label) references with other data. - - Data labels in the property value do not influence the type. - - labels: - A list with all labels pointing to the property, in the same order as the - labels appear, but with duplicates removed. - - 'label_1: label2: x = ...' gives 'labels' the value - {"label_1", "label_2"}. - - offset_labels: - A dictionary that maps any labels within the property's value to their - offset, in bytes. For example, 'x = < 0 label_1: 1 label_2: >' gives - 'offset_labels' the value {"label_1": 4, "label_2": 8}. - - Iteration order will match the order of the labels on Python versions - that preserve dict insertion order. - - node: - The Node the property is on. - """ - - # - # Public interface - # - - def __init__(self, node, name): - if "@" in name: - node.dt._parse_error("'@' is only allowed in node names") - - self.name = name - self.node = node - self.value = b"" - self.labels = [] - self.offset_labels = [] - - # A list of (offset, label, type) tuples (sorted by offset), giving the - # locations of references within the value. 'type' is either _REF_PATH, - # for a node path reference, _REF_PHANDLE, for a phandle reference, or - # _REF_LABEL, for a label on/within data. Node paths and phandles need - # to be patched in after parsing. - self._markers = [] - - def to_num(self, signed=False): - """ - Returns the value of the property as a number. - - Raises DTError if the property was not assigned with this syntax (has - Property.type TYPE_NUM): - - foo = < 1 >; - - signed (default: False): - If True, the value will be interpreted as signed rather than - unsigned. - """ - if self.type is not TYPE_NUM: - _err("expected property '{0}' on {1} in {2} to be assigned with " - "'{0} = < (number) >;', not '{3}'" - .format(self.name, self.node.path, self.node.dt.filename, - self)) - - return int.from_bytes(self.value, "big", signed=signed) - - def to_nums(self, signed=False): - """ - Returns the value of the property as a list of numbers. - - Raises DTError if the property was not assigned with this syntax (has - Property.type TYPE_NUM or TYPE_NUMS): - - foo = < 1 2 ... >; - - signed (default: False): - If True, the values will be interpreted as signed rather than - unsigned. - """ - if self.type not in (TYPE_NUM, TYPE_NUMS): - _err("expected property '{0}' on {1} in {2} to be assigned with " - "'{0} = < (number) (number) ... >;', not '{3}'" - .format(self.name, self.node.path, self.node.dt.filename, - self)) - - return [int.from_bytes(self.value[i:i + 4], "big", signed=signed) - for i in range(0, len(self.value), 4)] - - def to_bytes(self): - """ - Returns the value of the property as a raw 'bytes', like - Property.value, except with added type checking. - - Raises DTError if the property was not assigned with this syntax (has - Property.type TYPE_BYTES): - - foo = [ 01 ... ]; - """ - if self.type is not TYPE_BYTES: - _err("expected property '{0}' on {1} in {2} to be assigned with " - "'{0} = [ (byte) (byte) ... ];', not '{3}'" - .format(self.name, self.node.path, self.node.dt.filename, - self)) - - return self.value - - def to_string(self): - """ - Returns the value of the property as a string. - - Raises DTError if the property was not assigned with this syntax (has - Property.type TYPE_STRING): - - foo = "string"; - - This function might also raise UnicodeDecodeError if the string is - not valid UTF-8. - """ - if self.type is not TYPE_STRING: - _err("expected property '{0}' on {1} in {2} to be assigned with " - "'{0} = \"string\";', not '{3}'" - .format(self.name, self.node.path, self.node.dt.filename, - self)) - - try: - return self.value.decode("utf-8")[:-1] # Strip null - except UnicodeDecodeError: - _err("value of property '{}' ({}) on {} in {} is not valid UTF-8" - .format(self.name, self.value, self.node.path, - self.node.dt.filename)) - - def to_strings(self): - """ - Returns the value of the property as a list of strings. - - Raises DTError if the property was not assigned with this syntax (has - Property.type TYPE_STRING or TYPE_STRINGS): - - foo = "string", "string", ... ; - - Also raises DTError if any of the strings are not valid UTF-8. - """ - if self.type not in (TYPE_STRING, TYPE_STRINGS): - _err("expected property '{0}' on {1} in {2} to be assigned with " - "'{0} = \"string\", \"string\", ... ;', not '{3}'" - .format(self.name, self.node.path, self.node.dt.filename, - self)) - - try: - return self.value.decode("utf-8").split("\0")[:-1] - except UnicodeDecodeError: - _err("value of property '{}' ({}) on {} in {} is not valid UTF-8" - .format(self.name, self.value, self.node.path, - self.node.dt.filename)) - - def to_node(self): - """ - Returns the Node the phandle in the property points to. - - Raises DTError if the property was not assigned with this syntax (has - Property.type TYPE_PHANDLE). - - foo = < &bar >; - """ - if self.type is not TYPE_PHANDLE: - _err("expected property '{0}' on {1} in {2} to be assigned with " - "'{0} = < &foo >;', not '{3}'" - .format(self.name, self.node.path, self.node.dt.filename, - self)) - - return self.node.dt.phandle2node[int.from_bytes(self.value, "big")] - - def to_nodes(self): - """ - Returns a list with the Nodes the phandles in the property point to. - - Raises DTError if the property value contains anything other than - phandles. All of the following are accepted: - - foo = < > - foo = < &bar >; - foo = < &bar &baz ... >; - foo = < &bar ... >, < &baz ... >; - """ - def type_ok(): - if self.type in (TYPE_PHANDLE, TYPE_PHANDLES): - return True - # Also accept 'foo = < >;' - return self.type is TYPE_NUMS and not self.value - - if not type_ok(): - _err("expected property '{0}' on {1} in {2} to be assigned with " - "'{0} = < &foo &bar ... >;', not '{3}'" - .format(self.name, self.node.path, - self.node.dt.filename, self)) - - return [self.node.dt.phandle2node[int.from_bytes(self.value[i:i + 4], - "big")] - for i in range(0, len(self.value), 4)] - - def to_path(self): - """ - Returns the Node referenced by the path stored in the property. - - Raises DTError if the property was not assigned with either of these - syntaxes (has Property.type TYPE_PATH or TYPE_STRING): - - foo = &bar; - foo = "/bar"; - - For the second case, DTError is raised if the path does not exist. - """ - if self.type not in (TYPE_PATH, TYPE_STRING): - _err("expected property '{0}' on {1} in {2} to be assigned with " - "either '{0} = &foo' or '{0} = \"/path/to/node\"', not '{3}'" - .format(self.name, self.node.path, self.node.dt.filename, - self)) - - try: - path = self.value.decode("utf-8")[:-1] - except UnicodeDecodeError: - _err("value of property '{}' ({}) on {} in {} is not valid UTF-8" - .format(self.name, self.value, self.node.path, - self.node.dt.filename)) - - try: - return self.node.dt.get_node(path) - except DTError: - _err("property '{}' on {} in {} points to the non-existent node " - "\"{}\"".format(self.name, self.node.path, - self.node.dt.filename, path)) - - @property - def type(self): - """ - See the class docstring. - """ - # Data labels (e.g. 'foo = label: <3>') are irrelevant, so filter them - # out - types = [marker[1] for marker in self._markers - if marker[1] != _REF_LABEL] - - if not types: - return TYPE_EMPTY - - if types == [_TYPE_UINT8]: - return TYPE_BYTES - - if types == [_TYPE_UINT32]: - return TYPE_NUM if len(self.value) == 4 else TYPE_NUMS - - # Treat 'foo = <1 2 3>, <4 5>, ...' as TYPE_NUMS too - if set(types) == {_TYPE_UINT32}: - return TYPE_NUMS - - if set(types) == {_TYPE_STRING}: - return TYPE_STRING if len(types) == 1 else TYPE_STRINGS - - if types == [_REF_PATH]: - return TYPE_PATH - - if types == [_TYPE_UINT32, _REF_PHANDLE] and len(self.value) == 4: - return TYPE_PHANDLE - - if set(types) == {_TYPE_UINT32, _REF_PHANDLE}: - if len(self.value) == 4*types.count(_REF_PHANDLE): - # Array with just phandles in it - return TYPE_PHANDLES - # Array with both phandles and numbers - return TYPE_PHANDLES_AND_NUMS - - return TYPE_COMPOUND - - def __str__(self): - s = "".join(label + ": " for label in self.labels) + self.name - if not self.value: - return s + ";" - - s += " =" - - for i, (pos, marker_type, ref) in enumerate(self._markers): - if i < len(self._markers) - 1: - next_marker = self._markers[i + 1] - else: - next_marker = None - - # End of current marker - end = next_marker[0] if next_marker else len(self.value) - - if marker_type is _TYPE_STRING: - # end - 1 to strip off the null terminator - s += ' "{}"'.format(_decode_and_escape( - self.value[pos:end - 1])) - if end != len(self.value): - s += "," - elif marker_type is _REF_PATH: - s += " &" + ref - if end != len(self.value): - s += "," - else: - # <> or [] - - if marker_type is _REF_LABEL: - s += " {}:".format(ref) - elif marker_type is _REF_PHANDLE: - s += " &" + ref - pos += 4 - # Subtle: There might be more data between the phandle and - # the next marker, so we can't 'continue' here - else: # marker_type is _TYPE_UINT* - elm_size = _TYPE_TO_N_BYTES[marker_type] - s += _N_BYTES_TO_START_STR[elm_size] - - while pos != end: - num = int.from_bytes(self.value[pos:pos + elm_size], - "big") - if elm_size == 1: - s += " {:02X}".format(num) - else: - s += " " + hex(num) - - pos += elm_size - - if pos != 0 and \ - (not next_marker or - next_marker[1] not in (_REF_PHANDLE, _REF_LABEL)): - - s += _N_BYTES_TO_END_STR[elm_size] - if pos != len(self.value): - s += "," - - return s + ";" - - - def __repr__(self): - return "" \ - .format(self.name, self.node.path, self.node.dt.filename) - - # - # Internal functions - # - - def _add_marker(self, marker_type, data=None): - # Helper for registering markers in the value that are processed after - # parsing. See _fixup_props(). 'marker_type' identifies the type of - # marker, and 'data' has any optional data associated with the marker. - - # len(self.value) gives the current offset. This function is called - # while the value is built. We use a list instead of a tuple to be able - # to fix up offsets later (they might increase if the value includes - # path references, e.g. 'foo = &bar, <3>;', which are expanded later). - self._markers.append([len(self.value), marker_type, data]) - - # For phandle references, add a dummy value with the same length as a - # phandle. This is handy for the length check in _register_phandles(). - if marker_type is _REF_PHANDLE: - self.value += b"\0\0\0\0" - + self._parse_error(f"'{filename}' could not be found") # # Public functions # - -def to_num(data, length=None, signed=False): +def to_num(data: bytes, length: Optional[int] = None, + signed: bool = False) -> int: """ Converts the 'bytes' array 'data' to a number. The value is expected to be in big-endian format, which is standard in devicetree. @@ -1734,13 +1840,11 @@ def to_num(data, length=None, signed=False): if length is not None: _check_length_positive(length) if len(data) != length: - _err("{} is {} bytes long, expected {}" - .format(data, len(data), length)) + _err(f"{data!r} is {len(data)} bytes long, expected {length}") return int.from_bytes(data, "big", signed=signed) - -def to_nums(data, length=4, signed=False): +def to_nums(data: bytes, length: int = 4, signed: bool = False) -> List[int]: """ Like Property.to_nums(), but takes an arbitrary 'bytes' array. The values are assumed to be in big-endian format, which is standard in devicetree. @@ -1749,42 +1853,24 @@ def to_nums(data, length=4, signed=False): _check_length_positive(length) if len(data) % length: - _err("{} is {} bytes long, expected a length that's a a multiple of {}" - .format(data, len(data), length)) + _err(f"{data!r} is {len(data)} bytes long, " + f"expected a length that's a a multiple of {length}") return [int.from_bytes(data[i:i + length], "big", signed=signed) for i in range(0, len(data), length)] - # # Public constants # -# See Property.type -TYPE_EMPTY = 0 -TYPE_BYTES = 1 -TYPE_NUM = 2 -TYPE_NUMS = 3 -TYPE_STRING = 4 -TYPE_STRINGS = 5 -TYPE_PATH = 6 -TYPE_PHANDLE = 7 -TYPE_PHANDLES = 8 -TYPE_PHANDLES_AND_NUMS = 9 -TYPE_COMPOUND = 10 - - def _check_is_bytes(data): if not isinstance(data, bytes): - _err("'{}' has type '{}', expected 'bytes'" - .format(data, type(data).__name__)) - + _err(f"'{data}' has type '{type(data).__name__}', expected 'bytes'") def _check_length_positive(length): if length < 1: _err("'length' must be greater than zero, was " + str(length)) - def _append_no_dup(lst, elm): # Appends 'elm' to 'lst', but only if it isn't already in 'lst'. Lets us # preserve order, which a set() doesn't. @@ -1792,7 +1878,6 @@ def _append_no_dup(lst, elm): if elm not in lst: lst.append(elm) - def _decode_and_escape(b): # Decodes the 'bytes' array 'b' as UTF-8 and backslash-escapes special # characters @@ -1806,7 +1891,6 @@ def _decode_and_escape(b): .encode("utf-8", "surrogateescape") \ .decode("utf-8", "backslashreplace") - def _root_and_path_to_node(cur, path, fullpath): # Returns the node pointed at by 'path', relative to the Node 'cur'. For # example, if 'cur' has path /foo/bar, and 'path' is "baz/qaz", then the @@ -1819,18 +1903,16 @@ def _root_and_path_to_node(cur, path, fullpath): continue if component not in cur.nodes: - _err("component '{}' in path '{}' does not exist" - .format(component, fullpath)) + _err(f"component '{component}' in path '{fullpath}' " + "does not exist") cur = cur.nodes[component] return cur - -def _err(msg): +def _err(msg) -> NoReturn: raise DTError(msg) - _escape_table = str.maketrans({ "\\": "\\\\", '"': '\\"', @@ -1842,13 +1924,6 @@ _escape_table = str.maketrans({ "\f": "\\f", "\r": "\\r"}) - -class DTError(Exception): - "Exception raised for devicetree-related errors" - - -_Token = collections.namedtuple("Token", "id val") - # Lexer states _DEFAULT = 0 _EXPECT_PROPNODENAME = 1 @@ -1860,6 +1935,9 @@ _num_re = re.compile(r"(0[xX][0-9a-fA-F]+|[0-9]+)(?:ULL|UL|LL|U|L)?") # names that would clash with other stuff _propnodename_re = re.compile(r"\\?([a-zA-Z0-9,._+*#?@-]+)") +# Node names are more restrictive than property names. +_nodename_chars = set(string.ascii_letters + string.digits + ',._+-@') + # Misc. tokens that are tried after a property/node name. This is important, as # there's overlap with the allowed characters in names. _misc_re = re.compile( @@ -1874,92 +1952,59 @@ _byte_re = re.compile(r"[0-9a-fA-F]{2}") # '\c', where c might be a single character or an octal/hex escape. _unescape_re = re.compile(br'\\([0-7]{1,3}|x[0-9A-Fa-f]{1,2}|.)') -# #line directive (this is the regex the C tools use) -_line_re = re.compile( - r'^#(?:line)?[ \t]+([0-9]+)[ \t]+"((?:[^\\"]|\\.)*)"(?:[ \t]+[0-9]+)?', - re.MULTILINE) - - def _init_tokens(): - # Builds a ()|()|... regex and assigns the index of each - # capturing group to a corresponding _T_ variable. This makes the - # token type appear in match.lastindex after a match. - - global _token_re - global _T_NUM - global _T_PROPNODENAME - global _T_MISC - global _T_BYTE - global _T_BAD + # Builds a ()|()|... regex and returns it. The + # way this is constructed makes the token's value as an int appear + # in match.lastindex after a match. # Each pattern must have exactly one capturing group, which can capture any # part of the pattern. This makes match.lastindex match the token type. # _Token.val is based on the captured string. - token_spec = (("_T_INCLUDE", r'(/include/\s*"(?:[^\\"]|\\.)*")'), - ("_T_LINE", # #line directive - r'^#(?:line)?[ \t]+([0-9]+[ \t]+"(?:[^\\"]|\\.)*")(?:[ \t]+[0-9]+)?'), - ("_T_STRING", r'"((?:[^\\"]|\\.)*)"'), - ("_T_DTS_V1", r"(/dts-v1/)"), - ("_T_PLUGIN", r"(/plugin/)"), - ("_T_MEMRESERVE", r"(/memreserve/)"), - ("_T_BITS", r"(/bits/)"), - ("_T_DEL_PROP", r"(/delete-property/)"), - ("_T_DEL_NODE", r"(/delete-node/)"), - ("_T_OMIT_IF_NO_REF", r"(/omit-if-no-ref/)"), - ("_T_LABEL", r"([a-zA-Z_][a-zA-Z0-9_]*):"), - ("_T_CHAR_LITERAL", r"'((?:[^\\']|\\.)*)'"), - ("_T_REF", - r"&([a-zA-Z_][a-zA-Z0-9_]*|{[a-zA-Z0-9,._+*#?@/-]*})"), - ("_T_INCBIN", r"(/incbin/)"), - # Whitespace, C comments, and C++ comments - ("_T_SKIP", r"(\s+|(?:/\*(?:.|\n)*?\*/)|//.*$)"), - # Return a token for end-of-file so that the parsing code can - # always assume that there are more tokens when looking - # ahead. This simplifies things. - ("_T_EOF", r"(\Z)")) + token_spec = { + _T.INCLUDE: r'(/include/\s*"(?:[^\\"]|\\.)*")', + # #line directive or GCC linemarker + _T.LINE: + r'^#(?:line)?[ \t]+([0-9]+[ \t]+"(?:[^\\"]|\\.)*")(?:[ \t]+[0-9]+){0,4}', + + _T.STRING: r'"((?:[^\\"]|\\.)*)"', + _T.DTS_V1: r"(/dts-v1/)", + _T.PLUGIN: r"(/plugin/)", + _T.MEMRESERVE: r"(/memreserve/)", + _T.BITS: r"(/bits/)", + _T.DEL_PROP: r"(/delete-property/)", + _T.DEL_NODE: r"(/delete-node/)", + _T.OMIT_IF_NO_REF: r"(/omit-if-no-ref/)", + _T.LABEL: r"([a-zA-Z_][a-zA-Z0-9_]*):", + _T.CHAR_LITERAL: r"'((?:[^\\']|\\.)*)'", + _T.REF: r"&([a-zA-Z_][a-zA-Z0-9_]*|{[a-zA-Z0-9,._+*#?@/-]*})", + _T.INCBIN: r"(/incbin/)", + # Whitespace, C comments, and C++ comments + _T.SKIP: r"(\s+|(?:/\*(?:.|\n)*?\*/)|//.*$)", + # Return a token for end-of-file so that the parsing code can + # always assume that there are more tokens when looking + # ahead. This simplifies things. + _T.EOF: r"(\Z)", + } # MULTILINE is needed for C++ comments and #line directives - _token_re = re.compile("|".join(spec[1] for spec in token_spec), - re.MULTILINE | re.ASCII) + return re.compile("|".join(token_spec[tok_id] for tok_id in + range(1, _T.EOF + 1)), + re.MULTILINE | re.ASCII) - for i, spec in enumerate(token_spec, 1): - globals()[spec[0]] = i - - # pylint: disable=undefined-loop-variable - _T_NUM = i + 1 - _T_PROPNODENAME = i + 2 - _T_MISC = i + 3 - _T_BYTE = i + 4 - _T_BAD = i + 5 - - -_init_tokens() - -# Markers in property values - -# References -_REF_PATH = 0 # &foo -_REF_PHANDLE = 1 # <&foo> -_REF_LABEL = 2 # foo: <1 2 3> -# Start of data blocks of specific type -_TYPE_UINT8 = 3 # [00 01 02] (and also used for /incbin/) -_TYPE_UINT16 = 4 # /bits/ 16 <1 2 3> -_TYPE_UINT32 = 5 # <1 2 3> -_TYPE_UINT64 = 6 # /bits/ 64 <1 2 3> -_TYPE_STRING = 7 # "foo" +_token_re = _init_tokens() _TYPE_TO_N_BYTES = { - _TYPE_UINT8: 1, - _TYPE_UINT16: 2, - _TYPE_UINT32: 4, - _TYPE_UINT64: 8, + _MarkerType.UINT8: 1, + _MarkerType.UINT16: 2, + _MarkerType.UINT32: 4, + _MarkerType.UINT64: 8, } _N_BYTES_TO_TYPE = { - 1: _TYPE_UINT8, - 2: _TYPE_UINT16, - 4: _TYPE_UINT32, - 8: _TYPE_UINT64, + 1: _MarkerType.UINT8, + 2: _MarkerType.UINT16, + 4: _MarkerType.UINT32, + 8: _MarkerType.UINT64, } _N_BYTES_TO_START_STR = { diff --git a/src/devicetree/edtlib.py b/src/devicetree/edtlib.py index 12d9fa0..508085f 100644 --- a/src/devicetree/edtlib.py +++ b/src/devicetree/edtlib.py @@ -2,7 +2,7 @@ # Copyright (c) 2019 Linaro Limited # SPDX-License-Identifier: BSD-3-Clause -# Tip: You can view just the documentation with 'pydoc3 edtlib' +# Tip: You can view just the documentation with 'pydoc3 devicetree.edtlib' """ Library for working with devicetrees at a higher level compared to dtlib. Like @@ -25,7 +25,7 @@ See their constructor docstrings for details. There is also a bindings_from_paths() helper function. """ -# NOTE: testedtlib.py is the test suite for this library. +# NOTE: tests/test_edtlib.py is the test suite for this library. # Implementation notes # -------------------- @@ -68,9 +68,11 @@ bindings_from_paths() helper function. # variables. See the existing @properties for a template. from collections import OrderedDict, defaultdict +from copy import deepcopy import logging import os import re +from typing import Set import yaml try: @@ -78,12 +80,10 @@ try: # This makes e.g. gen_defines.py more than twice as fast. from yaml import CLoader as Loader except ImportError: - from yaml import Loader + from yaml import Loader # type: ignore -from .dtlib import DT, DTError, to_num, to_nums, TYPE_EMPTY, TYPE_BYTES, \ - TYPE_NUM, TYPE_NUMS, TYPE_STRING, TYPE_STRINGS, \ - TYPE_PHANDLE, TYPE_PHANDLES, TYPE_PHANDLES_AND_NUMS -from .grutils import Graph +from devicetree.dtlib import DT, DTError, to_num, to_nums, Type +from devicetree.grutils import Graph # @@ -150,7 +150,8 @@ class EDT: default_prop_types=True, support_fixed_partitions_on_any_bus=True, infer_binding_for_paths=None, - err_on_deprecated_properties=False): + vendor_prefixes=None, + werror=False): """EDT constructor. dts: @@ -179,20 +180,31 @@ class EDT: should be inferred from the node content. (Child nodes are not processed.) Pass none if no nodes should support inferred bindings. - err_on_deprecated_properties (default: False): - If True and 'dts' has any deprecated properties set, raise an error. + vendor_prefixes (default: None): + A dict mapping vendor prefixes in compatible properties to their + descriptions. If given, compatibles in the form "manufacturer,device" + for which "manufacturer" is neither a key in the dict nor a specially + exempt set of grandfathered-in cases will cause warnings. + werror (default: False): + If True, some edtlib specific warnings become errors. This currently + errors out if 'dts' has any deprecated properties set, or an unknown + vendor prefix is used. """ self._warn_reg_unit_address_mismatch = warn_reg_unit_address_mismatch self._default_prop_types = default_prop_types self._fixed_partitions_no_bus = support_fixed_partitions_on_any_bus self._infer_binding_for_paths = set(infer_binding_for_paths or []) - self._err_on_deprecated_properties = bool(err_on_deprecated_properties) + self._werror = bool(werror) + self._vendor_prefixes = vendor_prefixes or {} self.dts_path = dts self.bindings_dirs = bindings_dirs - self._dt = DT(dts) + try: + self._dt = DT(dts) + except DTError as e: + raise EDTError(e) from e _check_dt(self._dt) self._init_compat2binding() @@ -244,8 +256,8 @@ class EDT: return f"{self._dt}" def __repr__(self): - return "".format( - self.dts_path, self.bindings_dirs) + return f"" @property def scc_order(self): @@ -335,7 +347,7 @@ class EDT: # representing the file) raw = yaml.load(contents, Loader=_BindingLoader) except yaml.YAMLError as e: - _LOG.warning( + _err( f"'{binding_path}' appears in binding directories " f"but isn't valid YAML: {e}") continue @@ -344,26 +356,12 @@ class EDT: # if necessary. binding = self._binding(raw, binding_path, dt_compats) - if binding is None: - # Either the file is not a binding or it's a binding - # whose compatible does not appear in the devicetree - # (picked up via some unrelated text in the binding - # file that happened to match a compatible). - continue - - # Do not allow two different bindings to have the same - # 'compatible:'/'on-bus:' combo - old_binding = self._compat2binding.get((binding.compatible, - binding.on_bus)) - if old_binding: - msg = (f"both {old_binding.path} and {binding_path} have " - f"'compatible: {binding.compatible}'") - if binding.on_bus is not None: - msg += f" and 'on-bus: {binding.on_bus}'" - _err(msg) - - # Register the binding. - self._compat2binding[binding.compatible, binding.on_bus] = binding + # Register the binding in self._compat2binding, along with + # any child bindings that have their own compatibles. + while binding is not None: + if binding.compatible: + self._register_binding(binding) + binding = binding.child_binding def _binding(self, raw, binding_path, dt_compats): # Convert a 'raw' binding from YAML to a Binding object and return it. @@ -387,6 +385,21 @@ class EDT: # Initialize and return the Binding object. return Binding(binding_path, self._binding_fname2path, raw=raw) + def _register_binding(self, binding): + # Do not allow two different bindings to have the same + # 'compatible:'/'on-bus:' combo + old_binding = self._compat2binding.get((binding.compatible, + binding.on_bus)) + if old_binding: + msg = (f"both {old_binding.path} and {binding.path} have " + f"'compatible: {binding.compatible}'") + if binding.on_bus is not None: + msg += f" and 'on-bus: {binding.on_bus}'" + _err(msg) + + # Register the binding. + self._compat2binding[binding.compatible, binding.on_bus] = binding + def _init_nodes(self): # Creates a list of edtlib.Node objects from the dtlib.Node objects, in # self.nodes @@ -409,6 +422,7 @@ class EDT: node.bus_node = node._bus_node(self._fixed_partitions_no_bus) node._init_binding() node._init_regs() + node._init_ranges() self.nodes.append(node) self._node2enode[dt_node] = node @@ -418,8 +432,7 @@ class EDT: # they (either always or sometimes) reference other nodes, so we # run them separately node._init_props(default_prop_types=self._default_prop_types, - err_on_deprecated= - self._err_on_deprecated_properties) + err_on_deprecated=self._werror) node._init_interrupts() node._init_pinctrls() @@ -475,6 +488,55 @@ class EDT: 'in lowercase: ' + ', '.join(repr(x) for x in spec.enum)) + # Validate the contents of compatible properties. + self._checked_compatibles: Set[str] = set() + for node in self.nodes: + if 'compatible' not in node.props: + continue + + compatibles = node.props['compatible'].val + + # _check() runs after _init_compat2binding() has called + # _dt_compats(), which already converted every compatible + # property to a list of strings. So we know 'compatibles' + # is a list, but add an assert for future-proofing. + assert isinstance(compatibles, list) + + for compat in compatibles: + # This is also just for future-proofing. + assert isinstance(compat, str) + + self._check_compatible(node, compat) + del self._checked_compatibles # We have no need for this anymore. + + def _check_compatible(self, node, compat): + if compat in self._checked_compatibles: + return + + # The regular expression comes from dt-schema. + compat_re = r'^[a-zA-Z][a-zA-Z0-9,+\-._]+$' + if not re.match(compat_re, compat): + _err(f"node '{node.path}' compatible '{compat}' " + 'must match this regular expression: ' + f"'{compat_re}'") + + if ',' in compat and self._vendor_prefixes: + vendor = compat.split(',', 1)[0] + # As an exception, the root node can have whatever + # compatibles it wants. Other nodes get checked. + if node.path != '/' and \ + vendor not in self._vendor_prefixes and \ + vendor not in _VENDOR_PREFIX_ALLOWED: + if self._werror: + handler_fn = _err + else: + handler_fn = _LOG.warning + handler_fn( + f"node '{node.path}' compatible '{compat}' " + f"has unknown vendor prefix '{vendor}'") + + self._checked_compatibles.add(compat) + class Node: """ Represents a devicetree node, augmented with information from bindings, and @@ -555,6 +617,10 @@ class Node: A list of 'compatible' strings for the node, in the same order that they're listed in the .dts file + ranges: + A list if Range objects extracted from the node's ranges property. + The list is empty if the node does not have a range property. + regs: A list of Register objects for the node's registers @@ -616,7 +682,7 @@ class Node: try: addr = int(self.name.split("@", 1)[1], 16) except ValueError: - _err("{!r} has non-hex unit address".format(self)) + _err(f"{self!r} has non-hex unit address") return _translate(addr, self._node) @@ -658,6 +724,21 @@ class Node: return OrderedDict((name, self.edt._node2enode[node]) for name, node in self._node.nodes.items()) + def child_index(self, node): + """Get the index of *node* in self.children. + Raises KeyError if the argument is not a child of this node. + """ + if not hasattr(self, '_child2index'): + # Defer initialization of this lookup table until this + # method is callable to handle parents needing to be + # initialized before their chidlren. By the time we + # return from __init__, 'self.children' is callable. + self._child2index = OrderedDict() + for index, child in enumerate(self.children.values()): + self._child2index[child] = index + + return self._child2index[node] + @property def required_by(self): "See the class docstring" @@ -719,8 +800,7 @@ class Node: # parent of the flash node. if not self.parent or not self.parent.parent: - _err("flash partition {!r} lacks parent or grandparent node" - .format(self)) + _err(f"flash partition {self!r} lacks parent or grandparent node") controller = self.parent.parent if controller.matching_compat == "soc-nv-flash": @@ -735,25 +815,26 @@ class Node: return None if not self.regs: - _err("{!r} needs a 'reg' property, to look up the chip select index " - "for SPI".format(self)) + _err(f"{self!r} needs a 'reg' property, to look up the " + "chip select index for SPI") parent_cs_lst = self.bus_node.props["cs-gpios"].val # cs-gpios is indexed by the unit address cs_index = self.regs[0].addr if cs_index >= len(parent_cs_lst): - _err("index from 'regs' in {!r} ({}) is >= number of cs-gpios " - "in {!r} ({})".format( - self, cs_index, self.bus_node, len(parent_cs_lst))) + _err(f"index from 'regs' in {self!r} ({cs_index}) " + "is >= number of cs-gpios in " + f"{self.bus_node!r} ({len(parent_cs_lst)})") return parent_cs_lst[cs_index] def __repr__(self): - return "".format( - self.path, self.edt.dts_path, - "binding " + self.binding_path if self.binding_path - else "no binding") + if self.binding_path: + binding = "binding " + self.binding_path + else: + binding = "no binding" + return f"" def _init_binding(self): # Initializes Node.matching_compat, Node._binding, and @@ -820,26 +901,29 @@ class Node: } for name, prop in self._node.props.items(): pp = OrderedDict() - if prop.type == TYPE_EMPTY: + if prop.type == Type.EMPTY: pp["type"] = "boolean" - elif prop.type == TYPE_BYTES: + elif prop.type == Type.BYTES: pp["type"] = "uint8-array" - elif prop.type == TYPE_NUM: + elif prop.type == Type.NUM: pp["type"] = "int" - elif prop.type == TYPE_NUMS: + elif prop.type == Type.NUMS: pp["type"] = "array" - elif prop.type == TYPE_STRING: + elif prop.type == Type.STRING: pp["type"] = "string" - elif prop.type == TYPE_STRINGS: + elif prop.type == Type.STRINGS: pp["type"] = "string-array" - elif prop.type == TYPE_PHANDLE: + elif prop.type == Type.PHANDLE: pp["type"] = "phandle" - elif prop.type == TYPE_PHANDLES: + elif prop.type == Type.PHANDLES: pp["type"] = "phandles" - elif prop.type == TYPE_PHANDLES_AND_NUMS: + elif prop.type == Type.PHANDLES_AND_NUMS: pp["type"] = "phandle-array" + elif prop.type == Type.PATH: + pp["type"] = "path" else: - _err(f"cannot infer binding from property: {prop}") + _err(f"cannot infer binding from property: {prop} " + f"with type {prop.type!r}") raw['properties'][name] = pp # Set up Node state. @@ -910,7 +994,7 @@ class Node: continue prop_spec = _DEFAULT_PROP_SPECS[name] val = self._prop_val(name, prop_spec.type, False, False, None, - err_on_deprecated) + None, err_on_deprecated) self.props[name] = Property(prop_spec, val, self) def _init_prop(self, prop_spec, err_on_deprecated): @@ -920,11 +1004,11 @@ class Node: name = prop_spec.name prop_type = prop_spec.type if not prop_type: - _err("'{}' in {} lacks 'type'".format(name, self.binding_path)) + _err(f"'{name}' in {self.binding_path} lacks 'type'") val = self._prop_val(name, prop_type, prop_spec.deprecated, prop_spec.required, prop_spec.default, - err_on_deprecated) + prop_spec.specifier_space, err_on_deprecated) if val is None: # 'required: false' property that wasn't there, or a property type @@ -933,17 +1017,16 @@ class Node: enum = prop_spec.enum if enum and val not in enum: - _err("value of property '{}' on {} in {} ({!r}) is not in 'enum' " - "list in {} ({!r})" - .format(name, self.path, self.edt.dts_path, val, - self.binding_path, enum)) + _err(f"value of property '{name}' on {self.path} in " + f"{self.edt.dts_path} ({val!r}) is not in 'enum' list in " + f"{self.binding_path} ({enum!r})") const = prop_spec.const if const is not None and val != const: - _err("value of property '{}' on {} in {} ({!r}) is different from " - "the 'const' value specified in {} ({!r})" - .format(name, self.path, self.edt.dts_path, val, - self.binding_path, const)) + _err(f"value of property '{name}' on {self.path} in " + f"{self.edt.dts_path} ({val!r}) " + "is different from the 'const' value specified in " + f"{self.binding_path} ({const!r})") # Skip properties that start with '#', like '#size-cells', and mapping # properties like 'gpio-map'/'interrupt-map' @@ -953,7 +1036,7 @@ class Node: self.props[name] = Property(prop_spec, val, self) def _prop_val(self, name, prop_type, deprecated, required, default, - err_on_deprecated): + specifier_space, err_on_deprecated): # _init_prop() helper for getting the property's value # # name: @@ -972,6 +1055,9 @@ class Node: # Default value to use when the property doesn't exist, or None if # the binding doesn't give a default value # + # specifier_space: + # Property specifier-space from binding (if prop_type is "phandle-array") + # # err_on_deprecated: # If True, a deprecated property is an error instead of warning. @@ -988,9 +1074,8 @@ class Node: if not prop: if required and self.status == "okay": - _err("'{}' is marked as required in 'properties:' in {}, but " - "does not appear in {!r}".format( - name, self.binding_path, node)) + _err(f"'{name}' is marked as required in 'properties:' in " + f"{self.binding_path}, but does not appear in {node!r}") if default is not None: # YAML doesn't have a native format for byte arrays. We need to @@ -1004,7 +1089,7 @@ class Node: return False if prop_type == "boolean" else None if prop_type == "boolean": - if prop.type is not TYPE_EMPTY: + if prop.type != Type.EMPTY: _err("'{0}' in {1!r} is defined with 'type: boolean' in {2}, " "but is assigned a value ('{3}') instead of being empty " "('{0};')".format(name, node, self.binding_path, prop)) @@ -1035,13 +1120,13 @@ class Node: # This type is a bit high-level for dtlib as it involves # information from bindings and *-names properties, so there's no # to_phandle_array() in dtlib. Do the type check ourselves. - if prop.type not in (TYPE_PHANDLE, TYPE_PHANDLES, TYPE_PHANDLES_AND_NUMS): + if prop.type not in (Type.PHANDLE, Type.PHANDLES, Type.PHANDLES_AND_NUMS): _err(f"expected property '{name}' in {node.path} in " f"{node.dt.filename} to be assigned " f"with '{name} = < &foo ... &bar 1 ... &baz 2 3 >' " f"(a mix of phandles and numbers), not '{prop}'") - return self._standard_phandle_val_list(prop) + return self._standard_phandle_val_list(prop, specifier_space) if prop_type == "path": return self.edt._node2enode[prop.to_path()] @@ -1068,10 +1153,70 @@ class Node: continue if prop_name not in self._binding.prop2specs: - _err("'{}' appears in {} in {}, but is not declared in " - "'properties:' in {}" - .format(prop_name, self._node.path, self.edt.dts_path, - self.binding_path)) + _err(f"'{prop_name}' appears in {self._node.path} in " + f"{self.edt.dts_path}, but is not declared in " + f"'properties:' in {self.binding_path}") + + def _init_ranges(self): + # Initializes self.ranges + node = self._node + + self.ranges = [] + + if "ranges" not in node.props: + return + + child_address_cells = node.props.get("#address-cells") + parent_address_cells = _address_cells(node) + if child_address_cells is None: + child_address_cells = 2 # Default value per DT spec. + else: + child_address_cells = child_address_cells.to_num() + child_size_cells = node.props.get("#size-cells") + if child_size_cells is None: + child_size_cells = 1 # Default value per DT spec. + else: + child_size_cells = child_size_cells.to_num() + + # Number of cells for one translation 3-tuple in 'ranges' + entry_cells = child_address_cells + parent_address_cells + child_size_cells + + if entry_cells == 0: + if len(node.props["ranges"].value) == 0: + return + else: + _err(f"'ranges' should be empty in {self._node.path} since " + f"<#address-cells> = {child_address_cells}, " + f"<#address-cells for parent> = {parent_address_cells} and " + f"<#size-cells> = {child_size_cells}") + + for raw_range in _slice(node, "ranges", 4*entry_cells, + f"4*(<#address-cells> (= {child_address_cells}) + " + "<#address-cells for parent> " + f"(= {parent_address_cells}) + " + f"<#size-cells> (= {child_size_cells}))"): + + range = Range() + range.node = self + range.child_bus_cells = child_address_cells + if child_address_cells == 0: + range.child_bus_addr = None + else: + range.child_bus_addr = to_num(raw_range[:4*child_address_cells]) + range.parent_bus_cells = parent_address_cells + if parent_address_cells == 0: + range.parent_bus_addr = None + else: + range.parent_bus_addr = to_num(raw_range[(4*child_address_cells):\ + (4*child_address_cells + 4*parent_address_cells)]) + range.length_cells = child_size_cells + if child_size_cells == 0: + range.length = None + else: + range.length = to_num(raw_range[(4*child_address_cells + \ + 4*parent_address_cells):]) + + self.ranges.append(range) def _init_regs(self): # Initializes self.regs @@ -1087,8 +1232,8 @@ class Node: size_cells = _size_cells(node) for raw_reg in _slice(node, "reg", 4*(address_cells + size_cells), - "4*(<#address-cells> (= {}) + <#size-cells> (= {}))" - .format(address_cells, size_cells)): + f"4*(<#address-cells> (= {address_cells}) + " + f"<#size-cells> (= {size_cells}))"): reg = Register() reg.node = self if address_cells == 0: @@ -1100,9 +1245,9 @@ class Node: else: reg.size = to_num(raw_reg[4*address_cells:]) if size_cells != 0 and reg.size == 0: - _err("zero-sized 'reg' in {!r} seems meaningless (maybe you " - "want a size of one or #size-cells = 0 instead)" - .format(self._node)) + _err(f"zero-sized 'reg' in {self._node!r} seems meaningless " + "(maybe you want a size of one or #size-cells = 0 " + "instead)") self.regs.append(reg) @@ -1122,8 +1267,8 @@ class Node: # Check indices for i, prop in enumerate(pinctrl_props): if prop.name != "pinctrl-" + str(i): - _err("missing 'pinctrl-{}' property on {!r} - indices should " - "be contiguous and start from zero".format(i, node)) + _err(f"missing 'pinctrl-{i}' property on {node!r} " + "- indices should be contiguous and start from zero") self.pinctrls = [] for prop in pinctrl_props: @@ -1154,41 +1299,59 @@ class Node: _add_names(node, "interrupt", self.interrupts) - def _standard_phandle_val_list(self, prop): + def _standard_phandle_val_list(self, prop, specifier_space): # Parses a property like # - # s = - # (e.g., pwms = <&foo 1 2 &bar 3 4>) + # = ; # - # , where each phandle points to a node that has a + # where each phandle points to a controller node that has a # - # #-cells = + # #-cells = ; # # property that gives the number of cells in the value after the - # phandle. These values are given names in *-cells in the binding for - # the controller. + # controller's phandle in the property. + # + # E.g. with a property like + # + # pwms = <&foo 1 2 &bar 3>; + # + # If 'specifier_space' is "pwm", then we should have this elsewhere + # in the tree: + # + # foo: ... { + # #pwm-cells = <2>; + # }; + # + # bar: ... { + # #pwm-cells = <1>; + # }; + # + # These values can be given names using the -names: + # list in the binding for the phandle nodes. # # Also parses any # - # -names = "...", "...", ... + # -names = "...", "...", ... # # Returns a list of Optional[ControllerAndData] instances. - # An index is None if the underlying phandle-array element - # is unspecified. + # + # An index is None if the underlying phandle-array element is + # unspecified. - if prop.name.endswith("gpios"): - # There's some slight special-casing for *-gpios properties in that - # e.g. foo-gpios still maps to #gpio-cells rather than - # #foo-gpio-cells - basename = "gpio" - else: - # Strip -s. We've already checked that the property names end in -s - # in _check_prop_type_and_default(). - basename = prop.name[:-1] + if not specifier_space: + if prop.name.endswith("gpios"): + # There's some slight special-casing for *-gpios properties in that + # e.g. foo-gpios still maps to #gpio-cells rather than + # #foo-gpio-cells + specifier_space = "gpio" + else: + # Strip -s. We've already checked that property names end in -s + # if there is no specifier space in _check_prop_type_and_default(). + specifier_space = prop.name[:-1] res = [] - for item in _phandle_val_list(prop, basename): + for item in _phandle_val_list(prop, specifier_space): if item is None: res.append(None) continue @@ -1196,17 +1359,18 @@ class Node: controller_node, data = item mapped_controller, mapped_data = \ _map_phandle_array_entry(prop.node, controller_node, data, - basename) + specifier_space) entry = ControllerAndData() entry.node = self entry.controller = self.edt._node2enode[mapped_controller] + entry.basename = specifier_space entry.data = self._named_cells(entry.controller, mapped_data, - basename) + specifier_space) res.append(entry) - _add_names(self._node, basename, res) + _add_names(self._node, specifier_space, res) return res @@ -1216,8 +1380,8 @@ class Node: # byte array. if not controller._binding: - _err("{} controller {!r} for {!r} lacks binding" - .format(basename, controller._node, self._node)) + _err(f"{basename} controller {controller._node!r} " + f"for {self._node!r} lacks binding") if basename in controller._binding.specifier2cells: cell_names = controller._binding.specifier2cells[basename] @@ -1229,14 +1393,62 @@ class Node: data_list = to_nums(data) if len(data_list) != len(cell_names): - _err("unexpected '{}-cells:' length in binding for {!r} - {} " - "instead of {}" - .format(basename, controller._node, len(cell_names), - len(data_list))) + _err(f"unexpected '{basename}-cells:' length in binding for " + f"{controller._node!r} - {len(cell_names)} " + f"instead of {len(data_list)}") return OrderedDict(zip(cell_names, data_list)) +class Range: + """ + Represents a translation range on a node as described by the 'ranges' property. + + These attributes are available on Range objects: + + node: + The Node instance this range is from + + child_bus_cells: + Is the number of cells (4-bytes wide words) describing the child bus address. + + child_bus_addr: + Is a physical address within the child bus address space, or None if the + child address-cells is equal 0. + + parent_bus_cells: + Is the number of cells (4-bytes wide words) describing the parent bus address. + + parent_bus_addr: + Is a physical address within the parent bus address space, or None if the + parent address-cells is equal 0. + + length_cells: + Is the number of cells (4-bytes wide words) describing the size of range in + the child address space. + + length: + Specifies the size of the range in the child address space, or None if the + child size-cells is equal 0. + """ + def __repr__(self): + fields = [] + + if self.child_bus_cells is not None: + fields.append("child-bus-cells: " + hex(self.child_bus_cells)) + if self.child_bus_addr is not None: + fields.append("child-bus-addr: " + hex(self.child_bus_addr)) + if self.parent_bus_cells is not None: + fields.append("parent-bus-cells: " + hex(self.parent_bus_cells)) + if self.parent_bus_addr is not None: + fields.append("parent-bus-addr: " + hex(self.parent_bus_addr)) + if self.length_cells is not None: + fields.append("length-cells " + hex(self.length_cells)) + if self.length is not None: + fields.append("length " + hex(self.length)) + + return "".format(", ".join(fields)) + class Register: """ Represents a register on a node. @@ -1297,6 +1509,9 @@ class ControllerAndData: The name of the entry as given in 'interrupt-names'/'gpio-names'/'pwm-names'/etc., or None if there is no *-names property + + basename: + Basename for the controller when supporting named cells """ def __repr__(self): fields = [] @@ -1304,8 +1519,8 @@ class ControllerAndData: if self.name is not None: fields.append("name: " + self.name) - fields.append("controller: {}".format(self.controller)) - fields.append("data: {}".format(self.data)) + fields.append(f"controller: {self.controller}") + fields.append(f"data: {self.data}") return "".format(", ".join(fields)) @@ -1324,12 +1539,21 @@ class PinCtrl: The name of the configuration, as given in pinctrl-names, or None if there is no pinctrl-names property + name_as_token: + Like 'name', but with non-alphanumeric characters converted to underscores. + conf_nodes: A list of Node instances for the pin configuration nodes, e.g. the nodes pointed at by &state_1 and &state_2 in pinctrl-0 = <&state_1 &state_2>; """ + + @property + def name_as_token(self): + "See the class docstring" + return _val_as_token(self.name) if self.name is not None else None + def __repr__(self): fields = [] @@ -1421,7 +1645,7 @@ class Property: @property def val_as_token(self): "See the class docstring" - return re.sub(_NOT_ALPHANUM_OR_UNDERSCORE, '_', self.val) + return _val_as_token(self.val) @property def enum_index(self): @@ -1436,7 +1660,7 @@ class Property: "value: " + repr(self.val)] if self.enum_index is not None: - fields.append("enum index: {}".format(self.enum_index)) + fields.append(f"enum index: {self.enum_index}") return "".format(", ".join(fields)) @@ -1605,26 +1829,51 @@ class Binding: return raw include = raw.pop("include") - fnames = [] - if isinstance(include, str): - fnames.append(include) - elif isinstance(include, list): - if not all(isinstance(elem, str) for elem in include): - _err(f"all elements in 'include:' in {binding_path} " - "should be strings") - fnames += include - else: - _err(f"'include:' in {binding_path} " - "should be a string or a list of strings") # First, merge the included files together. If more than one included # file has a 'required:' for a particular property, OR the values # together, so that 'required: true' wins. merged = {} - for fname in fnames: - _merge_props(merged, self._load_raw(fname), None, binding_path, - check_required=False) + + if isinstance(include, str): + # Simple scalar string case + _merge_props(merged, self._load_raw(include), None, binding_path, + False) + elif isinstance(include, list): + # List of strings and maps. These types may be intermixed. + for elem in include: + if isinstance(elem, str): + _merge_props(merged, self._load_raw(elem), None, + binding_path, False) + elif isinstance(elem, dict): + name = elem.pop('name', None) + allowlist = elem.pop('property-allowlist', None) + blocklist = elem.pop('property-blocklist', None) + child_filter = elem.pop('child-binding', None) + + if elem: + # We've popped out all the valid keys. + _err(f"'include:' in {binding_path} should not have " + f"these unexpected contents: {elem}") + + _check_include_dict(name, allowlist, blocklist, + child_filter, binding_path) + + contents = self._load_raw(name) + + _filter_properties(contents, allowlist, blocklist, + child_filter, binding_path) + _merge_props(merged, contents, None, binding_path, False) + else: + _err(f"all elements in 'include:' in {binding_path} " + "should be either strings or maps with a 'name' key " + "and optional 'property-allowlist' or " + f"'property-blocklist' keys, but got: {elem}") + else: + # Invalid item. + _err(f"'include:' in {binding_path} " + f"should be a string or list, but has type {type(include)}") # Next, merge the merged included files into 'raw'. Error out if # 'raw' has 'required: false' while the merged included files have @@ -1719,7 +1968,8 @@ class Binding: return ok_prop_keys = {"description", "type", "required", - "enum", "const", "default", "deprecated"} + "enum", "const", "default", "deprecated", + "specifier-space"} for prop_name, options in raw["properties"].items(): for key in options: @@ -1729,9 +1979,7 @@ class Binding: f"expected one of {', '.join(ok_prop_keys)}") _check_prop_type_and_default( - prop_name, options.get("type"), - options.get("default"), - self.path) + prop_name, options, self.path) for true_false_opt in ["required", "deprecated"]: if true_false_opt in options: @@ -1831,6 +2079,9 @@ class PropertySpec: required: True if the property is marked required; False otherwise. + + specifier_space: + The specifier space for the property as given in the binding, or None. """ def __init__(self, name, binding): @@ -1910,9 +2161,40 @@ class PropertySpec: "See the class docstring" return self._raw.get("deprecated", False) + @property + def specifier_space(self): + "See the class docstring" + return self._raw.get("specifier-space") + + class EDTError(Exception): "Exception raised for devicetree- and binding-related errors" +# +# Public global functions +# + + +def load_vendor_prefixes_txt(vendor_prefixes): + """Load a vendor-prefixes.txt file and return a dict + representation mapping a vendor prefix to the vendor name. + """ + vnd2vendor = {} + with open(vendor_prefixes, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + + if not line or line.startswith('#'): + # Comment or empty line. + continue + + # Other lines should be in this form: + # + # + vnd_vendor = line.split('\t', 1) + assert len(vnd_vendor) == 2, line + vnd2vendor[vnd_vendor[0]] = vnd_vendor[1] + return vnd2vendor # # Private global functions @@ -1938,7 +2220,7 @@ def _binding_paths(bindings_dirs): for bindings_dir in bindings_dirs: for root, _, filenames in os.walk(bindings_dir): for filename in filenames: - if filename.endswith(".yaml"): + if filename.endswith(".yaml") or filename.endswith(".yml"): binding_paths.append(os.path.join(root, filename)) return binding_paths @@ -1950,6 +2232,89 @@ def _binding_inc_error(msg): raise yaml.constructor.ConstructorError(None, None, "error: " + msg) +def _check_include_dict(name, allowlist, blocklist, child_filter, + binding_path): + # Check that an 'include:' named 'name' with property-allowlist + # 'allowlist', property-blocklist 'blocklist', and + # child-binding filter 'child_filter' has valid structure. + + if name is None: + _err(f"'include:' element in {binding_path} " + "should have a 'name' key") + + if allowlist is not None and blocklist is not None: + _err(f"'include:' of file '{name}' in {binding_path} " + "should not specify both 'property-allowlist:' " + "and 'property-blocklist:'") + + while child_filter is not None: + child_copy = deepcopy(child_filter) + child_allowlist = child_copy.pop('property-allowlist', None) + child_blocklist = child_copy.pop('property-blocklist', None) + next_child_filter = child_copy.pop('child-binding', None) + + if child_copy: + # We've popped out all the valid keys. + _err(f"'include:' of file '{name}' in {binding_path} " + "should not have these unexpected contents in a " + f"'child-binding': {child_copy}") + + if child_allowlist is not None and child_blocklist is not None: + _err(f"'include:' of file '{name}' in {binding_path} " + "should not specify both 'property-allowlist:' and " + "'property-blocklist:' in a 'child-binding:'") + + child_filter = next_child_filter + + +def _filter_properties(raw, allowlist, blocklist, child_filter, + binding_path): + # Destructively modifies 'raw["properties"]' and + # 'raw["child-binding"]', if they exist, according to + # 'allowlist', 'blocklist', and 'child_filter'. + + props = raw.get('properties') + _filter_properties_helper(props, allowlist, blocklist, binding_path) + + child_binding = raw.get('child-binding') + while child_filter is not None and child_binding is not None: + _filter_properties_helper(child_binding.get('properties'), + child_filter.get('property-allowlist'), + child_filter.get('property-blocklist'), + binding_path) + child_filter = child_filter.get('child-binding') + child_binding = child_binding.get('child-binding') + + +def _filter_properties_helper(props, allowlist, blocklist, binding_path): + if props is None or (allowlist is None and blocklist is None): + return + + _check_prop_filter('property-allowlist', allowlist, binding_path) + _check_prop_filter('property-blocklist', blocklist, binding_path) + + if allowlist is not None: + allowset = set(allowlist) + to_del = [prop for prop in props if prop not in allowset] + else: + blockset = set(blocklist) + to_del = [prop for prop in props if prop in blockset] + + for prop in to_del: + del props[prop] + + +def _check_prop_filter(name, value, binding_path): + # Ensure an include: ... property-allowlist or property-blocklist + # is a list. + + if value is None: + return + + if not isinstance(value, list): + _err(f"'{name}' value {value} in {binding_path} should be a list") + + def _merge_props(to_dict, from_dict, parent, binding_path, check_required): # Recursively merges 'from_dict' into 'to_dict', to implement 'include:'. # @@ -1977,17 +2342,16 @@ def _merge_props(to_dict, from_dict, parent, binding_path, check_required): elif prop not in to_dict: to_dict[prop] = from_dict[prop] elif _bad_overwrite(to_dict, from_dict, prop, check_required): - _err("{} (in '{}'): '{}' from included file overwritten " - "('{}' replaced with '{}')".format( - binding_path, parent, prop, from_dict[prop], - to_dict[prop])) + _err(f"{binding_path} (in '{parent}'): '{prop}' " + f"from included file overwritten ('{from_dict[prop]}' " + f"replaced with '{to_dict[prop]}')") elif prop == "required": # Need a separate check here, because this code runs before # Binding._check() if not (isinstance(from_dict["required"], bool) and isinstance(to_dict["required"], bool)): - _err("malformed 'required:' setting for '{}' in 'properties' " - "in {}, expected true/false".format(parent, binding_path)) + _err(f"malformed 'required:' setting for '{parent}' in " + f"'properties' in {binding_path}, expected true/false") # 'required: true' takes precedence to_dict["required"] = to_dict["required"] or from_dict["required"] @@ -2027,29 +2391,35 @@ def _binding_include(loader, node): _binding_inc_error("unrecognised node type in !include statement") -def _check_prop_type_and_default(prop_name, prop_type, default, binding_path): - # Binding._check_properties() helper. Checks 'type:' and 'default:' for the - # property named 'prop_name' +def _check_prop_type_and_default(prop_name, options, binding_path): + # Binding._check_properties() helper. Checks 'type:', 'default:' and + # 'specifier-space:' for the property named 'prop_name' + + prop_type = options.get("type") + default = options.get("default") if prop_type is None: - _err("missing 'type:' for '{}' in 'properties' in {}" - .format(prop_name, binding_path)) + _err(f"missing 'type:' for '{prop_name}' in 'properties' in " + f"{binding_path}") ok_types = {"boolean", "int", "array", "uint8-array", "string", "string-array", "phandle", "phandles", "phandle-array", "path", "compound"} if prop_type not in ok_types: - _err("'{}' in 'properties:' in {} has unknown type '{}', expected one " - "of {}".format(prop_name, binding_path, prop_type, - ", ".join(ok_types))) + _err(f"'{prop_name}' in 'properties:' in {binding_path} " + f"has unknown type '{prop_type}', expected one of " + + ", ".join(ok_types)) - if prop_type == "phandle-array" and not prop_name.endswith("s"): - _err("'{}' in 'properties:' in {} is 'type: phandle-array', but its " - "name does not end in -s. This is required since property names " - "like '#pwm-cells' and 'pwm-names' get derived from 'pwms', for " - "example.".format(prop_name, binding_path)) + if "specifier-space" in options and prop_type != "phandle-array": + _err(f"'specifier-space' in 'properties: {prop_name}' " + f"has type '{prop_type}', expected 'phandle-array'") + if prop_type == "phandle-array": + if not prop_name.endswith("s") and not "specifier-space" in options: + _err(f"'{prop_name}' in 'properties:' in {binding_path} " + f"has type 'phandle-array' and its name does not end in 's', " + f"but no 'specifier-space' was provided.") # Check default if default is None: @@ -2057,8 +2427,9 @@ def _check_prop_type_and_default(prop_name, prop_type, default, binding_path): if prop_type in {"boolean", "compound", "phandle", "phandles", "phandle-array", "path"}: - _err("'default:' can't be combined with 'type: {}' for '{}' in " - "'properties:' in {}".format(prop_type, prop_name, binding_path)) + _err("'default:' can't be combined with " + f"'type: {prop_type}' for '{prop_name}' in " + f"'properties:' in {binding_path}") def ok_default(): # Returns True if 'default' is an okay default for the property's type @@ -2084,8 +2455,9 @@ def _check_prop_type_and_default(prop_name, prop_type, default, binding_path): return all(isinstance(val, str) for val in default) if not ok_default(): - _err("'default: {}' is invalid for '{}' in 'properties:' in {}, which " - "has type {}".format(default, prop_name, binding_path, prop_type)) + _err(f"'default: {default}' is invalid for '{prop_name}' " + f"in 'properties:' in {binding_path}, " + f"which has type {prop_type}") def _translate(addr, node): @@ -2118,11 +2490,10 @@ def _translate(addr, node): entry_cells = child_address_cells + parent_address_cells + child_size_cells for raw_range in _slice(node.parent, "ranges", 4*entry_cells, - "4*(<#address-cells> (= {}) + " - "<#address-cells for parent> (= {}) + " - "<#size-cells> (= {}))" - .format(child_address_cells, parent_address_cells, - child_size_cells)): + f"4*(<#address-cells> (= {child_address_cells}) + " + "<#address-cells for parent> " + f"(= {parent_address_cells}) + " + f"<#size-cells> (= {child_size_cells}))"): child_addr = to_num(raw_range[:4*child_address_cells]) raw_range = raw_range[4*child_address_cells:] @@ -2157,9 +2528,9 @@ def _add_names(node, names_ident, objs): if full_names_ident in node.props: names = node.props[full_names_ident].to_strings() if len(names) != len(objs): - _err("{} property in {} in {} has {} strings, expected {} strings" - .format(full_names_ident, node.path, node.dt.filename, - len(names), len(objs))) + _err(f"{full_names_ident} property in {node.path} " + f"in {node.dt.filename} has {len(names)} strings, " + f"expected {len(objs)} strings") for obj, name in zip(objs, names): if obj is None: @@ -2181,8 +2552,8 @@ def _interrupt_parent(node): return node.props["interrupt-parent"].to_node() node = node.parent - _err("{!r} has an 'interrupts' property, but neither the node nor any " - "of its parents has an 'interrupt-parent' property".format(node)) + _err(f"{node!r} has an 'interrupts' property, but neither the node " + f"nor any of its parents has an 'interrupt-parent' property") def _interrupts(node): @@ -2226,8 +2597,8 @@ def _map_interrupt(child, parent, child_spec): address_cells = node.props.get("#address-cells") if not address_cells: - _err("missing #address-cells on {!r} (while handling interrupt-map)" - .format(node)) + _err(f"missing #address-cells on {node!r} " + "(while handling interrupt-map)") return address_cells.to_num() def spec_len_fn(node): @@ -2249,10 +2620,10 @@ def _map_phandle_array_entry(child, parent, child_spec, basename): # _map_interrupt(). def spec_len_fn(node): - prop_name = "#{}-cells".format(basename) + prop_name = f"#{basename}-cells" if prop_name not in node.props: - _err("expected '{}' property on {!r} (referenced by {!r})" - .format(prop_name, node, child)) + _err(f"expected '{prop_name}' property on {node!r} " + f"(referenced by {child!r})") return node.props[prop_name].to_num() # Do not require -controller for anything but interrupts for now @@ -2289,8 +2660,8 @@ def _map(prefix, child, parent, child_spec, spec_len_fn, require_controller): map_prop = parent.props.get(prefix + "-map") if not map_prop: if require_controller and prefix + "-controller" not in parent.props: - _err("expected '{}-controller' property on {!r} " - "(referenced by {!r})".format(prefix, parent, child)) + _err(f"expected '{prefix}-controller' property on {parent!r} " + f"(referenced by {child!r})") # No mapping return (parent, child_spec) @@ -2300,26 +2671,23 @@ def _map(prefix, child, parent, child_spec, spec_len_fn, require_controller): raw = map_prop.value while raw: if len(raw) < len(child_spec): - _err("bad value for {!r}, missing/truncated child data" - .format(map_prop)) + _err(f"bad value for {map_prop!r}, missing/truncated child data") child_spec_entry = raw[:len(child_spec)] raw = raw[len(child_spec):] if len(raw) < 4: - _err("bad value for {!r}, missing/truncated phandle" - .format(map_prop)) + _err(f"bad value for {map_prop!r}, missing/truncated phandle") phandle = to_num(raw[:4]) raw = raw[4:] # Parent specified in *-map map_parent = parent.dt.phandle2node.get(phandle) if not map_parent: - _err("bad phandle ({}) in {!r}".format(phandle, map_prop)) + _err(f"bad phandle ({phandle}) in {map_prop!r}") map_parent_spec_len = 4*spec_len_fn(map_parent) if len(raw) < map_parent_spec_len: - _err("bad value for {!r}, missing/truncated parent data" - .format(map_prop)) + _err(f"bad value for {map_prop!r}, missing/truncated parent data") parent_spec = raw[:map_parent_spec_len] raw = raw[map_parent_spec_len:] @@ -2333,8 +2701,8 @@ def _map(prefix, child, parent, child_spec, spec_len_fn, require_controller): return _map(prefix, parent, map_parent, parent_spec, spec_len_fn, require_controller) - _err("child specifier for {!r} ({}) does not appear in {!r}" - .format(child, child_spec, map_prop)) + _err(f"child specifier for {child!r} ({child_spec}) " + f"does not appear in {map_prop!r}") def _mask(prefix, child, parent, child_spec): @@ -2348,8 +2716,8 @@ def _mask(prefix, child, parent, child_spec): mask = mask_prop.value if len(mask) != len(child_spec): - _err("{!r}: expected '{}-mask' in {!r} to be {} bytes, is {} bytes" - .format(child, prefix, parent, len(child_spec), len(mask))) + _err(f"{child!r}: expected '{prefix}-mask' in {parent!r} " + f"to be {len(child_spec)} bytes, is {len(mask)} bytes") return _and(child_spec, mask) @@ -2370,8 +2738,8 @@ def _pass_thru(prefix, child, parent, child_spec, parent_spec): pass_thru = pass_thru_prop.value if len(pass_thru) != len(child_spec): - _err("{!r}: expected '{}-map-pass-thru' in {!r} to be {} bytes, is {} bytes" - .format(child, prefix, parent, len(child_spec), len(pass_thru))) + _err(f"{child!r}: expected '{prefix}-map-pass-thru' in {parent!r} " + f"to be {len(child_spec)} bytes, is {len(pass_thru)} bytes") res = _or(_and(child_spec, pass_thru), _and(parent_spec, _not(pass_thru))) @@ -2385,14 +2753,14 @@ def _raw_unit_addr(node): # #address-cells) as a raw 'bytes' if 'reg' not in node.props: - _err("{!r} lacks 'reg' property (needed for 'interrupt-map' unit " - "address lookup)".format(node)) + _err(f"{node!r} lacks 'reg' property " + "(needed for 'interrupt-map' unit address lookup)") addr_len = 4*_address_cells(node) if len(node.props['reg'].value) < addr_len: - _err("{!r} has too short 'reg' property (while doing 'interrupt-map' " - "unit address lookup)".format(node)) + _err(f"{node!r} has too short 'reg' property " + "(while doing 'interrupt-map' unit address lookup)") return node.props['reg'].value[:addr_len] @@ -2442,7 +2810,7 @@ def _phandle_val_list(prop, n_cells_name): # is the node pointed at by . If does not refer # to a node, the entire list element is None. - full_n_cells_name = "#{}-cells".format(n_cells_name) + full_n_cells_name = f"#{n_cells_name}-cells" res = [] @@ -2462,7 +2830,7 @@ def _phandle_val_list(prop, n_cells_name): continue if full_n_cells_name not in node.props: - _err("{!r} lacks {}".format(node, full_n_cells_name)) + _err(f"{node!r} lacks {full_n_cells_name}") n_cells = node.props[full_n_cells_name].to_num() if len(raw) < 4*n_cells: @@ -2497,7 +2865,7 @@ def _interrupt_cells(node): # 'node' has no #interrupt-cells property if "#interrupt-cells" not in node.props: - _err("{!r} lacks #interrupt-cells".format(node)) + _err(f"{node!r} lacks #interrupt-cells") return node.props["#interrupt-cells"].to_num() @@ -2509,11 +2877,10 @@ def _slice(node, prop_name, size, size_hint): raw = node.props[prop_name].value if len(raw) % size: - _err("'{}' property in {!r} has length {}, which is not evenly " - "divisible by {} (= {}). Note that #*-cells " - "properties come either from the parent node or from the " - "controller (in the case of 'interrupts')." - .format(prop_name, node, len(raw), size, size_hint)) + _err(f"'{prop_name}' property in {node!r} has length {len(raw)}, " + f"which is not evenly divisible by {size} (= {size_hint}). " + "Note that #*-cells properties come either from the parent node or " + "from the controller (in the case of 'interrupts').") return [raw[i:i + size] for i in range(0, len(raw), size)] @@ -2537,17 +2904,17 @@ def _check_dt(dt): _err(str(e)) if status_val not in ok_status: - _err("unknown 'status' value \"{}\" in {} in {}, expected one " - "of {} (see the devicetree specification)" - .format(status_val, node.path, node.dt.filename, - ", ".join(ok_status))) + _err(f"unknown 'status' value \"{status_val}\" in {node.path} " + f"in {node.dt.filename}, expected one of " + + ", ".join(ok_status) + + " (see the devicetree specification)") ranges_prop = node.props.get("ranges") if ranges_prop: - if ranges_prop.type not in (TYPE_EMPTY, TYPE_NUMS): - _err("expected 'ranges = < ... >;' in {} in {}, not '{}' " - "(see the devicetree specification)" - .format(node.path, node.dt.filename, ranges_prop)) + if ranges_prop.type not in (Type.EMPTY, Type.NUMS): + _err(f"expected 'ranges = < ... >;' in {node.path} in " + f"{node.dt.filename}, not '{ranges_prop}' " + "(see the devicetree specification)") def _err(msg): @@ -2559,6 +2926,11 @@ _LOG = logging.getLogger(__name__) # Regular expression for non-alphanumeric-or-underscore characters. _NOT_ALPHANUM_OR_UNDERSCORE = re.compile(r'\W', re.ASCII) + +def _val_as_token(val): + return re.sub(_NOT_ALPHANUM_OR_UNDERSCORE, '_', val) + + # Custom PyYAML binding loader class to avoid modifying yaml.Loader directly, # which could interfere with YAML loading in clients class _BindingLoader(Loader): @@ -2626,3 +2998,14 @@ _DEFAULT_PROP_SPECS = { name: PropertySpec(name, _DEFAULT_PROP_BINDING) for name in _DEFAULT_PROP_TYPES } + +# A set of vendor prefixes which are grandfathered in by Linux, +# and therefore by us as well. +_VENDOR_PREFIX_ALLOWED = set([ + "at25", "bm", "devbus", "dmacap", "dsa", + "exynos", "fsia", "fsib", "gpio-fan", "gpio-key", "gpio", "gpmc", + "hdmi", "i2c-gpio", "keypad", "m25p", "max8952", "max8997", + "max8998", "mpmc", "pinctrl-single", "#pinctrl-single", "PowerPC", + "pl022", "pxa-mmc", "rcar_sound", "rotary-encoder", "s5m8767", + "sdhci", "simple-audio-card", "st-plgpio", "st-spics", "ts", +]) diff --git a/tests/test-bindings-include/README.rst b/tests/test-bindings-include/README.rst new file mode 100644 index 0000000..c8ab889 --- /dev/null +++ b/tests/test-bindings-include/README.rst @@ -0,0 +1 @@ +This directory contains bindings used to test the 'include:' feature. diff --git a/tests/test-bindings-include/allow-and-blocklist-child.yaml b/tests/test-bindings-include/allow-and-blocklist-child.yaml new file mode 100644 index 0000000..1771120 --- /dev/null +++ b/tests/test-bindings-include/allow-and-blocklist-child.yaml @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: BSD-3-Clause + +description: | + An include must not give both an allowlist and a blocklist in a + child binding. This binding should cause an error. +compatible: allow-and-blocklist-child +include: + - name: include.yaml + child-binding: + property-blocklist: [x] + property-allowlist: [y] diff --git a/tests/test-bindings-include/allow-and-blocklist.yaml b/tests/test-bindings-include/allow-and-blocklist.yaml new file mode 100644 index 0000000..80af465 --- /dev/null +++ b/tests/test-bindings-include/allow-and-blocklist.yaml @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: BSD-3-Clause + +description: | + An include must not give both an allowlist and a blocklist. + This binding should cause an error. +compatible: allow-and-blocklist +include: + - name: include.yaml + property-blocklist: [x] + property-allowlist: [y] diff --git a/tests/test-bindings-include/allow-not-list.yaml b/tests/test-bindings-include/allow-not-list.yaml new file mode 100644 index 0000000..4ec3d87 --- /dev/null +++ b/tests/test-bindings-include/allow-not-list.yaml @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: BSD-3-Clause + +description: | + A property-allowlist, if given, must be a list. This binding should + cause an error. +compatible: allow-not-list +include: + - name: include.yaml + property-allowlist: + foo: diff --git a/tests/test-bindings-include/allowlist.yaml b/tests/test-bindings-include/allowlist.yaml new file mode 100644 index 0000000..4cd2c52 --- /dev/null +++ b/tests/test-bindings-include/allowlist.yaml @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: BSD-3-Clause + +description: Valid property-allowlist. +compatible: allowlist +include: + - name: include.yaml + property-allowlist: [x] diff --git a/tests/test-bindings-include/block-not-list.yaml b/tests/test-bindings-include/block-not-list.yaml new file mode 100644 index 0000000..bf58da5 --- /dev/null +++ b/tests/test-bindings-include/block-not-list.yaml @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: BSD-3-Clause + +description: | + A property-blocklist, if given, must be a list. This binding should + cause an error. +compatible: block-not-list +include: + - name: include.yaml + property-blocklist: + foo: diff --git a/tests/test-bindings-include/blocklist.yaml b/tests/test-bindings-include/blocklist.yaml new file mode 100644 index 0000000..a56db02 --- /dev/null +++ b/tests/test-bindings-include/blocklist.yaml @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: BSD-3-Clause + +description: Valid property-blocklist. +compatible: blocklist +include: + - name: include.yaml + property-blocklist: [x] diff --git a/tests/test-bindings-include/empty-allowlist.yaml b/tests/test-bindings-include/empty-allowlist.yaml new file mode 100644 index 0000000..c217d06 --- /dev/null +++ b/tests/test-bindings-include/empty-allowlist.yaml @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: BSD-3-Clause + +description: An empty property-allowlist is valid. +compatible: empty-allowlist +include: + - name: include.yaml + property-allowlist: [] diff --git a/tests/test-bindings-include/empty-blocklist.yaml b/tests/test-bindings-include/empty-blocklist.yaml new file mode 100644 index 0000000..76bed12 --- /dev/null +++ b/tests/test-bindings-include/empty-blocklist.yaml @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: BSD-3-Clause + +description: An empty property-blocklist is valid. +compatible: empty-blocklist +include: + - name: include.yaml + property-blocklist: [] diff --git a/tests/test-bindings-include/filter-child-bindings.yaml b/tests/test-bindings-include/filter-child-bindings.yaml new file mode 100644 index 0000000..1547e36 --- /dev/null +++ b/tests/test-bindings-include/filter-child-bindings.yaml @@ -0,0 +1,11 @@ +description: Test binding for filtering 'child-binding' properties + +include: + - name: include.yaml + property-allowlist: [x] + child-binding: + property-blocklist: [child-prop-1] + child-binding: + property-allowlist: [grandchild-prop-1] + +compatible: filter-child-bindings diff --git a/tests/test-bindings-include/include-2.yaml b/tests/test-bindings-include/include-2.yaml new file mode 100644 index 0000000..6fe5822 --- /dev/null +++ b/tests/test-bindings-include/include-2.yaml @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: BSD-3-Clause + +description: Second file for testing "intermixed" includes. +compatible: include-2 +properties: + a: + type: int diff --git a/tests/test-bindings-include/include-invalid-keys.yaml b/tests/test-bindings-include/include-invalid-keys.yaml new file mode 100644 index 0000000..1b75430 --- /dev/null +++ b/tests/test-bindings-include/include-invalid-keys.yaml @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: BSD-3-Clause + +description: | + Invalid include element: invalid keys are present. +compatible: include-invalid-keys +include: + - name: include.yaml + property-allowlist: [x] + bad-key-1: 3 + bad-key-2: 3 diff --git a/tests/test-bindings-include/include-invalid-type.yaml b/tests/test-bindings-include/include-invalid-type.yaml new file mode 100644 index 0000000..a11085d --- /dev/null +++ b/tests/test-bindings-include/include-invalid-type.yaml @@ -0,0 +1,5 @@ +description: | + Invalid include: wrong top level type. +compatible: include-invalid-type +include: + a-map-is-not-allowed-here: 3 diff --git a/tests/test-bindings-include/include-no-list.yaml b/tests/test-bindings-include/include-no-list.yaml new file mode 100644 index 0000000..940e824 --- /dev/null +++ b/tests/test-bindings-include/include-no-list.yaml @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: BSD-3-Clause + +description: A map element with just a name is valid, and has no filters. +compatible: include-no-list +include: + - name: include.yaml diff --git a/tests/test-bindings-include/include-no-name.yaml b/tests/test-bindings-include/include-no-name.yaml new file mode 100644 index 0000000..3202879 --- /dev/null +++ b/tests/test-bindings-include/include-no-name.yaml @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: BSD-3-Clause + +description: | + Invalid include element: no name key is present. +compatible: include-no-name +include: + - property-allowlist: [x] diff --git a/tests/test-bindings-include/include.yaml b/tests/test-bindings-include/include.yaml new file mode 100644 index 0000000..4559920 --- /dev/null +++ b/tests/test-bindings-include/include.yaml @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: BSD-3-Clause + +description: Test file for including other bindings +compatible: include +properties: + x: + type: int + y: + type: int + z: + type: int +child-binding: + properties: + child-prop-1: + type: int + child-prop-2: + type: int + + child-binding: + properties: + grandchild-prop-1: + type: int + grandchild-prop-2: + type: int diff --git a/tests/test-bindings-include/intermixed.yaml b/tests/test-bindings-include/intermixed.yaml new file mode 100644 index 0000000..2837a3e --- /dev/null +++ b/tests/test-bindings-include/intermixed.yaml @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: BSD-3-Clause + +description: Including intermixed file names and maps is valid. +compatible: intermixed +include: + - name: include.yaml + property-allowlist: [x] + - include-2.yaml diff --git a/tests/test-wrong-bindings/wrong-phandle-array-name.yaml b/tests/test-wrong-bindings/wrong-phandle-array-name.yaml new file mode 100644 index 0000000..b66b36c --- /dev/null +++ b/tests/test-wrong-bindings/wrong-phandle-array-name.yaml @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: BSD-3-Clause + +description: Device.wrong_phandle_array_name test + +compatible: "wrong_phandle_array_name" + +properties: + wrong-phandle-array-name: + type: phandle-array diff --git a/tests/test-wrong-bindings/wrong-specifier-space-type.yaml b/tests/test-wrong-bindings/wrong-specifier-space-type.yaml new file mode 100644 index 0000000..d6aa7ae --- /dev/null +++ b/tests/test-wrong-bindings/wrong-specifier-space-type.yaml @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: BSD-3-Clause + +description: Device.wrong_specifier_space_type test + +compatible: "wrong_specifier_space_type" + +properties: + wrong-type-for-specifier-space: + type: phandle + specifier-space: foobar diff --git a/tests/test.dts b/tests/test.dts index 0e077fd..256b672 100644 --- a/tests/test.dts +++ b/tests/test.dts @@ -123,6 +123,110 @@ }; }; + // + // 'ranges' + // + + ranges-zero-cells { + #address-cells = <0>; + + node { + #address-cells = <0>; + #size-cells = <0>; + + ranges; + }; + }; + + ranges-zero-parent-cells { + #address-cells = <0>; + + node { + #address-cells = <1>; + #size-cells = <0>; + + ranges = <0xA>, + <0x1A>, + <0x2A>; + }; + }; + + ranges-one-address-cells { + #address-cells = <0>; + + node { + reg = <1>; + #address-cells = <1>; + + ranges = <0xA 0xB>, + <0x1A 0x1B>, + <0x2A 0x2B>; + }; + }; + + ranges-one-address-two-size-cells { + #address-cells = <0>; + + node { + reg = <1>; + #address-cells = <1>; + #size-cells = <2>; + + ranges = <0xA 0xB 0xC>, + <0x1A 0x1B 0x1C>, + <0x2A 0x2B 0x2C>; + }; + }; + + ranges-two-address-cells { + #address-cells = <1>; + + node@1 { + reg = <1 2>; + + ranges = <0xA 0xB 0xC 0xD>, + <0x1A 0x1B 0x1C 0x1D>, + <0x2A 0x2B 0x2C 0x2D>; + }; + }; + + ranges-two-address-two-size-cells { + #address-cells = <1>; + + node@1 { + reg = <1 2>; + #size-cells = <2>; + + ranges = <0xA 0xB 0xC 0xD 0xE>, + <0x1A 0x1B 0x1C 0x1D 0x1E>, + <0x2A 0x2B 0x2C 0x2D 0x1D>; + }; + }; + + ranges-three-address-cells { + node@1 { + reg = <0 1 2>; + #address-cells = <3>; + + ranges = <0xA 0xB 0xC 0xD 0xE 0xF>, + <0x1A 0x1B 0x1C 0x1D 0x1E 0x1F>, + <0x2A 0x2B 0x2C 0x2D 0x2E 0x2F>; + }; + }; + + ranges-three-address-two-size-cells { + node@1 { + reg = <0 1 2>; + #address-cells = <3>; + #size-cells = <2>; + + ranges = <0xA 0xB 0xC 0xD 0xE 0xF 0x10>, + <0x1A 0x1B 0x1C 0x1D 0x1E 0x1F 0x110>, + <0x2A 0x2B 0x2C 0x2D 0x2E 0x2F 0x210>; + }; + }; + + // // 'reg' // @@ -206,7 +310,7 @@ }; // - // For testing Node.parent and Node.children + // For testing hierarchy. // parent { diff --git a/tests/test_dtlib.py b/tests/test_dtlib.py index ac35e85..60717bb 100644 --- a/tests/test_dtlib.py +++ b/tests/test_dtlib.py @@ -14,7 +14,7 @@ from devicetree import dtlib # # Run it using pytest (https://docs.pytest.org/en/stable/usage.html): # -# $ pytest testdtlib.py +# $ pytest tests/test_dtlib.py # # Extra options you can pass to pytest for debugging: # @@ -24,13 +24,15 @@ from devicetree import dtlib # - to run a particular test function or functions, use # '-k test_function_pattern_goes_here' -def parse(dts, include_path=()): - '''Parse a DTS string 'dts', using the given include path.''' +def parse(dts, include_path=(), **kwargs): + '''Parse a DTS string 'dts', using the given include path. + + Any kwargs are passed on to DT().''' fd, path = tempfile.mkstemp(prefix='pytest-', suffix='.dts') try: os.write(fd, dts.encode('utf-8')) - return dtlib.DT(path, include_path) + return dtlib.DT(path, include_path, **kwargs) finally: os.close(fd) os.unlink(path) @@ -98,6 +100,16 @@ def temporary_chdir(dirname): finally: os.chdir(here) +def test_invalid_nodenames(): + # Regression test that verifies node names are not matched against + # the more permissive set of rules used for property names. + + verify_error_endswith(""" +/dts-v1/; +/ { node? {}; }; +""", + "/node?: bad character '?' in node name") + def test_cell_parsing(): '''Miscellaneous properties containing zero or more cells''' @@ -1538,30 +1550,30 @@ def test_prop_type(): }; """) - verify_type("empty", dtlib.TYPE_EMPTY) - verify_type("bytes1", dtlib.TYPE_BYTES) - verify_type("bytes2", dtlib.TYPE_BYTES) - verify_type("bytes3", dtlib.TYPE_BYTES) - verify_type("bytes4", dtlib.TYPE_BYTES) - verify_type("bytes5", dtlib.TYPE_BYTES) - verify_type("num", dtlib.TYPE_NUM) - verify_type("nums1", dtlib.TYPE_NUMS) - verify_type("nums2", dtlib.TYPE_NUMS) - verify_type("nums3", dtlib.TYPE_NUMS) - verify_type("nums4", dtlib.TYPE_NUMS) - verify_type("string", dtlib.TYPE_STRING) - verify_type("strings", dtlib.TYPE_STRINGS) - verify_type("phandle1", dtlib.TYPE_PHANDLE) - verify_type("phandle2", dtlib.TYPE_PHANDLE) - verify_type("phandles1", dtlib.TYPE_PHANDLES) - verify_type("phandles2", dtlib.TYPE_PHANDLES) - verify_type("phandle-and-nums-1", dtlib.TYPE_PHANDLES_AND_NUMS) - verify_type("phandle-and-nums-2", dtlib.TYPE_PHANDLES_AND_NUMS) - verify_type("phandle-and-nums-3", dtlib.TYPE_PHANDLES_AND_NUMS) - verify_type("path1", dtlib.TYPE_PATH) - verify_type("path2", dtlib.TYPE_PATH) - verify_type("compound1", dtlib.TYPE_COMPOUND) - verify_type("compound2", dtlib.TYPE_COMPOUND) + verify_type("empty", dtlib.Type.EMPTY) + verify_type("bytes1", dtlib.Type.BYTES) + verify_type("bytes2", dtlib.Type.BYTES) + verify_type("bytes3", dtlib.Type.BYTES) + verify_type("bytes4", dtlib.Type.BYTES) + verify_type("bytes5", dtlib.Type.BYTES) + verify_type("num", dtlib.Type.NUM) + verify_type("nums1", dtlib.Type.NUMS) + verify_type("nums2", dtlib.Type.NUMS) + verify_type("nums3", dtlib.Type.NUMS) + verify_type("nums4", dtlib.Type.NUMS) + verify_type("string", dtlib.Type.STRING) + verify_type("strings", dtlib.Type.STRINGS) + verify_type("phandle1", dtlib.Type.PHANDLE) + verify_type("phandle2", dtlib.Type.PHANDLE) + verify_type("phandles1", dtlib.Type.PHANDLES) + verify_type("phandles2", dtlib.Type.PHANDLES) + verify_type("phandle-and-nums-1", dtlib.Type.PHANDLES_AND_NUMS) + verify_type("phandle-and-nums-2", dtlib.Type.PHANDLES_AND_NUMS) + verify_type("phandle-and-nums-3", dtlib.Type.PHANDLES_AND_NUMS) + verify_type("path1", dtlib.Type.PATH) + verify_type("path2", dtlib.Type.PATH) + verify_type("compound1", dtlib.Type.COMPOUND) + verify_type("compound2", dtlib.Type.COMPOUND) def test_prop_type_casting(): '''Test Property.to_{num,nums,string,strings,node}()''' @@ -2077,7 +2089,7 @@ foo: / { def test_reprs(): '''Test the __repr__() functions.''' - dt = parse(""" + dts = """ /dts-v1/; / { @@ -2086,28 +2098,31 @@ def test_reprs(): y = < 1 >; }; }; -""", - include_path=("foo", "bar")) +""" - assert re.fullmatch(r"DT\(filename='.*', include_path=\('foo', 'bar'\)\)", + dt = parse(dts, include_path=("foo", "bar")) + + assert re.fullmatch(r"DT\(filename='.*', include_path=.'foo', 'bar'.\)", repr(dt)) assert re.fullmatch("", repr(dt.root.props["x"])) assert re.fullmatch("", repr(dt.root.nodes["sub"])) + dt = parse(dts, include_path=iter(("foo", "bar"))) + + assert re.fullmatch(r"DT\(filename='.*', include_path=.'foo', 'bar'.\)", + repr(dt)) + def test_names(): '''Tests for node/property names.''' - # The C tools disallow '@' in property names, but otherwise accept the same - # characters in node and property names. Emulate that instead of the DT spec - # (v0.2), which gives different characters for nodes and properties. verify_parse(r""" /dts-v1/; / { // A leading \ is accepted but ignored in node/propert names - \aA0,._+*#?- = &_, &{/aA0,._+*#?@-}; + \aA0,._+*#?- = &_, &{/aA0,._+@-}; // Names that overlap with operators and integer literals @@ -2118,7 +2133,8 @@ def test_names(): 0 = [ 04 ]; 0x123 = [ 05 ]; - _: \aA0,._+*#?@- { + // Node names are more restrictive than property names. + _: \aA0,._+@- { }; 0 { @@ -2129,14 +2145,14 @@ def test_names(): /dts-v1/; / { - aA0,._+*#?- = &_, &{/aA0,._+*#?@-}; + aA0,._+*#?- = &_, &{/aA0,._+@-}; + = [ 00 ]; * = [ 02 ]; - = [ 01 ]; ? = [ 03 ]; 0 = [ 04 ]; 0x123 = [ 05 ]; - _: aA0,._+*#?@- { + _: aA0,._+@- { }; 0 { }; @@ -2244,3 +2260,13 @@ l1: l2: &foo { / { }; """) + +def test_dangling_alias(): + dt = parse(''' +/dts-v1/; + +/ { + aliases { foo = "/missing"; }; +}; +''', force=True) + assert dt.get_node('/aliases').props['foo'].to_string() == '/missing' diff --git a/tests/test_edtlib.py b/tests/test_edtlib.py index 3a8e0cc..4879519 100644 --- a/tests/test_edtlib.py +++ b/tests/test_edtlib.py @@ -78,6 +78,43 @@ def test_interrupts(): assert str(edt.get_node("/interrupt-map-bitops-test/node@70000000E").interrupts) == \ f"[, data: OrderedDict([('one', 3), ('two', 2)])>]" +def test_ranges(): + '''Tests for the ranges property''' + with from_here(): + edt = edtlib.EDT("test.dts", ["test-bindings"]) + + assert str(edt.get_node("/reg-ranges/parent").ranges) == \ + "[, , ]" + + assert str(edt.get_node("/reg-nested-ranges/grandparent").ranges) == \ + "[]" + + assert str(edt.get_node("/reg-nested-ranges/grandparent/parent").ranges) == \ + "[]" + + assert str(edt.get_node("/ranges-zero-cells/node").ranges) == "[]" + + assert str(edt.get_node("/ranges-zero-parent-cells/node").ranges) == \ + "[, , ]" + + assert str(edt.get_node("/ranges-one-address-cells/node").ranges) == \ + "[, , ]" + + assert str(edt.get_node("/ranges-one-address-two-size-cells/node").ranges) == \ + "[, , ]" + + assert str(edt.get_node("/ranges-two-address-cells/node@1").ranges) == \ + "[, , ]" + + assert str(edt.get_node("/ranges-two-address-two-size-cells/node@1").ranges) == \ + "[, , ]" + + assert str(edt.get_node("/ranges-three-address-cells/node@1").ranges) == \ + "[, , ]" + + assert str(edt.get_node("/ranges-three-address-two-size-cells/node@1").ranges) == \ + "[, , ]" + def test_reg(): '''Tests for the regs property''' with from_here(): @@ -121,6 +158,20 @@ def test_hierarchy(): assert edt.get_node("/parent/child-1").children == {} +def test_child_index(): + '''Test Node.child_index.''' + with from_here(): + edt = edtlib.EDT("test.dts", ["test-bindings"]) + + parent, child_1, child_2 = [edt.get_node(path) for path in + ("/parent", + "/parent/child-1", + "/parent/child-2")] + assert parent.child_index(child_1) == 0 + assert parent.child_index(child_2) == 1 + with pytest.raises(KeyError): + parent.child_index(parent) + def test_include(): '''Test 'include:' and the legacy 'inherits: !include ...' in bindings''' with from_here(): @@ -132,6 +183,91 @@ def test_include(): assert str(edt.get_node("/binding-include").props) == \ "OrderedDict([('foo', ), ('bar', ), ('baz', ), ('qaz', )])" +def test_include_filters(): + '''Test property-allowlist and property-blocklist in an include.''' + + fname2path = {'include.yaml': 'test-bindings-include/include.yaml', + 'include-2.yaml': 'test-bindings-include/include-2.yaml'} + + with pytest.raises(edtlib.EDTError) as e: + with from_here(): + edtlib.Binding("test-bindings-include/allow-and-blocklist.yaml", fname2path) + assert ("should not specify both 'property-allowlist:' and 'property-blocklist:'" + in str(e.value)) + + with pytest.raises(edtlib.EDTError) as e: + with from_here(): + edtlib.Binding("test-bindings-include/allow-and-blocklist-child.yaml", fname2path) + assert ("should not specify both 'property-allowlist:' and 'property-blocklist:'" + in str(e.value)) + + with pytest.raises(edtlib.EDTError) as e: + with from_here(): + edtlib.Binding("test-bindings-include/allow-not-list.yaml", fname2path) + value_str = str(e.value) + assert value_str.startswith("'property-allowlist' value") + assert value_str.endswith("should be a list") + + with pytest.raises(edtlib.EDTError) as e: + with from_here(): + edtlib.Binding("test-bindings-include/block-not-list.yaml", fname2path) + value_str = str(e.value) + assert value_str.startswith("'property-blocklist' value") + assert value_str.endswith("should be a list") + + with pytest.raises(edtlib.EDTError) as e: + with from_here(): + binding = edtlib.Binding("test-bindings-include/include-invalid-keys.yaml", fname2path) + value_str = str(e.value) + assert value_str.startswith( + "'include:' in test-bindings-include/include-invalid-keys.yaml should not have these " + "unexpected contents: ") + assert 'bad-key-1' in value_str + assert 'bad-key-2' in value_str + + with pytest.raises(edtlib.EDTError) as e: + with from_here(): + binding = edtlib.Binding("test-bindings-include/include-invalid-type.yaml", fname2path) + value_str = str(e.value) + assert value_str.startswith( + "'include:' in test-bindings-include/include-invalid-type.yaml " + "should be a string or list, but has type ") + + with pytest.raises(edtlib.EDTError) as e: + with from_here(): + binding = edtlib.Binding("test-bindings-include/include-no-name.yaml", fname2path) + value_str = str(e.value) + assert value_str.startswith("'include:' element") + assert value_str.endswith( + "in test-bindings-include/include-no-name.yaml should have a 'name' key") + + with from_here(): + binding = edtlib.Binding("test-bindings-include/allowlist.yaml", fname2path) + assert set(binding.prop2specs.keys()) == {'x'} # 'x' is allowed + + binding = edtlib.Binding("test-bindings-include/empty-allowlist.yaml", fname2path) + assert set(binding.prop2specs.keys()) == set() # nothing is allowed + + binding = edtlib.Binding("test-bindings-include/blocklist.yaml", fname2path) + assert set(binding.prop2specs.keys()) == {'y', 'z'} # 'x' is blocked + + binding = edtlib.Binding("test-bindings-include/empty-blocklist.yaml", fname2path) + assert set(binding.prop2specs.keys()) == {'x', 'y', 'z'} # nothing is blocked + + binding = edtlib.Binding("test-bindings-include/intermixed.yaml", fname2path) + assert set(binding.prop2specs.keys()) == {'x', 'a'} + + binding = edtlib.Binding("test-bindings-include/include-no-list.yaml", fname2path) + assert set(binding.prop2specs.keys()) == {'x', 'y', 'z'} + + binding = edtlib.Binding("test-bindings-include/filter-child-bindings.yaml", fname2path) + child = binding.child_binding + grandchild = child.child_binding + assert set(binding.prop2specs.keys()) == {'x'} + assert set(child.prop2specs.keys()) == {'child-prop-2'} + assert set(grandchild.prop2specs.keys()) == {'grandchild-prop-1'} + + def test_bus(): '''Test 'bus:' and 'on-bus:' in bindings''' with from_here(): @@ -272,6 +408,8 @@ def test_nexus(): assert str(edt.get_node("/gpio-map/source").props["foo-gpios"]) == \ f", data: OrderedDict([('val', 6)])>, , data: OrderedDict([('val', 5)])>]>" + assert str(edt.get_node("/gpio-map/source").props["foo-gpios"].val[0].basename) == f"gpio" + def test_prop_defaults(): '''Test property default values given in bindings''' with from_here(): @@ -413,6 +551,40 @@ def test_slice_errs(tmp_path): dts_file, f"'ranges' property in has length 8, which is not evenly divisible by 24 (= 4*(<#address-cells> (= 2) + <#address-cells for parent> (= 1) + <#size-cells> (= 3))). Note that #*-cells properties come either from the parent node or from the controller (in the case of 'interrupts').") +def test_bad_compatible(tmp_path): + # An invalid compatible should cause an error, even on a node with + # no binding. + + dts_file = tmp_path / "error.dts" + + verify_error(""" +/dts-v1/; + +/ { + foo { + compatible = "no, whitespace"; + }; +}; +""", + dts_file, + r"node '/foo' compatible 'no, whitespace' must match this regular expression: '^[a-zA-Z][a-zA-Z0-9,+\-._]+$'") + +def test_wrong_props(): + '''Test Node.wrong_props (derived from DT and 'properties:' in the binding)''' + + with from_here(): + with pytest.raises(edtlib.EDTError) as e: + edtlib.Binding("test-wrong-bindings/wrong-specifier-space-type.yaml", None) + assert ("'specifier-space' in 'properties: wrong-type-for-specifier-space' has type 'phandle', expected 'phandle-array'" + in str(e.value)) + + with pytest.raises(edtlib.EDTError) as e: + edtlib.Binding("test-wrong-bindings/wrong-phandle-array-name.yaml", None) + value_str = str(e.value) + assert value_str.startswith("'wrong-phandle-array-name' in 'properties:'") + assert value_str.endswith("but no 'specifier-space' was provided.") + + def verify_error(dts, dts_file, expected_err): # Verifies that parsing a file 'dts_file' with the contents 'dts' # (a string) raises an EDTError with the message 'expected_err'. diff --git a/tox.ini b/tox.ini index 59d9304..54b08ec 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,15 @@ envlist=py3 deps = setuptools-scm pytest + types-PyYAML + mypy setenv = TOXTEMPDIR={envtmpdir} commands = python -m pytest {posargs:tests} + python -m mypy --config-file={toxinidir}/tox.ini --package=devicetree + +[mypy] +mypy_path=src +ignore_missing_imports=True +