Skip to content

scanner

Scanner

Scans a given text and returns tokens based on the provided configuration.

Example

from lectes import Rule, Configuration, Regex, Scanner

config = Configuration(
    [
        Rule(name="FOR", regex=Regex("for")),
        Rule(name="INT", regex=Regex("[1-9]+")),
        Rule(name="ID", regex=Regex("[a-zA-Z][a-zA-Z0-9]*")),
        Rule(name="WHITESPACE", regex=Regex("( )")),
    ]
)

scanner = Scanner(config)
program = "somevar in othervar for 9 let"

for token in scanner.scan(program):
    print(token)
Source code in src/lectes/scanner/scanner.py 
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
class Scanner:
    """
    Scans a given text and returns tokens based on the provided configuration.

    ## Example

    ```python
    from lectes import Rule, Configuration, Regex, Scanner

    config = Configuration(
        [
            Rule(name="FOR", regex=Regex("for")),
            Rule(name="INT", regex=Regex("[1-9]+")),
            Rule(name="ID", regex=Regex("[a-zA-Z][a-zA-Z0-9]*")),
            Rule(name="WHITESPACE", regex=Regex("( )")),
        ]
    )

    scanner = Scanner(config)
    program = "somevar in othervar for 9 let"

    for token in scanner.scan(program):
        print(token)
    ```
    """

    def __init__(self, configuration: Configuration, debug: bool = False) -> None:
        self.configuration = configuration
        self.set_text("")
        self._unmatched_handler = self._handle_unmatched
        self._matched_handlers = {
            rule: self._handle_matched for rule in configuration.rules
        }
        self._debug = debug
        self._logger = None
        self._match = None
        self._matched_rule = None

    def scan(self, text: str) -> Generator[Token]:
        """
        Scan the given text and yield tokens as they are recognized.
        """
        if len(text) == 0:
            return

        self.set_text(text)

        for character in text:
            self.logger().debug(f"character: '{character}'")

            current_string = self.current_string()
            self.logger().debug(f"current_string: '{current_string}'")

            lookahead_string = self.lookahead_string()
            self.logger().debug(f"lookahead_string: '{lookahead_string}'")

            for rule in self.configuration.rules:
                if not self._is_last_char() and rule.regex.fullmatch(lookahead_string):
                    self.logger().debug(
                        f"rule {rule.name} fullmatched lookahead_string: '{lookahead_string}'"
                    )
                    break

                if match := rule.regex.search(current_string):
                    self.logger().debug(
                        f"rule {rule.name} matched current_string: '{current_string}'"
                    )
                    self._update_matched_state(rule, match)

            if self._match is not None:
                if self._match.unmatched is not None:
                    self._unmatched_handler(self._match.unmatched)

                if self._matched_rule is not None:
                    result = self._matched_handlers[self._matched_rule](
                        self._match.string, self._matched_rule
                    )

                    if result is not None:
                        yield result

                self.last_position = self.current_position

            self.current_position += 1
            self._reset_matched_state()

    def set_unmatched_handler(self, handler: Callable[[str], None]) -> None:
        """
        Set the given function as the handler that executes when a string is not
        matched to a configured rule.

        The handler receives the string as argument and does returns None.
        """
        self._unmatched_handler = handler

    def set_handler(self, rule: Rule, handler: Callable[[str, Rule], Any]) -> None:
        """
        Set the given function as the handler that executes when a string is matched
        against rule.

        The handler should receive the matched string literal and the rule as arguments.
        """
        self._matched_handlers[rule] = handler

    def set_text(self, text: str) -> None:
        """
        Set the text to scan.
        """
        self.text = text
        self.current_position = 1
        self.last_position = 0

    def current_string(self) -> str:
        """
        Return the string that the scanner is currently reading; that is,
        the characters from the last matched string up to the character that
        the scanner is currently reading.
        """
        return self.text[self.last_position : self.current_position]

    def lookahead_string(self) -> str:
        """
        Return the string that the scanner is currently reading plus one character.
        """
        return self.text[self.last_position : self.current_position + 1]

    def logger(self) -> Logger:
        """
        Return the scanner's logger instance.
        """
        if self._logger is None:
            self._logger = self._build_logger()

        return self._logger

    def _build_logger(self) -> Logger:
        logger = Logger()

        if self._debug:
            logger.set_level(LogLevel.DEBUG)

        return logger

    def _is_last_char(self) -> bool:
        return self.current_position == len(self.text)

    def _update_matched_state(self, rule: Rule, match: Match) -> None:
        if self._match is None or len(match) > len(self._match):
            self.logger().debug(f"updating match from {self._match} to {match.string}")
            self._matched_rule = rule
            self._match = match

    def _reset_matched_state(self) -> None:
        self._match = None
        self._matched_rule = None

    @staticmethod
    def _handle_unmatched(unmatched: str) -> None:
        print(f"unmatched: {unmatched}")

    @staticmethod
    def _handle_matched(matched: str, rule: Rule) -> Token:
        return Token(rule=rule, literal=matched)

current_string()

Return the string that the scanner is currently reading; that is, the characters from the last matched string up to the character that the scanner is currently reading.

Source code in src/lectes/scanner/scanner.py 
121
122
123
124
125
126
127
def current_string(self) -> str:
    """
    Return the string that the scanner is currently reading; that is,
    the characters from the last matched string up to the character that
    the scanner is currently reading.
    """
    return self.text[self.last_position : self.current_position]

logger()

Return the scanner's logger instance.

Source code in src/lectes/scanner/scanner.py 
135
136
137
138
139
140
141
142
def logger(self) -> Logger:
    """
    Return the scanner's logger instance.
    """
    if self._logger is None:
        self._logger = self._build_logger()

    return self._logger

lookahead_string()

Return the string that the scanner is currently reading plus one character.

Source code in src/lectes/scanner/scanner.py 
129
130
131
132
133
def lookahead_string(self) -> str:
    """
    Return the string that the scanner is currently reading plus one character.
    """
    return self.text[self.last_position : self.current_position + 1]

scan(text)

Scan the given text and yield tokens as they are recognized.

Source code in src/lectes/scanner/scanner.py 
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
def scan(self, text: str) -> Generator[Token]:
    """
    Scan the given text and yield tokens as they are recognized.
    """
    if len(text) == 0:
        return

    self.set_text(text)

    for character in text:
        self.logger().debug(f"character: '{character}'")

        current_string = self.current_string()
        self.logger().debug(f"current_string: '{current_string}'")

        lookahead_string = self.lookahead_string()
        self.logger().debug(f"lookahead_string: '{lookahead_string}'")

        for rule in self.configuration.rules:
            if not self._is_last_char() and rule.regex.fullmatch(lookahead_string):
                self.logger().debug(
                    f"rule {rule.name} fullmatched lookahead_string: '{lookahead_string}'"
                )
                break

            if match := rule.regex.search(current_string):
                self.logger().debug(
                    f"rule {rule.name} matched current_string: '{current_string}'"
                )
                self._update_matched_state(rule, match)

        if self._match is not None:
            if self._match.unmatched is not None:
                self._unmatched_handler(self._match.unmatched)

            if self._matched_rule is not None:
                result = self._matched_handlers[self._matched_rule](
                    self._match.string, self._matched_rule
                )

                if result is not None:
                    yield result

            self.last_position = self.current_position

        self.current_position += 1
        self._reset_matched_state()

set_handler(rule, handler)

Set the given function as the handler that executes when a string is matched against rule.

The handler should receive the matched string literal and the rule as arguments.

Source code in src/lectes/scanner/scanner.py 
104
105
106
107
108
109
110
111
def set_handler(self, rule: Rule, handler: Callable[[str, Rule], Any]) -> None:
    """
    Set the given function as the handler that executes when a string is matched
    against rule.

    The handler should receive the matched string literal and the rule as arguments.
    """
    self._matched_handlers[rule] = handler

set_text(text)

Set the text to scan.

Source code in src/lectes/scanner/scanner.py 
113
114
115
116
117
118
119
def set_text(self, text: str) -> None:
    """
    Set the text to scan.
    """
    self.text = text
    self.current_position = 1
    self.last_position = 0

set_unmatched_handler(handler)

Set the given function as the handler that executes when a string is not matched to a configured rule.

The handler receives the string as argument and does returns None.

Source code in src/lectes/scanner/scanner.py 
 95
 96
 97
 98
 99
100
101
102
def set_unmatched_handler(self, handler: Callable[[str], None]) -> None:
    """
    Set the given function as the handler that executes when a string is not
    matched to a configured rule.

    The handler receives the string as argument and does returns None.
    """
    self._unmatched_handler = handler