Implement context sensitive grammar conversion

Author: v--

src/notebook/math/grammars/context_sensitive.py

@@ -4,66 +4,86 @@
 from ...support.coderefs import collector
 from .epsilon_rules import is_epsilon_rule
 from .exceptions import IncompatibleGrammarError
-from .symbols import Terminal
+from .symbols import Terminal, new_non_terminal
+from .length_increasing import is_essentially_length_increasing_grammar
+from .grammar import GrammarRule, Grammar, GrammarSchema
 
+from .symbols import NonTerminal
 
-if TYPE_CHECKING:
-    from .grammar import Grammar, GrammarSchema
-    from .symbols import NonTerminal
 
+def is_context_sensitive_rule(rule: GrammarRule) -> bool:
+    for i in range(len(rule.src)):
+        if not isinstance(rule.src[i], NonTerminal) or rule.src[:i] != rule.dest[:i]:
+            continue
 
-def is_length_increasing_grammar(grammar: Grammar) -> bool:
-    return all(len(rule.src) <= len(rule.dest) for rule in grammar.schema.rules)
+        for j in range(len(rule.dest), i, -1):
+            if rule.src[i + 1:] == rule.dest[j:]:
+                return True
+
+    return False
 
 
-def is_essentially_length_increasing_grammar(grammar: Grammar) -> bool:
+def is_context_sensitive_grammar(grammar: Grammar) -> bool:
     return all(
-        (is_epsilon_rule(rule) and rule.src == [grammar.start]) or len(rule.src) <= len(rule.dest)
+        (is_epsilon_rule(rule) and rule.src == [grammar.start]) or is_context_sensitive_rule(rule)
         for rule in grammar.schema.rules
     )
 
 
-def _iter_all_derivations_one_step(schema: GrammarSchema, current: Iterable[Sequence[Terminal | NonTerminal]]) -> Iterable[Sequence[Terminal | NonTerminal]]:
-    for derivation in current:
-        for rule in schema.rules:
-            if is_epsilon_rule(rule):
-                continue
+@collector.ref('alg:length_increasing_to_context_sensitive')
+def length_increasing_to_context_sensitive(grammar: Grammar) -> Grammar:
+    if not is_essentially_length_increasing_grammar(grammar):
+        raise IncompatibleGrammarError('Expected an essentially length-increasing grammar')
 
-            for i in range(len(derivation) - len(rule.src) + 1):
-                if rule.src == derivation[i: len(rule.src) + i]:
-                    yield [*derivation[:i], *rule.dest, *derivation[len(rule.src) + i:]]
+    non_terminals = list(grammar.schema.get_non_terminals())
+    terminal_map = dict[Terminal, NonTerminal]()
+    current_schema = GrammarSchema()
 
+    for sym in grammar.schema.get_terminals():
+        active_non_terminal = new_non_terminal(sym.value, non_terminals)
+        current_schema.rules.append(
+            GrammarRule(
+                src=[active_non_terminal],
+                dest=[sym]
+            )
+        )
 
-def iter_derivations(grammar: Grammar, max_derivation_length: int) -> Iterable[Sequence[Terminal]]:
-    if not is_essentially_length_increasing_grammar(grammar):
-        raise IncompatibleGrammarError('Expected an essentially ε-free grammar')
+        terminal_map[sym] = active_non_terminal
+        non_terminals.append(active_non_terminal)
 
+    # We iterate over rules from the original grammar and modify the result at every step
     for rule in grammar.schema.rules:
         if is_epsilon_rule(rule):
-            yield []
-
-    derivable: Sequence[Sequence[Terminal | NonTerminal]] = [[grammar.start]]
-
-    for _ in range(max_derivation_length):
-        derivable = list(_iter_all_derivations_one_step(grammar.schema, derivable))
-
-        for derivation in derivable:
-            if all(isinstance(sym, Terminal) for sym in derivation):
-                yield cast(Sequence[Terminal], derivation)
-
-
-@collector.ref('alg:context_sensitive_string_membership')
-def naive_membership(grammar: Grammar, string: str) -> bool:
-    if not is_essentially_length_increasing_grammar(grammar):
-        raise IncompatibleGrammarError('Expected an essentially ε-free grammar')
-
-    m = len(grammar.schema.get_non_terminals()) + len(grammar.schema.get_terminals())
-    n = len(string)
-
-    max_derivation_length = (1 - m ** (n + 1)) // (1 - m) if m > 1 else n
-
-    for derivation in iter_derivations(grammar, max_derivation_length):
-        if ''.join(sym.value for sym in derivation) == string:
-            return True
-
-    return False
+            current_schema.rules.append(rule)
+            continue
+
+        modified_src = [terminal_map[sym] if isinstance(sym, Terminal) else sym for sym in rule.src]
+        modified_dest = [terminal_map[sym] if isinstance(sym, Terminal) else sym for sym in rule.dest]
+        dest_tail = modified_dest[len(rule.src):]
+        new_non_terminals = list[NonTerminal]()
+
+        for i, sym in enumerate(rule.src):
+            tail = modified_src[i + 1:] if i + 1 < len(rule.src) else dest_tail
+            active_non_terminal = new_non_terminal(sym.value, non_terminals)
+            current_schema.rules.append(
+                GrammarRule(
+                    src=[*new_non_terminals, *modified_src[i:]],
+                    dest=[*new_non_terminals, active_non_terminal, *tail]
+                )
+            )
+
+            new_non_terminals.append(active_non_terminal)
+            non_terminals.append(active_non_terminal)
+
+        for i in range(len(new_non_terminals)):
+            current_schema.rules.append(
+                GrammarRule(
+                    src=[*modified_dest[:i], *new_non_terminals[i:], *dest_tail],
+                    dest=[*modified_dest[:i + 1], *new_non_terminals[i + 1:], *dest_tail],
+                )
+            )
+
+    print()
+    print(current_schema)
+
+    return current_schema.instantiate(grammar.start)

src/notebook/math/grammars/epsilon_rules.py

@@ -1,5 +1,6 @@
 from typing import TYPE_CHECKING
 
+from ...support.coderefs import collector
 from .exceptions import IncompatibleRuleError
 from .grammar import Grammar, GrammarRule, GrammarSchema
 from .symbols import NonTerminal, Terminal, new_non_terminal
@@ -61,6 +62,7 @@ def iter_rules_without_nullables(nullable: Collection[NonTerminal], dest: Sequen
                 yield part
 
 
+@collector.ref('alg:epsilon_rule_removal')
 def remove_epsilon_rules(grammar: Grammar) -> Grammar:
     nullable = identify_nullable_non_terminals(grammar)
     new_start = new_non_terminal(

src/notebook/math/grammars/length_increasing.py

@@ -0,0 +1,69 @@
+from collections.abc import Iterable, Sequence
+from typing import TYPE_CHECKING, cast
+
+from ...support.coderefs import collector
+from .epsilon_rules import is_epsilon_rule
+from .exceptions import IncompatibleGrammarError
+from .symbols import Terminal
+
+
+if TYPE_CHECKING:
+    from .grammar import Grammar, GrammarSchema
+    from .symbols import NonTerminal
+
+
+def is_length_increasing_grammar(grammar: Grammar) -> bool:
+    return all(len(rule.src) <= len(rule.dest) for rule in grammar.schema.rules)
+
+
+def is_essentially_length_increasing_grammar(grammar: Grammar) -> bool:
+    return all(
+        (is_epsilon_rule(rule) and rule.src == [grammar.start]) or len(rule.src) <= len(rule.dest)
+        for rule in grammar.schema.rules
+    )
+
+
+def _iter_all_derivations_one_step(schema: GrammarSchema, current: Iterable[Sequence[Terminal | NonTerminal]]) -> Iterable[Sequence[Terminal | NonTerminal]]:
+    for derivation in current:
+        for rule in schema.rules:
+            if is_epsilon_rule(rule):
+                continue
+
+            for i in range(len(derivation) - len(rule.src) + 1):
+                if rule.src == derivation[i: len(rule.src) + i]:
+                    yield [*derivation[:i], *rule.dest, *derivation[len(rule.src) + i:]]
+
+
+def iter_derivations(grammar: Grammar, max_derivation_length: int) -> Iterable[Sequence[Terminal]]:
+    if not is_essentially_length_increasing_grammar(grammar):
+        raise IncompatibleGrammarError('Expected an essentially length-increasing grammar')
+
+    for rule in grammar.schema.rules:
+        if is_epsilon_rule(rule):
+            yield []
+
+    derivable: Sequence[Sequence[Terminal | NonTerminal]] = [[grammar.start]]
+
+    for _ in range(max_derivation_length):
+        derivable = list(_iter_all_derivations_one_step(grammar.schema, derivable))
+
+        for derivation in derivable:
+            if all(isinstance(sym, Terminal) for sym in derivation):
+                yield cast(Sequence[Terminal], derivation)
+
+
+@collector.ref('alg:context_sensitive_string_membership')
+def naive_membership(grammar: Grammar, string: str) -> bool:
+    if not is_essentially_length_increasing_grammar(grammar):
+        raise IncompatibleGrammarError('Expected an essentially length-increasing grammar')
+
+    m = len(grammar.schema.get_non_terminals()) + len(grammar.schema.get_terminals())
+    n = len(string)
+
+    max_derivation_length = (1 - m ** (n + 1)) // (1 - m) if m > 1 else n
+
+    for derivation in iter_derivations(grammar, max_derivation_length):
+        if ''.join(sym.value for sym in derivation) == string:
+            return True
+
+    return False

src/notebook/math/grammars/parsing/__init__.py

@@ -1 +1 @@
-from .parser import parse_grammar_rule_line, parse_grammar_schema, parse_nonterminal, parse_terminal
+from .parser import parse_grammar_schema, parse_nonterminal, parse_terminal

src/notebook/math/grammars/parsing/parser.py

@@ -158,7 +158,7 @@ def parse_rule_dest(self, context: GrammarSymbolRunContext) -> Iterable[Terminal
         if is_epsilon_rule == is_non_epsilon_rule:
             raise context.annotate_context_error('The right side of a rule must contain terminals and nonterminals and at most a single ε')
 
-    def iter_rules_on_line(self) -> Iterable[GrammarRule]:
+    def _iter_rules_on_line(self) -> Iterable[GrammarRule]:
         head = self._skip_spaces()
 
         if not head:
@@ -192,7 +192,10 @@ def iter_rules_on_line(self) -> Iterable[GrammarRule]:
                 if head.kind == 'LINE_BREAK':
                     return
 
-            if not head or head.kind != 'PIPE':
+            if head is None:
+                return
+
+            if head.kind != 'PIPE':
                 break
 
         while (head := self.peek()) and head.kind != 'LINE_BREAK':
@@ -201,15 +204,15 @@ def iter_rules_on_line(self) -> Iterable[GrammarRule]:
         context.close_at_previous_token()
         raise context.annotate_context_error('The right side of a rule must contain a pipe between runs of terminals, nonterminals and ε')
 
-    def iter_rules(self) -> Iterable[GrammarRule]:
+    def _iter_rules(self) -> Iterable[GrammarRule]:
         self._skip_empty_lines()
 
         while self.peek():
-            yield from self.iter_rules_on_line()
+            yield from self._iter_rules_on_line()
             self._skip_empty_lines()
 
     def parse_schema(self) -> GrammarSchema:
-        rules = list(self.iter_rules())
+        rules = list(self._iter_rules())
 
         if len(rules) == 0:
             raise ParserError('Expected at least one grammar rule')
@@ -224,13 +227,6 @@ def parse_grammar_schema(source: str) -> GrammarSchema:
         return parser.parse_schema()
 
 
-def parse_grammar_rule_line(source: str) -> GrammarSchema:
-    tokens = tokenize_grammar(source)
-
-    with GrammarParser(source, tokens) as parser:
-        return GrammarSchema(list(parser.iter_rules_on_line()))
-
-
 def parse_terminal(source: str) -> Terminal:
     tokens = tokenize_grammar(source)
 

src/notebook/math/grammars/parsing/test_parser.py

@@ -6,7 +6,7 @@
 from ....support.pytest import pytest_parametrize_kwargs, pytest_parametrize_lists
 from ..grammar import GrammarRule, GrammarSchema
 from ..symbols import NonTerminal, Terminal
-from .parser import parse_grammar_rule_line, parse_grammar_schema, parse_nonterminal, parse_terminal
+from .parser import parse_grammar_schema, parse_nonterminal, parse_terminal
 
 
 @pytest_parametrize_lists(
@@ -168,14 +168,14 @@ def test_parsing_nested_nonterminal() -> None:
 
 def test_parsing_empty_rule() -> None:
     with pytest.raises(ParserError) as excinfo:
-        parse_grammar_rule_line('')
+        parse_grammar_schema('')
 
-    assert str(excinfo.value) == 'Expected a rule'
+    assert str(excinfo.value) == 'Expected at least one grammar rule'
 
 
 def test_parsing_rule_with_empty_left_side() -> None:
     with pytest.raises(ParserError) as excinfo:
-        parse_grammar_rule_line('→ ε')
+        parse_grammar_schema('→ ε')
 
     assert str(excinfo.value) == 'The left side of a rule must be nonempty'
     assert excinfo.value.__notes__[0] == dedent('''\

src/notebook/math/grammars/test_context_sensitive.py

@@ -1,53 +1,48 @@
-from textwrap import dedent
 from typing import TYPE_CHECKING
 
 from ...support.pytest import pytest_parametrize_lists
-from .context_sensitive import is_essentially_length_increasing_grammar, is_length_increasing_grammar, naive_membership
+from .context_sensitive import (
+    is_context_sensitive_grammar,
+    is_context_sensitive_rule,
+    length_increasing_to_context_sensitive,
+)
 from .parsing import parse_grammar_schema
-from .symbols import NonTerminal
 
 
 if TYPE_CHECKING:
     from .conftest import GrammarFixture
 
 
-def test_is_length_increasing_grammar_an(an: GrammarFixture) -> None:
-    assert not is_length_increasing_grammar(an.grammar)
-    assert is_essentially_length_increasing_grammar(an.grammar)
+@pytest_parametrize_lists(
+    rule=[
+        '<S> → <S>',
+        '<S> → "a"',
+        '<S> → "a" "b" "c"',
+        '"a" <B> "c" → "a" "b" "c"',
+        '"a" <BC> "d" → "a" "b" "c" "d"',
+    ],
+)
+def test_is_context_sensitive_rule_success(rule: str) -> None:
+    rule_ = parse_grammar_schema(rule).rules[0]
+    assert is_context_sensitive_rule(rule_)
 
 
-def test_is_length_increasing_grammar_binary(binary: GrammarFixture) -> None:
-    assert is_length_increasing_grammar(binary.grammar)
+@pytest_parametrize_lists(
+    rule=[
+        '<S> → ε',
+        '"a" <B> "c" → "a" "c"',
+        '"a" <BC> "d" → "b" "c"',
+    ],
+)
+def test_is_context_sensitive_rule_failure(rule: str) -> None:
+    rule_ = parse_grammar_schema(rule).rules[0]
+    assert not is_context_sensitive_rule(rule_)
 
 
-def test_is_length_increasing_grammar_anbncn(anbncn: GrammarFixture) -> None:
-    assert is_essentially_length_increasing_grammar(anbncn.grammar)
+def test_is_context_sensitive_grammar_anbncn(anbncn: GrammarFixture) -> None:
+    assert not is_context_sensitive_grammar(anbncn.grammar)
 
 
-def test_is_length_increasing_grammar_failure() -> None:
-    schema = parse_grammar_schema(
-        dedent('''\
-            <S> → "a" <S> "b"
-            "a" <S> "b" → <S>
-            ''',
-        ),
-    )
-
-    grammar = schema.instantiate(NonTerminal('S'))
-    assert not is_essentially_length_increasing_grammar(grammar)
-
-
-def test_naive_membership_an(an: GrammarFixture) -> None:
-    for string in an.whitelist:
-        assert naive_membership(an.grammar, string)
-
-    for string in an.blacklist:
-        assert not naive_membership(an.grammar, string)
-
-
-def test_naive_membership_anbn(anbn: GrammarFixture) -> None:
-    for string in anbn.whitelist:
-        assert naive_membership(anbn.grammar, string)
-
-    for string in anbn.blacklist:
-        assert not naive_membership(anbn.grammar, string)
+def test_length_increasing_to_context_sensitive_anbncn(anbncn: GrammarFixture) -> None:
+    transformed = length_increasing_to_context_sensitive(anbncn.grammar)
+    assert is_context_sensitive_grammar(transformed)