Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions evaluation_function/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,16 +105,16 @@ def main():
- If 2+ args provided: File-based communication (last 2 args are input/output paths)
- Otherwise: RPC/IPC server mode using lf_toolkit
"""
# # Check for file-based communication
# # shimmy passes input and output file paths as the last two arguments
# if len(sys.argv) >= 3:
# input_path = sys.argv[-2]
# output_path = sys.argv[-1]
# Check for file-based communication
# shimmy passes input and output file paths as the last two arguments
if len(sys.argv) >= 3:
input_path = sys.argv[-2]
output_path = sys.argv[-1]

# # Verify they look like file paths (basic check)
# if not input_path.startswith('-') and not output_path.startswith('-'):
# handle_file_based_communication(input_path, output_path)
# return
# Verify they look like file paths (basic check)
if not input_path.startswith('-') and not output_path.startswith('-'):
handle_file_based_communication(input_path, output_path)
return

# Fall back to RPC/IPC server mode
server = create_server()
Expand Down
116 changes: 116 additions & 0 deletions evaluation_function/test/test_correction.py
Original file line number Diff line number Diff line change
Expand Up @@ -207,5 +207,121 @@ def test_non_minimal_fsa_fails_when_required(self, equivalent_dfa):
assert any(e.code == ErrorCode.NOT_MINIMAL for e in result.fsa_feedback.errors)


# =============================================================================
# Test Epsilon Transitions (End-to-End)
# =============================================================================

class TestEpsilonTransitionCorrection:
"""Test the full correction pipeline with ε-NFA inputs."""

def test_epsilon_nfa_vs_equivalent_dfa_correct(self):
"""ε-NFA student answer equivalent to DFA expected should be correct."""
# ε-NFA accepts exactly "a": q0 --ε--> q1 --a--> q2
student_enfa = make_fsa(
states=["q0", "q1", "q2"],
alphabet=["a"],
transitions=[
{"from_state": "q0", "to_state": "q1", "symbol": "ε"},
{"from_state": "q1", "to_state": "q2", "symbol": "a"},
],
initial="q0",
accept=["q2"],
)
# DFA accepts exactly "a": s0 --a--> s1
expected_dfa = make_fsa(
states=["s0", "s1"],
alphabet=["a"],
transitions=[
{"from_state": "s0", "to_state": "s1", "symbol": "a"},
],
initial="s0",
accept=["s1"],
)
result = analyze_fsa_correction(student_enfa, expected_dfa)
assert isinstance(result, Result)
assert result.is_correct is True

def test_epsilon_nfa_vs_different_dfa_incorrect(self):
"""ε-NFA accepting 'a' vs DFA accepting 'b' should be incorrect."""
student_enfa = make_fsa(
states=["q0", "q1", "q2"],
alphabet=["a", "b"],
transitions=[
{"from_state": "q0", "to_state": "q1", "symbol": "ε"},
{"from_state": "q1", "to_state": "q2", "symbol": "a"},
],
initial="q0",
accept=["q2"],
)
expected_dfa = make_fsa(
states=["s0", "s1"],
alphabet=["a", "b"],
transitions=[
{"from_state": "s0", "to_state": "s1", "symbol": "b"},
],
initial="s0",
accept=["s1"],
)
result = analyze_fsa_correction(student_enfa, expected_dfa)
assert isinstance(result, Result)
assert result.is_correct is False
assert result.fsa_feedback is not None
assert len(result.fsa_feedback.errors) > 0

def test_multi_epsilon_nfa_vs_dfa_correct(self):
"""ε-NFA for (a|b) with branching epsilons should match equivalent DFA."""
student_enfa = make_fsa(
states=["q0", "q1", "q2", "q3"],
alphabet=["a", "b"],
transitions=[
{"from_state": "q0", "to_state": "q1", "symbol": "ε"},
{"from_state": "q0", "to_state": "q2", "symbol": "ε"},
{"from_state": "q1", "to_state": "q3", "symbol": "a"},
{"from_state": "q2", "to_state": "q3", "symbol": "b"},
],
initial="q0",
accept=["q3"],
)
expected_dfa = make_fsa(
states=["s0", "s1"],
alphabet=["a", "b"],
transitions=[
{"from_state": "s0", "to_state": "s1", "symbol": "a"},
{"from_state": "s0", "to_state": "s1", "symbol": "b"},
],
initial="s0",
accept=["s1"],
)
result = analyze_fsa_correction(student_enfa, expected_dfa)
assert isinstance(result, Result)
assert result.is_correct is True

def test_epsilon_nfa_structural_info_reports_nondeterministic(self):
"""ε-NFA should have structural info reporting non-deterministic."""
student_enfa = make_fsa(
states=["q0", "q1", "q2"],
alphabet=["a"],
transitions=[
{"from_state": "q0", "to_state": "q1", "symbol": "ε"},
{"from_state": "q1", "to_state": "q2", "symbol": "a"},
],
initial="q0",
accept=["q2"],
)
expected_dfa = make_fsa(
states=["s0", "s1"],
alphabet=["a"],
transitions=[
{"from_state": "s0", "to_state": "s1", "symbol": "a"},
],
initial="s0",
accept=["s1"],
)
result = analyze_fsa_correction(student_enfa, expected_dfa)
assert result.fsa_feedback is not None
assert result.fsa_feedback.structural is not None
assert result.fsa_feedback.structural.is_deterministic is False


if __name__ == "__main__":
pytest.main([__file__, "-v"])
222 changes: 176 additions & 46 deletions evaluation_function/test/test_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -314,55 +314,185 @@ def test_isomorphic_dfas(self):
initial="s0",
accept=["s1"],
)
assert are_isomorphic(fsa_user, fsa_sol).ok


# =============================================================================
# Test Minimality
# =============================================================================

@pytest.fixture
def dfa_accepts_a():
"""DFA that accepts exactly 'a'."""
return make_fsa(
states=["q0", "q1", "q2"],
alphabet=["a", "b"],
transitions=[
{"from_state": "q0", "to_state": "q1", "symbol": "a"},
{"from_state": "q0", "to_state": "q2", "symbol": "b"},
{"from_state": "q1", "to_state": "q2", "symbol": "a"},
{"from_state": "q1", "to_state": "q2", "symbol": "b"},
{"from_state": "q2", "to_state": "q2", "symbol": "a"},
{"from_state": "q2", "to_state": "q2", "symbol": "b"},
],
initial="q0",
accept=["q1"]
)


class TestCheckMinimality:
"""Test check_minimality function."""

def test_minimal_dfa(self, dfa_accepts_a):
result = is_minimal(dfa_accepts_a)
assert result.ok
assert result.value is True

def test_non_minimal_dfa_with_unreachable(self):
non_minimal = make_fsa(
states=["q0", "q1", "q2", "unreachable"],
assert are_isomorphic(fsa_user, fsa_sol) == []


class TestEpsilonTransitions:
"""Tests for epsilon transition handling across the validation pipeline."""

def test_valid_fsa_with_epsilon_unicode(self):
"""ε-NFA with Unicode ε should pass structural validation."""
fsa = make_fsa(
states=["q0", "q1", "q2"],
alphabet=["a"],
transitions=[
{"from_state": "q0", "to_state": "q1", "symbol": "ε"},
{"from_state": "q1", "to_state": "q2", "symbol": "a"},
],
initial="q0",
accept=["q2"],
)
assert is_valid_fsa(fsa) == []

def test_valid_fsa_with_epsilon_string(self):
"""ε-NFA with 'epsilon' string should pass structural validation."""
fsa = make_fsa(
states=["q0", "q1", "q2"],
alphabet=["a"],
transitions=[
{"from_state": "q0", "to_state": "q1", "symbol": "epsilon"},
{"from_state": "q1", "to_state": "q2", "symbol": "a"},
],
initial="q0",
accept=["q2"],
)
assert is_valid_fsa(fsa) == []

def test_valid_fsa_with_empty_string_epsilon(self):
"""ε-NFA with empty string epsilon should pass structural validation."""
fsa = make_fsa(
states=["q0", "q1", "q2"],
alphabet=["a"],
transitions=[
{"from_state": "q0", "to_state": "q1", "symbol": ""},
{"from_state": "q1", "to_state": "q2", "symbol": "a"},
],
initial="q0",
accept=["q2"],
)
assert is_valid_fsa(fsa) == []

def test_epsilon_nfa_is_not_deterministic(self):
"""ε-NFA should be flagged as non-deterministic."""
fsa = make_fsa(
states=["q0", "q1"],
alphabet=["a"],
transitions=[
{"from_state": "q0", "to_state": "q1", "symbol": "ε"},
],
initial="q0",
accept=["q1"],
)
errors = is_deterministic(fsa)
assert len(errors) > 0
assert ErrorCode.NOT_DETERMINISTIC in [e.code for e in errors]

def test_accepts_string_via_epsilon_closure(self):
"""ε-NFA should accept 'a' by following q0 --ε--> q1 --a--> q2."""
fsa = make_fsa(
states=["q0", "q1", "q2"],
alphabet=["a"],
transitions=[
{"from_state": "q0", "to_state": "q1", "symbol": "ε"},
{"from_state": "q1", "to_state": "q2", "symbol": "a"},
],
initial="q0",
accept=["q2"],
)
assert accepts_string(fsa, "a") == []

def test_rejects_string_with_epsilon_nfa(self):
"""ε-NFA that accepts 'a' should reject empty string."""
fsa = make_fsa(
states=["q0", "q1", "q2"],
alphabet=["a"],
transitions=[
{"from_state": "q0", "to_state": "q1", "symbol": "ε"},
{"from_state": "q1", "to_state": "q2", "symbol": "a"},
],
initial="q0",
accept=["q2"],
)
errors = accepts_string(fsa, "")
assert len(errors) > 0

def test_accepts_empty_string_via_epsilon(self):
"""ε-NFA should accept empty string when initial reaches accept via ε."""
fsa = make_fsa(
states=["q0", "q1"],
alphabet=["a"],
transitions=[
{"from_state": "q0", "to_state": "q1", "symbol": "ε"},
],
initial="q0",
accept=["q1"],
)
assert accepts_string(fsa, "") == []

def test_epsilon_nfa_equivalent_to_dfa(self):
"""ε-NFA and DFA accepting the same language should be equivalent."""
enfa = make_fsa(
states=["q0", "q1", "q2"],
alphabet=["a"],
transitions=[
{"from_state": "q0", "to_state": "q1", "symbol": "ε"},
{"from_state": "q1", "to_state": "q2", "symbol": "a"},
],
initial="q0",
accept=["q2"],
)
dfa = make_fsa(
states=["s0", "s1"],
alphabet=["a"],
transitions=[
{"from_state": "s0", "to_state": "s1", "symbol": "a"},
],
initial="s0",
accept=["s1"],
)
assert fsas_accept_same_language(enfa, dfa) == []

def test_epsilon_nfa_not_equivalent_to_different_dfa(self):
"""ε-NFA and DFA accepting different languages should not be equivalent."""
enfa = make_fsa(
states=["q0", "q1", "q2"],
alphabet=["a", "b"],
transitions=[
{"from_state": "q0", "to_state": "q1", "symbol": "a"},
{"from_state": "q0", "to_state": "q2", "symbol": "b"},
{"from_state": "q0", "to_state": "q1", "symbol": "ε"},
{"from_state": "q1", "to_state": "q2", "symbol": "a"},
{"from_state": "q1", "to_state": "q2", "symbol": "b"},
{"from_state": "q2", "to_state": "q2", "symbol": "a"},
{"from_state": "q2", "to_state": "q2", "symbol": "b"},
{"from_state": "unreachable", "to_state": "unreachable", "symbol": "a"},
],
initial="q0",
accept=["q1"]
accept=["q2"],
)
result = is_minimal(non_minimal)
assert not result.ok
dfa = make_fsa(
states=["s0", "s1"],
alphabet=["a", "b"],
transitions=[
{"from_state": "s0", "to_state": "s1", "symbol": "b"},
],
initial="s0",
accept=["s1"],
)
errors = fsas_accept_same_language(enfa, dfa)
assert len(errors) > 0

def test_multi_epsilon_nfa_equivalent_to_dfa(self):
"""ε-NFA for (a|b) with branching epsilons should match equivalent DFA."""
# q0 --ε--> q1, q0 --ε--> q2, q1 --a--> q3, q2 --b--> q3
enfa = make_fsa(
states=["q0", "q1", "q2", "q3"],
alphabet=["a", "b"],
transitions=[
{"from_state": "q0", "to_state": "q1", "symbol": "ε"},
{"from_state": "q0", "to_state": "q2", "symbol": "ε"},
{"from_state": "q1", "to_state": "q3", "symbol": "a"},
{"from_state": "q2", "to_state": "q3", "symbol": "b"},
],
initial="q0",
accept=["q3"],
)
dfa = make_fsa(
states=["s0", "s1"],
alphabet=["a", "b"],
transitions=[
{"from_state": "s0", "to_state": "s1", "symbol": "a"},
{"from_state": "s0", "to_state": "s1", "symbol": "b"},
],
initial="s0",
accept=["s1"],
)
assert fsas_accept_same_language(enfa, dfa) == []


if __name__ == "__main__":
pytest.main([__file__, "-v"])
Loading
Loading