Remove Spanish-specific TSV tooling

tests/test_core.py

@@ -26,27 +26,18 @@ from saiki.importer import (
     synthesize_tts_sample,
     supported_tts_backends,
 )
-from saiki.spanish import (
-    BUILTIN_LEMMA_CORRECTIONS,
-    check_card_for_issues,
-    clean_anki_field_text,
-    extract_detailed_counts,
-    lint_anki_cards,
-    load_lemma_corrections,
-    safe_spanish_lemma,
-    safe_spanish_lemma_info,
-    spanish_content_filter,
-    spanish_function_word_filter,
-    write_debug_tsv,
-    write_suspicious_tokens,
-)
 from saiki.text import extract_first_visible_line, extract_visible_text
 from saiki.words import (
     build_query_from_decks,
+    clean_anki_field_text,
     compare_word_files,
     compare_word_lists,
+    extract_detailed_counts,
+    extract_words_from_file,
     normalize_word_for_comparison,
     read_word_file,
+    spanish_filter,
+    write_debug_tsv,
 )
 from saiki.youtube import TranscriptLine, extract_video_id, sentence_vocab, write_sentence_export
 
@@ -177,6 +168,15 @@ class AnkiTsvTests(unittest.TestCase):
             self.assertEqual(records[0]["line_number"], 2)
             self.assertEqual(records[1]["line_number"], 3)
 
+    def test_parse_tsv_preserves_hash_data_rows_after_headers(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = os.path.join(tmp, "test.txt")
+            with open(path, "w", encoding="utf-8") as f:
+                f.write("#separator:tab\nalpha\tbeta\n#hashtag\tvalue\n")
+            records = parse_anki_tsv(path, field_index=1)
+            self.assertEqual([r["text"] for r in records], ["alpha", "#hashtag"])
+            self.assertEqual(records[1]["line_number"], 3)
+
     def test_parse_tsv_include_tags(self):
         with tempfile.TemporaryDirectory() as tmp:
             path = os.path.join(tmp, "test.txt")
@@ -215,17 +215,19 @@ class CliValidationTests(unittest.TestCase):
         self.assertEqual(code, 1)
         self.assertIn("1-based", stderr.getvalue())
 
-    def test_words_input_is_spanish_only(self):
+    def test_words_input_accepts_japanese_tsv(self):
         from io import StringIO
 
         with patch("saiki.cli.importlib.metadata.version", return_value="0"):
-            with patch("sys.stderr", new_callable=StringIO) as stderr:
-                code = main(["words", "jp", "--input", "deck.tsv", "--field", "2"])
-        self.assertEqual(code, 1)
-        self.assertIn("Spanish-only", stderr.getvalue())
+            with patch("saiki.cli.extract_words_from_file") as extract:
+                extract.return_value = {"records": 0, "written": 0, "out": "out.txt"}
+                with patch("sys.stdout", new_callable=StringIO):
+                    code = main(["words", "jp", "--input", "deck.tsv", "--field", "2"])
+        self.assertEqual(code, 0)
+        extract.assert_called_once()
 
 
-class SpanishTextCleaningTests(unittest.TestCase):
+class AnkiFieldCleaningTests(unittest.TestCase):
     def test_remove_sound_marker(self):
         self.assertEqual(
             clean_anki_field_text("[sound:es_001.mp3] Hola mundo"),
@@ -236,17 +238,23 @@ class SpanishTextCleaningTests(unittest.TestCase):
         result = clean_anki_field_text("<span>Para el lunes...</span>")
         self.assertEqual(result, "Para el lunes...")
 
-    def test_english_after_brbr_removed(self):
-        result = clean_anki_field_text("hasta luego<br><br>(see you later)")
-        self.assertEqual(result, "hasta luego")
+    def test_double_br_sections_are_preserved_by_default(self):
+        result = clean_anki_field_text("Target sentence<br><br>(Translation sentence)")
+        self.assertEqual(result, "Target sentence (Translation sentence)")
 
-    def test_english_gloss_after_brbr_removed(self):
-        result = clean_anki_field_text("Te quiero.<br><br>I love you / I want you")
-        self.assertEqual(result, "Te quiero.")
+    def test_first_field_section_drops_later_sections_without_phrase_rules(self):
+        result = clean_anki_field_text("Target sentence<br><br>Translation sentence")
+        self.assertEqual(result, "Target sentence Translation sentence")
 
-    def test_spanish_after_brbr_preserved(self):
-        result = clean_anki_field_text("Hola<br><br>Otra línea")
-        self.assertEqual(result, "Hola Otra línea")
+        result = clean_anki_field_text(
+            "Target sentence<br><br>Translation sentence",
+            field_section="first",
+        )
+        self.assertEqual(result, "Target sentence")
+
+    def test_later_field_sections_are_preserved_by_default(self):
+        result = clean_anki_field_text("First line<br><br>Second line")
+        self.assertEqual(result, "First line Second line")
 
     def test_normalize_whitespace(self):
         result = clean_anki_field_text("  Hola   mundo  ")
@@ -283,230 +291,13 @@ class MockToken:
         self.sent = None
 
 
-class SpanishLemmaTests(unittest.TestCase):
-    def test_builtin_correction_comar(self):
-        token = MockToken(text="como", lemma="comar")
-        self.assertEqual(safe_spanish_lemma(token), "comer")
+class DetailedCountsTests(unittest.TestCase):
+    def test_spanish_filter_uses_content_pos(self):
+        self.assertTrue(spanish_filter(MockToken(text="casa", lemma="casa", pos="NOUN")))
+        self.assertTrue(spanish_filter(MockToken(text="rápido", lemma="rápido", pos="ADV")))
+        self.assertFalse(spanish_filter(MockToken(text="Madrid", lemma="Madrid", pos="PROPN")))
+        self.assertFalse(spanish_filter(MockToken(text="para", lemma="para", pos="ADP")))
 
-    def test_builtin_correction_acabir(self):
-        token = MockToken(text="acabas", lemma="acabir")
-        self.assertEqual(safe_spanish_lemma(token), "acabar")
-
-    def test_builtin_correction_delicius(self):
-        token = MockToken(text="delicioso", lemma="deliciós")
-        self.assertEqual(safe_spanish_lemma(token), "delicioso")
-
-    def test_builtin_correction_llover_slash(self):
-        token = MockToken(text="llueve", lemma="llover/llover")
-        self.assertEqual(safe_spanish_lemma(token), "llover")
-
-    def test_multi_word_lemma_falls_back(self):
-        token = MockToken(text="ayudar", lemma="ayudar yo")
-        self.assertEqual(safe_spanish_lemma(token), "ayudar")
-
-    def test_multi_word_lemma_lavar_el(self):
-        token = MockToken(text="lava", lemma="lavar él")
-        self.assertEqual(safe_spanish_lemma(token), "lava")
-
-    def test_prir_corrected_when_source_is_pedir(self):
-        for form in ["pide", "pido", "piden", "pidiendo", "pedir"]:
-            token = MockToken(text=form, lemma="prir")
-            self.assertEqual(safe_spanish_lemma(token), "pedir")
-
-    def test_prir_not_corrected_for_unknown_form(self):
-        token = MockToken(text="prir", lemma="prir")
-        self.assertEqual(safe_spanish_lemma(token), "prir")
-
-    def test_empty_lemma_falls_back(self):
-        token = MockToken(text="hola", lemma="")
-        self.assertEqual(safe_spanish_lemma(token), "hola")
-
-    def test_lemma_with_punctuation_falls_back(self):
-        token = MockToken(text="comiendo", lemma="comiendo,")
-        self.assertEqual(safe_spanish_lemma(token), "comiendo")
-
-    def test_normal_lemma_passes_through(self):
-        token = MockToken(text="comiendo", lemma="comer")
-        self.assertEqual(safe_spanish_lemma(token), "comer")
-
-    def test_lemma_info_tracks_correction_reason(self):
-        token = MockToken(text="como", lemma="comar")
-        info = safe_spanish_lemma_info(token)
-        self.assertEqual(info.lemma, "comer")
-        self.assertEqual(info.original_lemma, "comar")
-        self.assertIn("corrected", info.status)
-
-    def test_non_spanish_lemma_falls_back(self):
-        token = MockToken(text="hola", lemma="hello_world")
-        info = safe_spanish_lemma_info(token)
-        self.assertEqual(info.lemma, "hola")
-        self.assertEqual(info.status, "fallback_bad_lemma_characters")
-
-    def test_extra_corrections_override_builtin(self):
-        extra = {"comar": "comprar"}
-        token = MockToken(text="como", lemma="comar")
-        self.assertEqual(
-            safe_spanish_lemma(token, extra_corrections=extra),
-            "comprar",
-        )
-
-    def test_load_lemma_corrections(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            path = os.path.join(tmp, "corrections.tsv")
-            with open(path, "w", encoding="utf-8") as f:
-                f.write("malo\tbueno\nfeo\tbonito\n")
-            corrections = load_lemma_corrections(path)
-            self.assertEqual(corrections, {"malo": "bueno", "feo": "bonito"})
-
-    def test_load_lemma_corrections_skips_comments(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            path = os.path.join(tmp, "corrections.tsv")
-            with open(path, "w", encoding="utf-8") as f:
-                f.write("# comment\nmalo\tbueno\n")
-            corrections = load_lemma_corrections(path)
-            self.assertEqual(corrections, {"malo": "bueno"})
-
-
-class SpanishFilterTests(unittest.TestCase):
-    def test_content_filter_keeps_noun(self):
-        token = MockToken(text="casa", lemma="casa", pos="NOUN")
-        self.assertTrue(spanish_content_filter(token))
-
-    def test_content_filter_keeps_verb(self):
-        token = MockToken(text="come", lemma="comer", pos="VERB")
-        self.assertTrue(spanish_content_filter(token))
-
-    def test_content_filter_keeps_adj(self):
-        token = MockToken(text="grande", lemma="grande", pos="ADJ")
-        self.assertTrue(spanish_content_filter(token))
-
-    def test_content_filter_keeps_adv(self):
-        token = MockToken(text="bien", lemma="bien", pos="ADV")
-        self.assertTrue(spanish_content_filter(token))
-
-    def test_content_filter_excludes_propn_by_default(self):
-        token = MockToken(text="Madrid", lemma="Madrid", pos="PROPN")
-        self.assertFalse(spanish_content_filter(token))
-
-    def test_content_filter_includes_propn_with_flag(self):
-        token = MockToken(text="Madrid", lemma="Madrid", pos="PROPN")
-        self.assertTrue(
-            spanish_content_filter(token, include_proper_nouns=True)
-        )
-
-    def test_content_filter_excludes_punct(self):
-        token = MockToken(text=".", lemma=".", pos="PUNCT", is_punct=True)
-        self.assertFalse(spanish_content_filter(token))
-
-    def test_content_filter_excludes_digit(self):
-        token = MockToken(text="123", lemma="123", pos="NUM", is_digit=True)
-        self.assertFalse(spanish_content_filter(token))
-
-    def test_content_filter_excludes_url(self):
-        token = MockToken(
-            text="http://x.com", lemma="http://x.com", pos="X", like_url=True
-        )
-        self.assertFalse(spanish_content_filter(token))
-
-    def test_content_filter_excludes_media_filename(self):
-        token = MockToken(text="es_001.mp3", lemma="es_001.mp3", pos="NOUN")
-        self.assertFalse(spanish_content_filter(token))
-
-    def test_content_filter_excludes_one_char_non_vowel(self):
-        token = MockToken(text="x", lemma="x", pos="NOUN")
-        self.assertFalse(spanish_content_filter(token))
-
-    def test_function_word_filter_keeps_preposition(self):
-        token = MockToken(text="para", lemma="para", pos="ADP")
-        self.assertTrue(spanish_function_word_filter(token))
-
-    def test_function_word_filter_keeps_conjunction(self):
-        token = MockToken(text="y", lemma="y", pos="CCONJ")
-        self.assertTrue(spanish_function_word_filter(token))
-
-    def test_function_word_filter_rejects_noun(self):
-        token = MockToken(text="casa", lemma="casa", pos="NOUN")
-        self.assertFalse(spanish_function_word_filter(token))
-
-
-class SpanishLintTests(unittest.TestCase):
-    def _check(self, text: str) -> list:
-        return check_card_for_issues(text, clean_anki_field_text(text))
-
-    def test_detect_tu_sabes_nada(self):
-        issues = self._check("Tú sabes nada")
-        self.assertTrue(any("Missing negation" in i["reason"] for i in issues))
-
-    def test_detect_dulce_suenos(self):
-        issues = self._check("Dulce sueños")
-        self.assertTrue(
-            any("agree in number" in i["reason"] for i in issues)
-        )
-
-    def test_detect_ella_sona(self):
-        issues = self._check("Ella soña con viajar.")
-        self.assertTrue(
-            any("present indicative" in i["reason"] for i in issues)
-        )
-
-    def test_detect_tengo_uno_libro(self):
-        issues = self._check("Tengo uno libro en la mesa.")
-        self.assertTrue(
-            any("instead of 'un'" in i["reason"] for i in issues)
-        )
-
-    def test_detect_observando_nadan(self):
-        issues = self._check(
-            "La pareja está en un bote observando a los peces nadan."
-        )
-        self.assertTrue(
-            any("'cómo'" in i["reason"] for i in issues)
-        )
-
-    def test_detect_le_vuelven_loca(self):
-        issues = self._check("Los perros pequeños le vuelven loca.")
-        self.assertTrue(
-            any("direct object" in i["reason"] for i in issues)
-        )
-
-    def test_detect_ai_generated_tts(self):
-        issues = self._check("AI-generated text-to-speech")
-        self.assertTrue(
-            any("Metadata contamination" in i["reason"] for i in issues)
-        )
-
-    def test_detect_necesito_missing_article(self):
-        issues = self._check("Necesito jardinero")
-        self.assertTrue(
-            any("Missing article" in i["reason"] for i in issues)
-        )
-
-    def test_detect_le_vuelven_loco(self):
-        issues = self._check("Estos problemas le vuelven loco.")
-        self.assertTrue(
-            any("direct object" in i["reason"] or "direct object pronoun" in i["reason"] for i in issues)
-        )
-
-    def test_veintiuno_anos(self):
-        issues = self._check("Tengo veintiuno años de edad.")
-        self.assertTrue(
-            any("veintiún" in i["suggested_fix"] for i in issues)
-        )
-
-    def test_lint_anki_cards_multiple_records(self):
-        records = [
-            {"text": "Hola mundo", "line_number": 1, "raw_line": "Hola mundo"},
-            {"text": "Tú sabes nada", "line_number": 2, "raw_line": "Tú sabes nada"},
-            {"text": "Ella soña con viajar.", "line_number": 3, "raw_line": "Ella soña con viajar."},
-        ]
-        results = lint_anki_cards(records)
-        self.assertEqual(len(results), 2)
-        reasons = results[0]["reason"] + results[1]["reason"]
-        self.assertIn("Missing negation", reasons)
-        self.assertIn("present indicative", reasons)
-
-
-class SpanishDetailedCountsTests(unittest.TestCase):
     def test_extract_detailed_counts_basic(self):
         records = [
             {"text": "Yo como manzanas.", "line_number": 1},
@@ -525,7 +316,7 @@ class SpanishDetailedCountsTests(unittest.TestCase):
                 if "como" in text:
                     return MockDoc([
                         MockToken(text="Yo", lemma="yo", pos="PRON"),
-                        MockToken(text="como", lemma="comar", pos="VERB"),
+                        MockToken(text="como", lemma="comer", pos="VERB"),
                         MockToken(text="manzanas", lemma="manzana", pos="NOUN"),
                         MockToken(text=".", lemma=".", pos="PUNCT", is_punct=True),
                     ])
@@ -540,12 +331,10 @@ class SpanishDetailedCountsTests(unittest.TestCase):
             records,
             MockNLP(),
             lambda t: t.pos_ in {"NOUN", "VERB", "ADJ", "ADV"},
-            safe_spanish_lemma,
+            lambda t: (t.lemma_ or t.text).lower(),
         )
 
-        # "comar" should be corrected to "comer"
         self.assertIn("comer", stats)
-        # "comer" should have count 2 (comes + como -> comer)
         self.assertEqual(stats["comer"]["count"], 2)
         self.assertIn("manzana", stats)
         self.assertIn("pan", stats)
@@ -571,27 +360,6 @@ class SpanishDetailedCountsTests(unittest.TestCase):
             self.assertIn("Yo como.", content)
             self.assertIn("1, 2, 3", content)
 
-    def test_suspicious_tokens_output(self):
-        stats = {
-            "comer": {
-                "count": 3,
-                "pos_counts": {"VERB": 3},
-                "surface_forms": ["como", "comes"],
-                "original_lemmas": ["comar", "comer"],
-                "lemma_statuses": {"corrected:comar->comer": 1, "ok": 2},
-                "example_sentences": ["Yo como."],
-                "source_lines": ["1"],
-            }
-        }
-        with tempfile.TemporaryDirectory() as tmp:
-            path = os.path.join(tmp, "suspicious.tsv")
-            write_suspicious_tokens(stats, path)
-            with open(path, "r", encoding="utf-8") as f:
-                content = f.read()
-            self.assertIn("comer", content)
-            self.assertIn("comar", content)
-            self.assertIn("corrected:comar->comer", content)
-
     def test_debug_tsv_min_freq_filters(self):
         stats = {
             "comer": {"count": 5, "pos_counts": {"VERB": 5},
@@ -638,29 +406,69 @@ class SpanishDetailedCountsTests(unittest.TestCase):
             lambda t: t.text.lower(),
             clean=True,
         )
-        # With clean=True, the sound marker is stripped → empty text → no stats
+        # With clean=True, the sound marker is stripped, so there is no text to count.
         self.assertEqual(len(stats_clean), 0)
 
-    def test_lint_anki_cards_verbose_includes_all(self):
-        records = [
-            {"text": "Hola mundo", "line_number": 1, "raw_line": "Hola mundo"},
-            {"text": "Tú sabes nada", "line_number": 2, "raw_line": "Tú sabes nada"},
-        ]
-        # Without verbose – only suspicious
-        results = lint_anki_cards(records, verbose=False)
-        self.assertEqual(len(results), 1)
+    def test_extract_detailed_counts_can_use_first_field_section(self):
+        class MockDoc:
+            def __init__(self, tokens):
+                self.tokens = tokens
+            def __iter__(self):
+                return iter(self.tokens)
 
-        # With verbose – both records
-        results = lint_anki_cards(records, verbose=True)
-        self.assertEqual(len(results), 2)
+        class MockNLP:
+            def __call__(self, text):
+                return MockDoc([
+                    MockToken(text=part, lemma=part.lower(), pos="NOUN")
+                    for part in text.split()
+                ])
 
-    def test_lint_anki_cards_verbose_empty_field(self):
-        records = [
-            {"text": "", "line_number": 1, "raw_line": ""},
-        ]
-        results = lint_anki_cards(records, verbose=True)
-        self.assertEqual(len(results), 1)
-        self.assertEqual(results[0]["reason"], "empty field")
+        records = [{"text": "Target<br><br>Translation", "line_number": 1}]
+        stats = extract_detailed_counts(
+            records,
+            MockNLP(),
+            lambda t: True,
+            lambda t: t.lemma_,
+            field_section="first",
+        )
+        self.assertIn("target", stats)
+        self.assertNotIn("translation", stats)
+
+    def test_extract_words_from_file_uses_language_profile(self):
+        class MockDoc:
+            def __init__(self, tokens):
+                self.tokens = tokens
+            def __iter__(self):
+                return iter(self.tokens)
+
+        class MockNLP:
+            def __call__(self, text):
+                return MockDoc([
+                    MockToken(text="猫", lemma="猫", pos="NOUN"),
+                    MockToken(text="です", lemma="です", pos="AUX"),
+                ])
+
+        with tempfile.TemporaryDirectory() as tmp:
+            source = os.path.join(tmp, "deck.tsv")
+            out = os.path.join(tmp, "words_jp.txt")
+            with open(source, "w", encoding="utf-8") as f:
+                f.write("#separator:tab\n[sound:jp_001.mp3]\t猫です。\n")
+
+            with patch("saiki.words.load_spacy_model", return_value=MockNLP()):
+                result = extract_words_from_file(
+                    Config(deepcopy(DEFAULT_CONFIG)),
+                    "jp",
+                    source,
+                    out=out,
+                    min_freq=1,
+                )
+
+            with open(out, "r", encoding="utf-8") as f:
+                content = f.read()
+
+        self.assertEqual(result["records"], 1)
+        self.assertIn("猫 1", content)
+        self.assertNotIn("です", content)
 
 
 class CompareWordListsTests(unittest.TestCase):