660 lines
28 KiB
Python
660 lines
28 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from copy import deepcopy
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
SRC_DIR = Path(__file__).resolve().parents[1] / "src"
|
|
if str(SRC_DIR) not in sys.path:
|
|
sys.path.insert(0, str(SRC_DIR))
|
|
|
|
from saiki.anki_tsv import parse_anki_headers, parse_anki_tsv
|
|
from saiki.audio import build_playlist, resolve_media_paths
|
|
from saiki.cli import build_parser, main
|
|
from saiki.config import Config, DEFAULT_CONFIG, deep_merge
|
|
from saiki.importer import (
|
|
PreparedTtsBackend,
|
|
import_sentences,
|
|
list_tts_voices,
|
|
parse_tags,
|
|
prepare_tts_backend,
|
|
read_sentences,
|
|
synthesize_tts_sample,
|
|
supported_tts_backends,
|
|
)
|
|
from saiki.text import extract_first_visible_line, extract_visible_text
|
|
from saiki.words import (
|
|
build_query_from_decks,
|
|
clean_anki_field_text,
|
|
compare_word_files,
|
|
compare_word_lists,
|
|
extract_detailed_counts,
|
|
extract_words_from_file,
|
|
normalize_word_for_comparison,
|
|
read_word_file,
|
|
spanish_filter,
|
|
write_debug_tsv,
|
|
)
|
|
from saiki.youtube import TranscriptLine, extract_video_id, sentence_vocab, write_sentence_export
|
|
|
|
|
|
class ConfigTests(unittest.TestCase):
|
|
def test_deep_merge_preserves_nested_defaults(self):
|
|
merged = deep_merge(DEFAULT_CONFIG, {"languages": {"es": {"decks": ["Spanish"]}}})
|
|
self.assertEqual(merged["languages"]["es"]["decks"], ["Spanish"])
|
|
self.assertEqual(merged["languages"]["es"]["transcript_code"], "es")
|
|
self.assertEqual(merged["tts_model_dir"], "~/.local/share/saiki/models")
|
|
self.assertIn("jp", merged["languages"])
|
|
|
|
|
|
class TextTests(unittest.TestCase):
|
|
def test_visible_text_helpers_strip_html(self):
|
|
raw = "<div>Hola mundo</div><br><p>segunda linea</p>"
|
|
self.assertEqual(extract_first_visible_line(raw), "Hola\xa0mundo")
|
|
self.assertEqual(extract_visible_text(raw), "Hola\xa0mundo\nsegunda linea")
|
|
|
|
|
|
class AudioTests(unittest.TestCase):
|
|
def test_resolve_media_paths_rejects_unsafe_names(self):
|
|
self.assertIsNone(resolve_media_paths("/media", "/out", "../secret.mp3"))
|
|
self.assertIsNone(resolve_media_paths("/media", "/out", "/tmp/secret.mp3"))
|
|
self.assertEqual(
|
|
resolve_media_paths("/media", "/out", "nested/audio.mp3"),
|
|
("/media/nested/audio.mp3", "/out/nested/audio.mp3"),
|
|
)
|
|
|
|
def test_build_playlist_includes_audio_files(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
os.makedirs(os.path.join(tmp, "nested"))
|
|
for rel in ["b.mp3", "nested/a.ogg", "note.txt", "spanish_concat.mp3"]:
|
|
with open(os.path.join(tmp, rel), "w", encoding="utf-8") as fh:
|
|
fh.write("x")
|
|
playlist = build_playlist(tmp, "spanish")
|
|
with open(playlist, "r", encoding="utf-8") as fh:
|
|
self.assertEqual(fh.read().splitlines(), ["b.mp3", "nested/a.ogg"])
|
|
|
|
|
|
class WordsTests(unittest.TestCase):
|
|
def test_build_query_from_decks(self):
|
|
self.assertEqual(build_query_from_decks(["A", "B"]), 'deck:"A" OR deck:"B"')
|
|
|
|
def test_compare_word_files(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
source = os.path.join(tmp, "source.txt")
|
|
known = os.path.join(tmp, "known.txt")
|
|
with open(source, "w", encoding="utf-8") as fh:
|
|
fh.write("comer 3\nhablar 2\n")
|
|
with open(known, "w", encoding="utf-8") as fh:
|
|
fh.write("Comer 10\n")
|
|
self.assertEqual(read_word_file(known), {"comer"})
|
|
self.assertEqual(compare_word_files(source, known), ["hablar 2"])
|
|
|
|
|
|
class AnkiTsvTests(unittest.TestCase):
|
|
def test_parse_headers_separator_tab(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "test.txt")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write("#separator:tab\n#html:true\n#tags column:3\ncol1\tcol2\tcol3\n")
|
|
headers = parse_anki_headers(path)
|
|
self.assertEqual(headers["separator"], "\t")
|
|
self.assertTrue(headers["html"])
|
|
self.assertEqual(headers["tags_column"], "3")
|
|
|
|
def test_parse_headers_default_separator(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "test.txt")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write("# some comment\ncol1\tcol2\n")
|
|
headers = parse_anki_headers(path)
|
|
self.assertNotIn("separator", headers)
|
|
|
|
def test_parse_tsv_skips_header_lines(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "test.txt")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write("#separator:tab\n#html:true\nA\tB\tC\nD\tE\tF\n")
|
|
records = parse_anki_tsv(path, field_index=2)
|
|
self.assertEqual(len(records), 2)
|
|
self.assertEqual(records[0]["text"], "B")
|
|
self.assertEqual(records[1]["text"], "E")
|
|
|
|
def test_parse_tsv_field_index_1_based(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "test.txt")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write("alpha\tbeta\tgamma\n")
|
|
records = parse_anki_tsv(path, field_index=1)
|
|
self.assertEqual(records[0]["text"], "alpha")
|
|
records = parse_anki_tsv(path, field_index=3)
|
|
self.assertEqual(records[0]["text"], "gamma")
|
|
|
|
def test_parse_tsv_default_field_2_ignores_sound_field(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "test.txt")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write("#separator:tab\n#tags column:3\n")
|
|
f.write("[sound:es_001.mp3]\tYo como manzanas.\tanki-tag\n")
|
|
records = parse_anki_tsv(path)
|
|
self.assertEqual(records[0]["text"], "Yo como manzanas.")
|
|
self.assertNotIn("tags", records[0])
|
|
|
|
def test_parse_tsv_rejects_zero_field_index(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "test.txt")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write("alpha\tbeta\tgamma\n")
|
|
with self.assertRaises(ValueError):
|
|
parse_anki_tsv(path, field_index=0)
|
|
|
|
def test_parse_tsv_rejects_missing_field(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "test.txt")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write("alpha\tbeta\n")
|
|
with self.assertRaises(ValueError):
|
|
parse_anki_tsv(path, field_index=3)
|
|
|
|
def test_parse_tsv_line_numbers(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "test.txt")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write("#header\nuno\tdos\nthree\tfour\n")
|
|
records = parse_anki_tsv(path, field_index=2)
|
|
self.assertEqual(records[0]["line_number"], 2)
|
|
self.assertEqual(records[1]["line_number"], 3)
|
|
|
|
def test_parse_tsv_preserves_hash_data_rows_after_headers(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "test.txt")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write("#separator:tab\nalpha\tbeta\n#hashtag\tvalue\n")
|
|
records = parse_anki_tsv(path, field_index=1)
|
|
self.assertEqual([r["text"] for r in records], ["alpha", "#hashtag"])
|
|
self.assertEqual(records[1]["line_number"], 3)
|
|
|
|
def test_parse_tsv_include_tags(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "test.txt")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write("#tags column:3\nx\ty\ttag_a\n")
|
|
records = parse_anki_tsv(path, field_index=2, include_tags=True)
|
|
self.assertEqual(records[0]["tags"], "tag_a")
|
|
|
|
def test_parse_tsv_without_tags(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "test.txt")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write("#tags column:3\nx\ty\ttag_a\n")
|
|
records = parse_anki_tsv(path, field_index=2, include_tags=False)
|
|
self.assertNotIn("tags", records[0])
|
|
|
|
|
|
class CliValidationTests(unittest.TestCase):
|
|
"""Test CLI-level validation against the real parser/main paths."""
|
|
|
|
def test_words_accepts_lang_option_and_output_alias(self):
|
|
with patch("saiki.cli.importlib.metadata.version", return_value="0"):
|
|
parser = build_parser()
|
|
args = parser.parse_args([
|
|
"words", "--lang", "es", "--input", "deck.tsv", "--output", "out.txt"
|
|
])
|
|
self.assertEqual(args.lang_option, "es")
|
|
self.assertEqual(args.out, "out.txt")
|
|
|
|
def test_words_input_rejects_zero_field(self):
|
|
from io import StringIO
|
|
|
|
with patch("saiki.cli.importlib.metadata.version", return_value="0"):
|
|
with patch("sys.stderr", new_callable=StringIO) as stderr:
|
|
code = main(["words", "--lang", "es", "--input", "deck.tsv", "--field", "0"])
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("1-based", stderr.getvalue())
|
|
|
|
def test_words_input_accepts_japanese_tsv(self):
|
|
from io import StringIO
|
|
|
|
with patch("saiki.cli.importlib.metadata.version", return_value="0"):
|
|
with patch("saiki.cli.extract_words_from_file") as extract:
|
|
extract.return_value = {"records": 0, "written": 0, "out": "out.txt"}
|
|
with patch("sys.stdout", new_callable=StringIO):
|
|
code = main(["words", "jp", "--input", "deck.tsv", "--field", "2"])
|
|
self.assertEqual(code, 0)
|
|
extract.assert_called_once()
|
|
|
|
|
|
class AnkiFieldCleaningTests(unittest.TestCase):
|
|
def test_remove_sound_marker(self):
|
|
self.assertEqual(
|
|
clean_anki_field_text("[sound:es_001.mp3] Hola mundo"),
|
|
"Hola mundo",
|
|
)
|
|
|
|
def test_strip_html_and_unescape(self):
|
|
result = clean_anki_field_text("<span>Para el lunes...</span>")
|
|
self.assertEqual(result, "Para el lunes...")
|
|
|
|
def test_double_br_sections_are_preserved_by_default(self):
|
|
result = clean_anki_field_text("Target sentence<br><br>(Translation sentence)")
|
|
self.assertEqual(result, "Target sentence (Translation sentence)")
|
|
|
|
def test_first_field_section_drops_later_sections_without_phrase_rules(self):
|
|
result = clean_anki_field_text("Target sentence<br><br>Translation sentence")
|
|
self.assertEqual(result, "Target sentence Translation sentence")
|
|
|
|
result = clean_anki_field_text(
|
|
"Target sentence<br><br>Translation sentence",
|
|
field_section="first",
|
|
)
|
|
self.assertEqual(result, "Target sentence")
|
|
|
|
def test_later_field_sections_are_preserved_by_default(self):
|
|
result = clean_anki_field_text("First line<br><br>Second line")
|
|
self.assertEqual(result, "First line Second line")
|
|
|
|
def test_normalize_whitespace(self):
|
|
result = clean_anki_field_text(" Hola mundo ")
|
|
self.assertEqual(result, "Hola mundo")
|
|
|
|
def test_br_variants(self):
|
|
result = clean_anki_field_text("Line1<br/>Line2<br />Line3")
|
|
self.assertEqual(result, "Line1 Line2 Line3")
|
|
|
|
def test_preserves_accents(self):
|
|
result = clean_anki_field_text("camión corazón")
|
|
self.assertEqual(result, "camión corazón")
|
|
|
|
def test_cleaned_text_empty_input(self):
|
|
self.assertEqual(clean_anki_field_text(""), "")
|
|
self.assertEqual(clean_anki_field_text(None), "")
|
|
|
|
|
|
class MockToken:
|
|
"""Minimal mock of a spaCy token for lemmatisation tests."""
|
|
|
|
def __init__(self, text="", lemma="", pos="", **kwargs):
|
|
self.text = text
|
|
self.lemma_ = lemma
|
|
self.pos_ = pos
|
|
self.is_stop = kwargs.get("is_stop", False)
|
|
self.is_alpha = kwargs.get("is_alpha", True)
|
|
self.is_punct = kwargs.get("is_punct", False)
|
|
self.is_space = kwargs.get("is_space", False)
|
|
self.is_digit = kwargs.get("is_digit", False)
|
|
self.is_currency = kwargs.get("is_currency", False)
|
|
self.like_url = kwargs.get("like_url", False)
|
|
self.like_email = kwargs.get("like_email", False)
|
|
self.sent = None
|
|
|
|
|
|
class DetailedCountsTests(unittest.TestCase):
|
|
def test_spanish_filter_uses_content_pos(self):
|
|
self.assertTrue(spanish_filter(MockToken(text="casa", lemma="casa", pos="NOUN")))
|
|
self.assertTrue(spanish_filter(MockToken(text="rápido", lemma="rápido", pos="ADV")))
|
|
self.assertFalse(spanish_filter(MockToken(text="Madrid", lemma="Madrid", pos="PROPN")))
|
|
self.assertFalse(spanish_filter(MockToken(text="para", lemma="para", pos="ADP")))
|
|
|
|
def test_extract_detailed_counts_basic(self):
|
|
records = [
|
|
{"text": "Yo como manzanas.", "line_number": 1},
|
|
{"text": "Tú comes pan.", "line_number": 2},
|
|
]
|
|
|
|
class MockDoc:
|
|
def __init__(self, tokens):
|
|
self.tokens = tokens
|
|
|
|
def __iter__(self):
|
|
return iter(self.tokens)
|
|
|
|
class MockNLP:
|
|
def __call__(self, text):
|
|
if "como" in text:
|
|
return MockDoc([
|
|
MockToken(text="Yo", lemma="yo", pos="PRON"),
|
|
MockToken(text="como", lemma="comer", pos="VERB"),
|
|
MockToken(text="manzanas", lemma="manzana", pos="NOUN"),
|
|
MockToken(text=".", lemma=".", pos="PUNCT", is_punct=True),
|
|
])
|
|
return MockDoc([
|
|
MockToken(text="Tú", lemma="tú", pos="PRON"),
|
|
MockToken(text="comes", lemma="comer", pos="VERB"),
|
|
MockToken(text="pan", lemma="pan", pos="NOUN"),
|
|
MockToken(text=".", lemma=".", pos="PUNCT", is_punct=True),
|
|
])
|
|
|
|
stats = extract_detailed_counts(
|
|
records,
|
|
MockNLP(),
|
|
lambda t: t.pos_ in {"NOUN", "VERB", "ADJ", "ADV"},
|
|
lambda t: (t.lemma_ or t.text).lower(),
|
|
)
|
|
|
|
self.assertIn("comer", stats)
|
|
self.assertEqual(stats["comer"]["count"], 2)
|
|
self.assertIn("manzana", stats)
|
|
self.assertIn("pan", stats)
|
|
|
|
def test_debug_tsv_output(self):
|
|
stats = {
|
|
"comer": {
|
|
"count": 3,
|
|
"pos_counts": {"VERB": 3},
|
|
"surface_forms": ["como", "comes", "come"],
|
|
"example_sentences": ["Yo como.", "Tú comes."],
|
|
"source_lines": ["1", "2", "3"],
|
|
}
|
|
}
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "debug.tsv")
|
|
write_debug_tsv(stats, path)
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
content = f.read()
|
|
self.assertIn("comer", content)
|
|
self.assertIn("VERB:3", content)
|
|
self.assertIn("como, comes, come", content)
|
|
self.assertIn("Yo como.", content)
|
|
self.assertIn("1, 2, 3", content)
|
|
|
|
def test_debug_tsv_min_freq_filters(self):
|
|
stats = {
|
|
"comer": {"count": 5, "pos_counts": {"VERB": 5},
|
|
"surface_forms": ["come"], "example_sentences": ["Yo come."],
|
|
"source_lines": ["1"]},
|
|
"hablar": {"count": 2, "pos_counts": {"VERB": 2},
|
|
"surface_forms": ["habla"], "example_sentences": ["Él habla."],
|
|
"source_lines": ["2"]},
|
|
}
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "debug.tsv")
|
|
write_debug_tsv(stats, path, min_freq=3)
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
content = f.read()
|
|
self.assertIn("comer", content)
|
|
self.assertNotIn("hablar", content)
|
|
|
|
def test_extract_detailed_counts_no_clean(self):
|
|
class MockDoc:
|
|
def __init__(self, tokens):
|
|
self.tokens = tokens
|
|
def __iter__(self):
|
|
return iter(self.tokens)
|
|
|
|
class MockNLP:
|
|
def __call__(self, text):
|
|
return MockDoc([
|
|
MockToken(text="[sound:x.mp3]", lemma="[sound:x.mp3]", pos="X"),
|
|
])
|
|
|
|
records = [{"text": "[sound:x.mp3]", "line_number": 1}]
|
|
stats = extract_detailed_counts(
|
|
records, MockNLP(),
|
|
lambda t: True,
|
|
lambda t: t.text.lower(),
|
|
clean=False,
|
|
)
|
|
# With clean=False, [sound:x.mp3] is NOT stripped
|
|
self.assertIn("[sound:x.mp3]", stats)
|
|
|
|
stats_clean = extract_detailed_counts(
|
|
records, MockNLP(),
|
|
lambda t: True,
|
|
lambda t: t.text.lower(),
|
|
clean=True,
|
|
)
|
|
# With clean=True, the sound marker is stripped, so there is no text to count.
|
|
self.assertEqual(len(stats_clean), 0)
|
|
|
|
def test_extract_detailed_counts_can_use_first_field_section(self):
|
|
class MockDoc:
|
|
def __init__(self, tokens):
|
|
self.tokens = tokens
|
|
def __iter__(self):
|
|
return iter(self.tokens)
|
|
|
|
class MockNLP:
|
|
def __call__(self, text):
|
|
return MockDoc([
|
|
MockToken(text=part, lemma=part.lower(), pos="NOUN")
|
|
for part in text.split()
|
|
])
|
|
|
|
records = [{"text": "Target<br><br>Translation", "line_number": 1}]
|
|
stats = extract_detailed_counts(
|
|
records,
|
|
MockNLP(),
|
|
lambda t: True,
|
|
lambda t: t.lemma_,
|
|
field_section="first",
|
|
)
|
|
self.assertIn("target", stats)
|
|
self.assertNotIn("translation", stats)
|
|
|
|
def test_extract_words_from_file_uses_language_profile(self):
|
|
class MockDoc:
|
|
def __init__(self, tokens):
|
|
self.tokens = tokens
|
|
def __iter__(self):
|
|
return iter(self.tokens)
|
|
|
|
class MockNLP:
|
|
def __call__(self, text):
|
|
return MockDoc([
|
|
MockToken(text="猫", lemma="猫", pos="NOUN"),
|
|
MockToken(text="です", lemma="です", pos="AUX"),
|
|
])
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
source = os.path.join(tmp, "deck.tsv")
|
|
out = os.path.join(tmp, "words_jp.txt")
|
|
with open(source, "w", encoding="utf-8") as f:
|
|
f.write("#separator:tab\n[sound:jp_001.mp3]\t猫です。\n")
|
|
|
|
with patch("saiki.words.load_spacy_model", return_value=MockNLP()):
|
|
result = extract_words_from_file(
|
|
Config(deepcopy(DEFAULT_CONFIG)),
|
|
"jp",
|
|
source,
|
|
out=out,
|
|
min_freq=1,
|
|
)
|
|
|
|
with open(out, "r", encoding="utf-8") as f:
|
|
content = f.read()
|
|
|
|
self.assertEqual(result["records"], 1)
|
|
self.assertIn("猫 1", content)
|
|
self.assertNotIn("です", content)
|
|
|
|
|
|
class CompareWordListsTests(unittest.TestCase):
|
|
def test_normalize_word_for_comparison(self):
|
|
self.assertEqual(normalize_word_for_comparison("camión"), "camion")
|
|
self.assertEqual(normalize_word_for_comparison("Corazón"), "corazon")
|
|
self.assertEqual(normalize_word_for_comparison("ESPAÑOL"), "espanol")
|
|
|
|
def test_compare_word_lists_identifies_missing(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
deck = os.path.join(tmp, "deck.txt")
|
|
target = os.path.join(tmp, "target.txt")
|
|
with open(deck, "w", encoding="utf-8") as f:
|
|
f.write("comer 5\nhablar 3\n")
|
|
with open(target, "w", encoding="utf-8") as f:
|
|
f.write("comer 10\nvivir 8\ncorrer 6\n")
|
|
result = compare_word_lists(deck, target)
|
|
self.assertIn("vivir", result["missing"])
|
|
self.assertIn("correr", result["missing"])
|
|
self.assertNotIn("comer", result["missing"])
|
|
|
|
def test_compare_word_lists_with_min_frequency(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
deck = os.path.join(tmp, "deck.txt")
|
|
target = os.path.join(tmp, "target.txt")
|
|
with open(deck, "w", encoding="utf-8") as f:
|
|
f.write("comer 1\nhablar 5\n")
|
|
with open(target, "w", encoding="utf-8") as f:
|
|
f.write("comer 10\nhablar 10\n")
|
|
result = compare_word_lists(deck, target, min_frequency=3)
|
|
# "comer" has freq 1 < 3, so it's excluded from deck entirely
|
|
# meaning it should be "missing" from deck's perspective
|
|
self.assertIn("comer", result["missing"])
|
|
self.assertNotIn("hablar", result["missing"])
|
|
self.assertIn("hablar", result["seen"])
|
|
|
|
def test_compare_word_lists_writes_output(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
deck = os.path.join(tmp, "deck.txt")
|
|
target = os.path.join(tmp, "target.txt")
|
|
out = os.path.join(tmp, "output.txt")
|
|
with open(deck, "w", encoding="utf-8") as f:
|
|
f.write("comer 5\n")
|
|
with open(target, "w", encoding="utf-8") as f:
|
|
f.write("comer 10\nvivir 8\n")
|
|
result = compare_word_lists(deck, target, output_path=out)
|
|
self.assertEqual(result["total_missing"], 1)
|
|
with open(out, "r", encoding="utf-8") as f:
|
|
self.assertIn("vivir", f.read())
|
|
|
|
def test_compare_word_lists_writes_seen_output(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
deck = os.path.join(tmp, "deck.txt")
|
|
target = os.path.join(tmp, "target.txt")
|
|
seen_out = os.path.join(tmp, "seen.txt")
|
|
with open(deck, "w", encoding="utf-8") as f:
|
|
f.write("comer 5\n")
|
|
with open(target, "w", encoding="utf-8") as f:
|
|
f.write("comer 10\nvivir 8\n")
|
|
result = compare_word_lists(deck, target, seen_output_path=seen_out)
|
|
self.assertIn("comer", result["seen"])
|
|
with open(seen_out, "r", encoding="utf-8") as f:
|
|
self.assertIn("comer", f.read())
|
|
self.assertNotIn("vivir", f.read())
|
|
|
|
|
|
class YoutubeTests(unittest.TestCase):
|
|
def test_extract_video_id(self):
|
|
self.assertEqual(extract_video_id("https://youtu.be/abc123"), "abc123")
|
|
self.assertEqual(extract_video_id("https://www.youtube.com/watch?v=abc123&t=5"), "abc123")
|
|
self.assertEqual(extract_video_id("abc123"), "abc123")
|
|
|
|
def test_sentence_vocab_filters_known_words(self):
|
|
self.assertEqual(sentence_vocab("Hola hola mundo", "es", {"hola"}), ["mundo"])
|
|
|
|
def test_write_sentence_export(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
out = os.path.join(tmp, "sentences.tsv")
|
|
written = write_sentence_export(
|
|
[TranscriptLine(12.3, "Hola mundo")],
|
|
out,
|
|
"abc123",
|
|
"es",
|
|
)
|
|
self.assertEqual(written, 1)
|
|
with open(out, "r", encoding="utf-8") as fh:
|
|
rows = fh.read().splitlines()
|
|
self.assertEqual(rows[0], "sentence\ttimestamp\tvideo_url\tvocab_guess")
|
|
self.assertEqual(rows[1], "Hola mundo\t12.30\thttps://www.youtube.com/watch?v=abc123\thola, mundo")
|
|
|
|
|
|
class ImporterTests(unittest.TestCase):
|
|
def test_parse_tags(self):
|
|
self.assertEqual(parse_tags(None), ["text-to-speech", "AI-generated"])
|
|
self.assertEqual(parse_tags("youtube, manual "), ["text-to-speech", "youtube", "manual"])
|
|
|
|
def test_read_sentences_from_tsv_export(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "youtube.tsv")
|
|
with open(path, "w", encoding="utf-8") as fh:
|
|
fh.write("sentence\ttimestamp\tvideo_url\tvocab_guess\n")
|
|
fh.write("Hola mundo\t1.00\thttps://example.test\tmundo\n")
|
|
self.assertEqual(read_sentences(path), ["Hola mundo"])
|
|
|
|
def test_supported_tts_backends_are_free_options(self):
|
|
self.assertEqual(supported_tts_backends(), ["edge-tts", "espeak-ng", "gtts", "kokoro", "piper"])
|
|
|
|
def test_list_gtts_voice_hint(self):
|
|
config = Config(deep_merge(deepcopy(DEFAULT_CONFIG), {"languages": {"es": {"tts_code": "es", "tts_tld": "es"}}}))
|
|
voices = list_tts_voices(config, "es", backend="gtts")
|
|
self.assertIn("gtts does not expose named voices.", voices)
|
|
self.assertIn("tts_code=es", voices[1])
|
|
|
|
def test_list_edge_voice_uses_configured_voice(self):
|
|
voices = list_tts_voices(Config(deepcopy(DEFAULT_CONFIG)), "es")
|
|
self.assertEqual(voices[0], "Configured edge-tts voice: es-ES-ElviraNeural")
|
|
|
|
def test_prepare_tts_backend_validates_required_keys(self):
|
|
with self.assertRaisesRegex(RuntimeError, "tts_voice"):
|
|
prepare_tts_backend({"tts_backend": "edge-tts"})
|
|
|
|
def test_prepare_tts_backend_expands_model_paths(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
model = os.path.join(tmp, "voice.onnx")
|
|
config = os.path.join(tmp, "voice.onnx.json")
|
|
for path in [model, config]:
|
|
with open(path, "w", encoding="utf-8") as fh:
|
|
fh.write("{}")
|
|
|
|
with patch("saiki.importer.require_command"):
|
|
backend = prepare_tts_backend(
|
|
{
|
|
"tts_backend": "piper",
|
|
"tts_model_dir": tmp,
|
|
"tts_model": "voice.onnx",
|
|
"tts_config": "voice.onnx.json",
|
|
}
|
|
)
|
|
|
|
with patch("saiki.importer.subprocess.run") as run:
|
|
backend.synthesize("Hola", "/tmp/out.wav")
|
|
|
|
args = run.call_args.args[0]
|
|
self.assertEqual(args[2], model)
|
|
self.assertEqual(args[4], config)
|
|
self.assertEqual(run.call_args.kwargs["input"], b"Hola\n")
|
|
|
|
def test_synthesize_tts_sample_uses_backend_and_speed_audio(self):
|
|
seen: dict[str, str] = {}
|
|
|
|
def synthesize(sentence: str, output: str) -> None:
|
|
seen["sentence"] = sentence
|
|
seen["raw_output"] = output
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
output = os.path.join(tmp, "sample.mp3")
|
|
with patch("saiki.importer.prepare_tts_backend") as prepare, patch(
|
|
"saiki.importer.require_command"
|
|
), patch("saiki.importer.speed_audio") as speed:
|
|
prepare.return_value = PreparedTtsBackend("fake", ".wav", synthesize)
|
|
result = synthesize_tts_sample(Config(deepcopy(DEFAULT_CONFIG)), "es", output=output)
|
|
|
|
self.assertEqual(result, output)
|
|
self.assertEqual(seen["sentence"], "Esta es una prueba.")
|
|
self.assertTrue(seen["raw_output"].endswith(".wav"))
|
|
speed.assert_called_once()
|
|
|
|
def test_import_sentences_returns_error_details(self):
|
|
def fail_synthesis(sentence: str, output: str) -> None:
|
|
raise RuntimeError("tts broke")
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "sentences.txt")
|
|
with open(path, "w", encoding="utf-8") as fh:
|
|
fh.write("Hola mundo\n")
|
|
|
|
with patch("saiki.importer.prepare_tts_backend") as prepare, patch("saiki.importer.require_command"):
|
|
prepare.return_value = PreparedTtsBackend("fake", ".mp3", fail_synthesis)
|
|
result = import_sentences(Config(deepcopy(DEFAULT_CONFIG)), "es", path, request=lambda *a, **k: None)
|
|
|
|
self.assertEqual(result.processed, 1)
|
|
self.assertEqual(result.added, 0)
|
|
self.assertEqual(result.failed, 1)
|
|
self.assertEqual(result.errors, ["'Hola mundo': tts broke"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|