Added pyproject.toml, version flag, dry-run flag, etc.
This commit is contained in:
@@ -2,3 +2,6 @@ __pycache__/
|
||||
*.py[cod]
|
||||
.venv/
|
||||
venv/
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@
|
||||
## Unreleased
|
||||
|
||||
- Renamed the project to Saiki (`採記`).
|
||||
- Replaced the old script collection with one CLI: `./saiki.py`.
|
||||
- Replaced the old script collection with one CLI: `saiki`.
|
||||
- Added YAML configuration loading from `~/.config/saiki/config.yaml`.
|
||||
- Added YouTube transcript exports for Anki-ready sentence mining.
|
||||
- Added known/new word comparison.
|
||||
|
||||
@@ -9,16 +9,14 @@ The name is a coined Japanese compound from `採` as in gathering/collecting and
|
||||
"sigh-key".
|
||||
|
||||
```shell
|
||||
./saiki.py --help
|
||||
saiki --help
|
||||
```
|
||||
|
||||
## Requirements
|
||||
|
||||
- Python 3.12 recommended
|
||||
- Python 3.12+
|
||||
- [Anki](https://apps.ankiweb.net/) with [AnkiConnect](https://github.com/amikey/anki-connect)
|
||||
- `ffmpeg`
|
||||
- Python dependencies from `requirements.txt`
|
||||
- Optional extra TTS backend tools: `piper`, `espeak-ng`, and `kokoro-onnx`.
|
||||
- spaCy models for word mining:
|
||||
|
||||
```shell
|
||||
@@ -31,19 +29,29 @@ Setup example:
|
||||
```shell
|
||||
python3.12 -m venv ~/.venv/saiki
|
||||
source ~/.venv/saiki/bin/activate
|
||||
python3 -m pip install -U pip
|
||||
pip install -r requirements.txt
|
||||
pip install -U pip
|
||||
pip install -e .
|
||||
sudo dnf install ffmpeg
|
||||
```
|
||||
|
||||
### Updating
|
||||
|
||||
If you installed with `pip install -e .` (editable mode), changes to the source
|
||||
are live immediately — no reinstall needed. Only re-run `pip install -e .` if
|
||||
you pull changes that add new dependencies or scripts.
|
||||
|
||||
```shell
|
||||
git pull
|
||||
pip install -e . # only needed if pyproject.toml changed
|
||||
```
|
||||
|
||||
### Optional TTS Backends
|
||||
|
||||
The default `edge-tts` backend is installed by `requirements.txt`. Install only
|
||||
the optional pieces you plan to test:
|
||||
The default `edge-tts` backend is included. To install the optional Python-backed
|
||||
engines (piper, kokoro):
|
||||
|
||||
```shell
|
||||
# Python-backed optional engines: piper, kokoro.
|
||||
pip install -r requirements-tts.txt
|
||||
pip install ".[tts]"
|
||||
|
||||
# System package for espeak-ng.
|
||||
sudo dnf install espeak-ng
|
||||
@@ -103,7 +111,7 @@ Defaults are built in, but you can override them with YAML:
|
||||
Or pass a config explicitly:
|
||||
|
||||
```shell
|
||||
./saiki.py --config ./config.yaml words jp
|
||||
saiki --config ./config.yaml words jp
|
||||
```
|
||||
|
||||
Example:
|
||||
@@ -125,7 +133,7 @@ languages:
|
||||
transcript_code: ja
|
||||
tts_backend: edge-tts
|
||||
tts_voice: ja-JP-NanamiNeural
|
||||
tts_tempo: 1.15
|
||||
tts_tempo: 1
|
||||
decks: ["日本語"]
|
||||
field: Back
|
||||
word_model: ja_core_news_lg
|
||||
@@ -157,9 +165,9 @@ Extract audio referenced by `[sound:...]` tags from configured decks and create
|
||||
an `.m3u` playlist.
|
||||
|
||||
```shell
|
||||
./saiki.py audio jp
|
||||
./saiki.py audio es --concat
|
||||
./saiki.py audio jp --media-dir ~/.local/share/Anki2/User\ 1/collection.media --copy-only-new
|
||||
saiki audio jp
|
||||
saiki audio es --concat
|
||||
saiki audio jp --media-dir ~/.local/share/Anki2/User\ 1/collection.media --copy-only-new
|
||||
```
|
||||
|
||||
Outputs go to `~/Languages/Anki/anki-audio/<language>/` by default.
|
||||
@@ -169,11 +177,11 @@ Outputs go to `~/Languages/Anki/anki-audio/<language>/` by default.
|
||||
Extract frequent words from Anki notes using AnkiConnect and spaCy.
|
||||
|
||||
```shell
|
||||
./saiki.py words jp
|
||||
./saiki.py words es --deck "Español"
|
||||
./saiki.py words es --query 'deck:"Español" tag:youtube'
|
||||
./saiki.py words jp --min-freq 3 --out words_jp.txt
|
||||
./saiki.py words jp --full-field
|
||||
saiki words jp
|
||||
saiki words es --deck "Español"
|
||||
saiki words es --query 'deck:"Español" tag:youtube'
|
||||
saiki words jp --min-freq 3 --out words_jp.txt
|
||||
saiki words jp --full-field
|
||||
```
|
||||
|
||||
Output format:
|
||||
@@ -196,22 +204,22 @@ hablar 9
|
||||
Mine vocabulary or sentence rows from YouTube subtitles.
|
||||
|
||||
```shell
|
||||
./saiki.py youtube es VIDEO_ID
|
||||
./saiki.py youtube es VIDEO_ID --top 50
|
||||
./saiki.py youtube jp VIDEO_ID --mode sentences
|
||||
./saiki.py youtube es VIDEO_ID --raw --no-stopwords
|
||||
saiki youtube es VIDEO_ID
|
||||
saiki youtube es VIDEO_ID --top 50
|
||||
saiki youtube jp VIDEO_ID --mode sentences
|
||||
saiki youtube es VIDEO_ID --raw --no-stopwords
|
||||
```
|
||||
|
||||
Export Anki-ready sentence rows:
|
||||
|
||||
```shell
|
||||
./saiki.py youtube es VIDEO_ID --mode sentences --out youtube.tsv
|
||||
saiki youtube es VIDEO_ID --mode sentences --out youtube.tsv
|
||||
```
|
||||
|
||||
Export only rows that appear to contain unknown vocabulary:
|
||||
|
||||
```shell
|
||||
./saiki.py youtube es VIDEO_ID \
|
||||
saiki youtube es VIDEO_ID \
|
||||
--mode sentences \
|
||||
--out youtube_new.tsv \
|
||||
--known-words ~/Languages/Anki/anki-words/spanish/words_es.txt \
|
||||
@@ -229,10 +237,11 @@ sentence timestamp video_url vocab_guess
|
||||
Generate TTS audio and add sentence cards to Anki.
|
||||
|
||||
```shell
|
||||
./saiki.py import es
|
||||
./saiki.py import jp ~/Languages/Anki/sentences_jp.txt
|
||||
./saiki.py import es youtube.tsv --tags youtube,manual
|
||||
./saiki.py import es --tts-voice es-MX-DaliaNeural
|
||||
saiki import es
|
||||
saiki import jp ~/Languages/Anki/sentences_jp.txt
|
||||
saiki import es youtube.tsv --tags youtube,manual
|
||||
saiki import es --tts-voice es-MX-DaliaNeural
|
||||
saiki import es --dry-run # preview sentences without touching Anki or TTS
|
||||
```
|
||||
|
||||
The importer accepts plain text sentence files and TSV/CSV files with a
|
||||
@@ -258,7 +267,7 @@ TTS is configured per language with `tts_backend`. Supported backends are:
|
||||
You can override backend settings for one import:
|
||||
|
||||
```shell
|
||||
./saiki.py import jp sentences_jp.txt \
|
||||
saiki import jp sentences_jp.txt \
|
||||
--tts-backend edge-tts \
|
||||
--tts-voice ja-JP-KeitaNeural
|
||||
```
|
||||
@@ -266,20 +275,20 @@ You can override backend settings for one import:
|
||||
Voice-listing helpers:
|
||||
|
||||
```shell
|
||||
./saiki.py tts-voices jp
|
||||
./saiki.py tts-voices es --backend edge-tts
|
||||
saiki tts-voices jp
|
||||
saiki tts-voices es --backend edge-tts
|
||||
```
|
||||
|
||||
Test a TTS backend without creating Anki cards:
|
||||
|
||||
```shell
|
||||
./saiki.py tts-test es --out /tmp/saiki_edge_default_es.mp3
|
||||
./saiki.py tts-test jp --tts-backend edge-tts --tts-voice ja-JP-NanamiNeural --out /tmp/saiki_edge_jp.mp3
|
||||
./saiki.py tts-test es --tts-backend edge-tts --tts-voice es-ES-ElviraNeural --out /tmp/saiki_edge_es.mp3
|
||||
./saiki.py tts-test es --tts-backend gtts --tts-code es --tts-tld es --out /tmp/saiki_gtts_es.mp3
|
||||
./saiki.py tts-test es --tts-backend piper --tts-model es_ES-davefx-medium.onnx --tts-config es_ES-davefx-medium.onnx.json --out /tmp/saiki_piper_es.mp3
|
||||
./saiki.py tts-test es --tts-backend espeak-ng --tts-voice es --out /tmp/saiki_espeak_es.mp3
|
||||
./saiki.py tts-test es --tts-backend kokoro --tts-model kokoro-v1.0.onnx --tts-voices voices-v1.0.bin --tts-voice ef_dora --out /tmp/saiki_kokoro_es.mp3
|
||||
saiki tts-test es --out /tmp/saiki_edge_default_es.mp3
|
||||
saiki tts-test jp --tts-backend edge-tts --tts-voice ja-JP-NanamiNeural --out /tmp/saiki_edge_jp.mp3
|
||||
saiki tts-test es --tts-backend edge-tts --tts-voice es-ES-ElviraNeural --out /tmp/saiki_edge_es.mp3
|
||||
saiki tts-test es --tts-backend gtts --tts-code es --tts-tld es --out /tmp/saiki_gtts_es.mp3
|
||||
saiki tts-test es --tts-backend piper --tts-model es_ES-davefx-medium.onnx --tts-config es_ES-davefx-medium.onnx.json --out /tmp/saiki_piper_es.mp3
|
||||
saiki tts-test es --tts-backend espeak-ng --tts-voice es --out /tmp/saiki_espeak_es.mp3
|
||||
saiki tts-test es --tts-backend kokoro --tts-model kokoro-v1.0.onnx --tts-voices voices-v1.0.bin --tts-voice ef_dora --out /tmp/saiki_kokoro_es.mp3
|
||||
```
|
||||
|
||||
For `kokoro`, put `tts_model`, `tts_voices`, and any needed `tts_vocab_config`
|
||||
@@ -290,7 +299,7 @@ in your config file rather than typing every path each time.
|
||||
Compare any generated word list against an existing known list:
|
||||
|
||||
```shell
|
||||
./saiki.py compare-words transcript_words.txt ~/Languages/Anki/anki-words/spanish/words_es.txt
|
||||
saiki compare-words transcript_words.txt ~/Languages/Anki/anki-words/spanish/words_es.txt
|
||||
```
|
||||
|
||||
This prints entries from the first file whose word key does not appear in the
|
||||
@@ -318,20 +327,16 @@ line by default; use `--full-field` to process the whole field.
|
||||
- Improve known/new word matching with better lemmatization for transcript
|
||||
vocabulary.
|
||||
- Add more language profiles beyond Japanese and Spanish.
|
||||
- Add a dry-run mode for imports that previews notes before sending anything to
|
||||
AnkiConnect.
|
||||
- Build a GUI for common workflows like transcript review, sentence selection,
|
||||
import previews, and configuration editing.
|
||||
- Add integration tests with mocked AnkiConnect responses.
|
||||
- Add shell completion or a small installed command once packaging becomes
|
||||
useful.
|
||||
- Add shell completion.
|
||||
|
||||
## Tests
|
||||
|
||||
Pure logic tests use the standard library test runner:
|
||||
|
||||
```shell
|
||||
python -m unittest discover -s tests
|
||||
pip install -e ".[dev]"
|
||||
pytest
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
@@ -22,7 +22,7 @@ languages:
|
||||
jp:
|
||||
name: japanese
|
||||
transcript_code: ja
|
||||
tts_tempo: 1.15
|
||||
tts_tempo: 1
|
||||
decks:
|
||||
- 日本語
|
||||
field: Back
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "saiki"
|
||||
version = "0.1.0"
|
||||
description = "Toolkit for Anki-based language learning workflows"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"requests",
|
||||
"regex",
|
||||
"spacy",
|
||||
"youtube-transcript-api",
|
||||
"fugashi[unidic-lite]",
|
||||
"gTTS",
|
||||
"edge-tts",
|
||||
"pyyaml",
|
||||
"genanki",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
tts = [
|
||||
"piper-tts",
|
||||
"kokoro-onnx",
|
||||
"soundfile",
|
||||
]
|
||||
dev = [
|
||||
"pytest",
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
|
||||
[project.scripts]
|
||||
saiki = "saiki.cli:main"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/saiki"]
|
||||
@@ -1,5 +0,0 @@
|
||||
# Optional free TTS backends.
|
||||
# Install this only if you want to test/use non-default TTS engines.
|
||||
piper-tts
|
||||
kokoro-onnx
|
||||
soundfile
|
||||
@@ -1,9 +0,0 @@
|
||||
requests
|
||||
regex
|
||||
spacy
|
||||
youtube-transcript-api
|
||||
fugashi[unidic-lite]
|
||||
gTTS
|
||||
edge-tts
|
||||
pyyaml
|
||||
genanki
|
||||
@@ -1,17 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Saiki CLI entrypoint."""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SRC_DIR = Path(__file__).resolve().parent / "src"
|
||||
if str(SRC_DIR) not in sys.path:
|
||||
# Let the repository checkout run directly without requiring an editable
|
||||
# install first. Installed packages will still resolve normally.
|
||||
sys.path.insert(0, str(SRC_DIR))
|
||||
|
||||
from saiki.cli import main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+11
-1
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import importlib.metadata
|
||||
import sys
|
||||
|
||||
from .audio import extract_audio
|
||||
@@ -69,6 +70,11 @@ def build_parser(config: Config | None = None) -> argparse.ArgumentParser:
|
||||
choices = language_choices(config or load_config())
|
||||
tts_backends = supported_tts_backends()
|
||||
parser = argparse.ArgumentParser(description="Saiki: sentence mining and listening tools for Anki.")
|
||||
parser.add_argument(
|
||||
"-V", "--version",
|
||||
action="version",
|
||||
version=f"%(prog)s {importlib.metadata.version('saiki')}",
|
||||
)
|
||||
add_config_arg(parser)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
@@ -111,6 +117,7 @@ def build_parser(config: Config | None = None) -> argparse.ArgumentParser:
|
||||
importer.add_argument("lang", choices=choices)
|
||||
importer.add_argument("sentence_file", nargs="?")
|
||||
importer.add_argument("--tags", help="Comma-separated tags. text-to-speech is always included.")
|
||||
importer.add_argument("--dry-run", action="store_true", help="Preview sentences without generating TTS or adding cards.")
|
||||
add_tts_override_args(importer, tts_backends)
|
||||
|
||||
test_tts = sub.add_parser("tts-test", help="Synthesize one TTS sample without importing into Anki.")
|
||||
@@ -179,7 +186,10 @@ def main(argv: list[str] | None = None) -> int:
|
||||
|
||||
if args.command == "import":
|
||||
tts_overrides = collect_tts_overrides(args)
|
||||
result = import_sentences(config, args.lang, args.sentence_file, args.tags, tts_overrides=tts_overrides)
|
||||
result = import_sentences(config, args.lang, args.sentence_file, args.tags, tts_overrides=tts_overrides, dry_run=args.dry_run)
|
||||
if args.dry_run:
|
||||
print(f"Dry run: {result.processed} sentence(s) would be imported.")
|
||||
else:
|
||||
print(f"Done. Added {result.added}/{result.processed} cards. Failed: {result.failed}")
|
||||
for error in result.errors:
|
||||
print(f"Error: {error}", file=sys.stderr)
|
||||
|
||||
+1
-1
@@ -32,7 +32,7 @@ DEFAULT_CONFIG: dict[str, Any] = {
|
||||
"transcript_code": "ja",
|
||||
"tts_backend": "edge-tts",
|
||||
"tts_voice": "ja-JP-NanamiNeural",
|
||||
"tts_tempo": 1.15,
|
||||
"tts_tempo": 1,
|
||||
"decks": ["日本語"],
|
||||
"word_model": "ja_core_news_lg",
|
||||
"field": "Back",
|
||||
|
||||
@@ -495,6 +495,7 @@ def import_sentences(
|
||||
tags_value: str | None = None,
|
||||
request: Callable = anki_request,
|
||||
tts_overrides: Mapping[str, Any] | None = None,
|
||||
dry_run: bool = False,
|
||||
) -> ImportResult:
|
||||
"""Generate TTS for each sentence and add cards through AnkiConnect.
|
||||
|
||||
@@ -514,6 +515,12 @@ def import_sentences(
|
||||
|
||||
source = os.path.expanduser(sentence_file) if sentence_file else config.sentence_file_for(lang)
|
||||
sentences = read_sentences(source)
|
||||
|
||||
if dry_run:
|
||||
for sentence in sentences:
|
||||
print(f" {_sentence_label(sentence)}")
|
||||
return ImportResult(processed=len(sentences), added=0, failed=0, errors=[])
|
||||
|
||||
tags = parse_tags(tags_value)
|
||||
front_field = config.fields.get("front", "Front")
|
||||
back_field = config.fields.get("back", "Back")
|
||||
|
||||
Reference in New Issue
Block a user