ps/Saiki
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add shared config and update tool scripts

Browse Source
This commit is contained in:
Pawel
2026-05-25 07:07:13 -04:00
parent f46ff5d9b0
commit 8ee1f8de25
8 changed files with 196 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored Normal file
View File

@@ -0,0 +1,4 @@
__pycache__/
*.py[cod]
.venv/
venv/

29
README.md
View File

@@ -29,10 +29,10 @@ Personally, I like to have one venv that contains all the prerequisites.
python3.12 -m venv ~/.venv/anki-tools
source ~/.venv/anki-tools/bin/activate
python3 -m pip install -U pip
pip install gtts jq yq spacy youtube-transcript-api pyyaml genanki fugashi regex requests
pip install -r requirements.txt
# Also install ffmpeg
sudo dnf install ffmpeg
# Also install system command-line dependencies
sudo dnf install ffmpeg jq
```
That way, whenever you want to run these scripts, you can just source the venv and run the appropriate script.
@@ -55,6 +55,17 @@ Most scripts assume:
- that your anki cards are basic, with audio on the front and the sentence (in the target language) on the back. These tools only look at the first line of the back, so you can have notes/translations/etc. on the following lines if you like.
![anki_basic_card_jp](./figures/anki_basic_card_jp.png)
### Shared configuration
Common settings live in `anki_common.py`, including:
- the AnkiConnect URL
- language code mappings (`jp`, `es`)
- deck-to-language mappings
- default output directories
- the default Anki `collection.media` path used by `audio_extractor.py`
If you rename your decks, add another language, or use a different default media location, update `anki_common.py` once instead of editing each script separately. Some settings can also be overridden at runtime, such as `audio_extractor.py --media-dir`.
### Language support
- 🇯🇵 日本語
- 🇪🇸 Español
@@ -66,15 +77,18 @@ Most scripts assume:
### Usage:
```bash
./audio_extractor.py jp [--concat] [--outdir DIR] [--copy-only-new]
./audio_extractor.py es [--concat] [--outdir DIR] [--copy-only-new]
./audio_extractor.py jp [--concat] [--outdir DIR] [--media-dir DIR] [--copy-only-new]
./audio_extractor.py es [--concat] [--outdir DIR] [--media-dir DIR] [--copy-only-new]
```
Outputs:
- Copies audio into `~/Languages/Anki/anki-audio/<language>/` by default
- Writes `<language>.m3u`
- Writes `<language>.m3u`, including audio copied into subfolders
- With `--concat`, writes `<language>_concat.mp3` (keeps individual files)
Options:
- `--media-dir DIR`: override the Anki `collection.media` directory. By default, this uses the common Flatpak path: `~/.var/app/net.ankiweb.Anki/data/Anki2/User 1/collection.media`
### Requirements
- Anki + AnkiConnect
- `requests`
@@ -103,7 +117,7 @@ Outputs:
### Requirements
- Anki + AnkiConnect
- `gtts-cli`, `ffmpeg`, `curl`
- `gtts-cli`, `ffmpeg`, `curl`, `jq`
### Sentence files
- Japanese: `~/Languages/Anki/sentences_jp.txt`
@@ -111,6 +125,7 @@ Outputs:
### Notes
- Audio files are generated in a temporary directory and cleaned up after import. No local audio files are retained.
- Sentences and tags are encoded as JSON with `jq`, so quotes and punctuation in sentence files are handled safely.
## word-scraper

47
anki_common.py Normal file
View File

@@ -0,0 +1,47 @@
#!/usr/bin/env python3
"""Shared configuration and AnkiConnect helpers for the toolkit scripts."""
from __future__ import annotations
import os
from typing import Dict
import requests
ANKI_CONNECT_URL = "http://localhost:8765"
LANG_MAP: Dict[str, str] = {
"jp": "japanese",
"es": "spanish",
}
TRANSCRIPT_LANG_MAP: Dict[str, str] = {
"jp": "ja",
"es": "es",
}
DECK_TO_LANGUAGE: Dict[str, str] = {
"日本語": "japanese",
"Español": "spanish",
}
DEFAULT_ANKI_MEDIA_DIR = os.path.expanduser(
"~/.var/app/net.ankiweb.Anki/data/Anki2/User 1/collection.media"
)
DEFAULT_AUDIO_OUTPUT_ROOT = os.path.expanduser("~/Languages/Anki/anki-audio")
DEFAULT_WORD_OUTPUT_ROOT = os.path.expanduser("~/Languages/Anki/anki-words")
def anki_request(action: str, **params):
"""Make an AnkiConnect request and return the result payload."""
resp = requests.post(
ANKI_CONNECT_URL,
json={"action": action, "version": 6, "params": params},
timeout=30,
)
resp.raise_for_status()
data = resp.json()
if data.get("error") is not None:
raise RuntimeError(f"AnkiConnect error for {action}: {data['error']}")
return data["result"]

98
audio_extractor.py
View File

@@ -28,66 +28,54 @@ import argparse
import shutil
import subprocess
import tempfile
from typing import Dict, List
from typing import List
import requests
# Map deck name -> language bucket
deck_to_language: Dict[str, str] = {
"日本語": "japanese",
"Español": "spanish",
# Add more mappings here
}
# Map CLI lang code -> language bucket
lang_map: Dict[str, str] = {
"jp": "japanese",
"es": "spanish",
}
# If Anki is installed as a flatpak, media dir is typically:
media_dir = os.path.expanduser("~/.var/app/net.ankiweb.Anki/data/Anki2/User 1/collection.media")
# Default export root (can be overridden by --outdir)
output_root = os.path.expanduser("~/Languages/Anki/anki-audio")
from anki_common import (
DEFAULT_ANKI_MEDIA_DIR,
DEFAULT_AUDIO_OUTPUT_ROOT,
DECK_TO_LANGUAGE,
LANG_MAP,
anki_request,
)
AUDIO_EXTS = (".mp3", ".wav", ".ogg", ".m4a", ".flac")
def anki_request(action: str, **params):
"""Make an AnkiConnect request and return 'result'. Raise on error."""
resp = requests.post(
"http://localhost:8765",
json={"action": action, "version": 6, "params": params},
timeout=30,
)
resp.raise_for_status()
data = resp.json()
if data.get("error") is not None:
raise RuntimeError(f"AnkiConnect error for {action}: {data['error']}")
return data["result"]
def ensure_ffmpeg_available() -> None:
"""Raise a helpful error if ffmpeg isn't installed."""
if shutil.which("ffmpeg") is None:
raise RuntimeError("ffmpeg not found in PATH. Install ffmpeg to use --concat.")
def resolve_media_paths(media_dir: str, out_dir: str, media_name: str) -> tuple[str, str] | None:
"""Return safe source/destination paths for an Anki media filename."""
normalized = os.path.normpath(media_name)
if os.path.isabs(normalized) or normalized.startswith(".."):
return None
return os.path.join(media_dir, normalized), os.path.join(out_dir, normalized)
def build_playlist(out_dir: str, language: str) -> str:
"""
Create an .m3u playlist listing audio files in out_dir (sorted by filename).
Create an .m3u playlist listing audio files under out_dir (sorted by filename).
Returns the playlist path.
"""
m3u_path = os.path.join(out_dir, f"{language}.m3u")
files = sorted(
f for f in os.listdir(out_dir)
if f.lower().endswith(AUDIO_EXTS) and os.path.isfile(os.path.join(out_dir, f))
)
concat_name = f"{language}_concat.mp3"
files: List[str] = []
for root, _, filenames in os.walk(out_dir):
for fname in filenames:
abs_path = os.path.join(root, fname)
rel_path = os.path.relpath(abs_path, out_dir)
if rel_path == os.path.basename(m3u_path):
continue
if rel_path == concat_name:
continue
if fname.lower().endswith(AUDIO_EXTS) and os.path.isfile(abs_path):
files.append(rel_path)
with open(m3u_path, "w", encoding="utf-8") as fh:
for fname in files:
for fname in sorted(files):
fh.write(f"{fname}\n")
return m3u_path
@@ -156,7 +144,7 @@ def main() -> int:
# REQUIRED positional language code: jp / es
parser.add_argument(
"lang",
choices=sorted(lang_map.keys()),
choices=sorted(LANG_MAP.keys()),
help="Language code (jp or es).",
)
@@ -170,6 +158,11 @@ def main() -> int:
"--outdir",
help="Output directory. Default: ~/Languages/Anki/anki-audio/<language>",
)
parser.add_argument(
"--media-dir",
default=DEFAULT_ANKI_MEDIA_DIR,
help="Anki collection.media directory. Defaults to the common Flatpak profile path.",
)
# Keep your existing useful behavior
parser.add_argument(
@@ -180,16 +173,17 @@ def main() -> int:
args = parser.parse_args()
language = lang_map[args.lang]
language = LANG_MAP[args.lang]
media_dir = os.path.expanduser(args.media_dir)
# Find all decks whose mapped language matches
selected_decks = [deck for deck, lang in deck_to_language.items() if lang == language]
selected_decks = [deck for deck, lang in DECK_TO_LANGUAGE.items() if lang == language]
if not selected_decks:
print(f"No decks found for language: {language}", file=sys.stderr)
return 1
# Output folder: either user-specified --outdir or default output_root/<language>
out_dir = os.path.expanduser(args.outdir) if args.outdir else os.path.join(output_root, language)
# Output folder: either user-specified --outdir or default output root/<language>
out_dir = os.path.expanduser(args.outdir) if args.outdir else os.path.join(DEFAULT_AUDIO_OUTPUT_ROOT, language)
os.makedirs(out_dir, exist_ok=True)
# Collect note IDs across selected decks
@@ -212,8 +206,11 @@ def main() -> int:
for field in fields.values():
val = field.get("value", "") or ""
for match in re.findall(r"\[sound:(.+?)\]", val):
src = os.path.join(media_dir, match)
dst = os.path.join(out_dir, match)
paths = resolve_media_paths(media_dir, out_dir, match)
if paths is None:
print(f"Skipping unsafe media reference: {match}", file=sys.stderr)
continue
src, dst = paths
if not os.path.exists(src):
continue
@@ -229,7 +226,7 @@ def main() -> int:
shutil.copy2(src, dst)
copied.append(match)
# Create playlist (top-level audio only; if you have subfolders, you can extend this)
# Create playlist, including audio in subfolders.
m3u_path = build_playlist(out_dir, language)
print(f"\n✅ Copied {len(copied)} files for {language}")
@@ -251,4 +248,3 @@ def main() -> int:
if __name__ == "__main__":
raise SystemExit(main())

78
batch_anki_import.sh
View File

@@ -60,19 +60,32 @@ done
[[ -z "$lang" ]] && arg_error_missing_lang
require_command() {
if ! command -v "$1" >/dev/null 2>&1; then
echo "$prog: error: required command not found: $1" >&2
exit 1
fi
}
require_command gtts-cli
require_command ffmpeg
require_command curl
require_command jq
# Build tags JSON array - text-to-speech is always included
TAGS='["text-to-speech"'
tags=("text-to-speech")
if [[ -n "$custom_tags" ]]; then
IFS=',' read -ra tag_array <<< "$custom_tags"
for tag in "${tag_array[@]}"; do
# Trim whitespace
tag="$(echo -e "$tag" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
TAGS+=", \"$tag\""
tag="$(printf '%s' "$tag" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
[[ -n "$tag" ]] && tags+=("$tag")
done
else
TAGS+=', "AI-generated"'
tags+=("AI-generated")
fi
TAGS+=']'
TAGS="$(printf '%s\n' "${tags[@]}" | jq -R . | jq -s .)"
case "$lang" in
jp)
@@ -93,8 +106,14 @@ esac
count=0
if [[ ! -f "$SENTENCE_FILE" ]]; then
echo "$prog: error: sentence file not found: $SENTENCE_FILE" >&2
exit 1
fi
# Use a temporary directory to handle processing
TEMP_DIR=$(mktemp -d)
trap 'rm -rf "$TEMP_DIR"' EXIT
while IFS= read -r sentence || [[ -n "$sentence" ]]; do
[[ -z "$sentence" ]] && continue
@@ -115,31 +134,39 @@ while IFS= read -r sentence || [[ -n "$sentence" ]]; do
if ffmpeg -loglevel error -i "$RAW_OUTPUT" -filter:a "atempo=$TEMPO" -y "$OUTPUT_PATH" < /dev/null; then
# 3. Add to Anki using the sped-up file
result=$(curl -s localhost:8765 -X POST -d "{
\"action\": \"addNote\",
\"version\": 6,
\"params\": {
\"note\": {
\"deckName\": \"$DECK_NAME\",
\"modelName\": \"Basic\",
\"fields\": {
\"Front\": \"\",
\"Back\": \"$sentence\"
payload="$(jq -n \
--arg deck "$DECK_NAME" \
--arg sentence "$sentence" \
--arg path "$OUTPUT_PATH" \
--arg filename "${BASENAME}.mp3" \
--argjson tags "$TAGS" \
'{
action: "addNote",
version: 6,
params: {
note: {
deckName: $deck,
modelName: "Basic",
fields: {
Front: "",
Back: $sentence
},
\"options\": {
\"allowDuplicate\": false
options: {
allowDuplicate: false
},
\"tags\": $TAGS,
\"audio\": [{
\"path\": \"$OUTPUT_PATH\",
\"filename\": \"${BASENAME}.mp3\",
\"fields\": [\"Front\"]
tags: $tags,
audio: [{
path: $path,
filename: $filename,
fields: ["Front"]
}]
}
}
}")
}')"
if [[ "$result" == *'"error": null'* ]]; then
result=$(curl -s localhost:8765 -X POST -H "Content-Type: application/json" -d "$payload")
if jq -e '.error == null' >/dev/null 2>&1 <<< "$result"; then
echo "✅ Added card: $sentence"
((count++))
else
@@ -158,7 +185,4 @@ while IFS= read -r sentence || [[ -n "$sentence" ]]; do
done <"$SENTENCE_FILE"
# Cleanup temp directory
rm -rf "$TEMP_DIR"
echo "🎉 Done! Added $count cards to deck \"$DECK_NAME\"."

8
requirements.txt Normal file
View File

@@ -0,0 +1,8 @@
requests
regex
spacy
youtube-transcript-api
fugashi[unidic-lite]
gTTS
pyyaml
genanki

56
word_scraper.py
View File

@@ -1,15 +1,15 @@
#!/usr/bin/env python3
"""
word_extractor.py
word_scraper.py
Extract frequent words/lemmas from Anki notes via AnkiConnect.
Howto:
./word_extractor.py jp [--deck "日本語"] [--field Back] [--min-freq 2] [--outdir DIR] [--out FILE]
./word_extractor.py es [--deck "Español"] [--field Back] [--min-freq 2] [--outdir DIR] [--out FILE]
./word_scraper.py jp [--deck "日本語"] [--field Back] [--min-freq 2] [--outdir DIR] [--out FILE]
./word_scraper.py es [--deck "Español"] [--field Back] [--min-freq 2] [--outdir DIR] [--out FILE]
By default, this:
- chooses decks based on the lang code (jp/es) using deck_to_language mappings
- chooses decks based on the lang code (jp/es) using shared deck mappings
- pulls notes from Anki via AnkiConnect (http://localhost:8765)
- reads a single field (default: Back)
- extracts the first visible line (HTML stripped) from that field
@@ -29,30 +29,11 @@ import os
import sys
from collections import Counter
from html import unescape
from typing import Callable, Dict, Iterable, List, Optional, Tuple
from typing import Callable, List
import requests
import regex as re
# -------------------------
# Shared “language plumbing”
# -------------------------
# Match the idea used in audio_extractor.py: CLI lang code -> language bucket. :contentReference[oaicite:2]{index=2}
LANG_MAP: Dict[str, str] = {
"jp": "japanese",
"es": "spanish",
}
# Map deck name -> language bucket (same pattern as audio_extractor.py). :contentReference[oaicite:3]{index=3}
DECK_TO_LANGUAGE: Dict[str, str] = {
"日本語": "japanese",
"Español": "spanish",
# Add more deck mappings here
}
# Default output root (mirrors the “one folder per language” idea)
DEFAULT_OUTPUT_ROOT = os.path.expanduser("~/Languages/Anki/anki-words")
from anki_common import DEFAULT_WORD_OUTPUT_ROOT, DECK_TO_LANGUAGE, LANG_MAP, anki_request
# -------------------------
@@ -90,26 +71,6 @@ def extract_visible_text(text: str) -> str:
return text.strip()
# -------------------------
# AnkiConnect helper
# -------------------------
def anki_request(action: str, **params):
"""
Make an AnkiConnect request and return 'result'.
Raises a helpful error if the HTTP call fails or AnkiConnect returns an error.
"""
resp = requests.post(
"http://localhost:8765",
json={"action": action, "version": 6, "params": params},
timeout=30,
)
resp.raise_for_status()
data = resp.json()
if data.get("error") is not None:
raise RuntimeError(f"AnkiConnect error for {action}: {data['error']}")
return data["result"]
def get_notes(query: str) -> List[dict]:
"""
Query Anki for notes and return notesInfo payload.
@@ -333,7 +294,7 @@ def main() -> int:
)
parser.add_argument(
"--logfile",
default=os.path.expanduser("~/Languages/Anki/anki-words/extract_words.log"),
default=os.path.join(DEFAULT_WORD_OUTPUT_ROOT, "extract_words.log"),
help="Log file path.",
)
@@ -361,7 +322,7 @@ def main() -> int:
query = build_query_from_decks(decks)
# Output paths
out_dir = os.path.expanduser(args.outdir) if args.outdir else os.path.join(DEFAULT_OUTPUT_ROOT, language_bucket)
out_dir = os.path.expanduser(args.outdir) if args.outdir else os.path.join(DEFAULT_WORD_OUTPUT_ROOT, language_bucket)
default_outfile = os.path.join(out_dir, f"words_{args.lang}.txt")
out_path = os.path.expanduser(args.out) if args.out else default_outfile
@@ -419,4 +380,3 @@ def main() -> int:
if __name__ == "__main__":
raise SystemExit(main())

12
yt-transcript.py
View File

@@ -28,14 +28,7 @@ from urllib.parse import urlparse, parse_qs
from youtube_transcript_api import YouTubeTranscriptApi
# -------------------------
# Language mapping
# -------------------------
LANG_MAP = {
"jp": "ja",
"es": "es",
}
from anki_common import TRANSCRIPT_LANG_MAP
# Small starter stopword lists (you can grow these over time)
STOPWORDS = {
@@ -160,7 +153,7 @@ def main() -> int:
)
args = parser.parse_args()
lang_code = LANG_MAP[args.lang]
lang_code = TRANSCRIPT_LANG_MAP[args.lang]
video_id = extract_video_id(args.video)
try:
@@ -196,4 +189,3 @@ def main() -> int:
if __name__ == "__main__":
raise SystemExit(main())