from format import LyricLine, FuriBlock from fugashi import Tagger import pykakasi import re kks = pykakasi.kakasi() def to_romaji(japanese: str, separator=' ') -> str: return separator.join([word['hepburn'] for word in kks.convert(japanese)]) katakana_to_hiragana_dict = {0x30A0+i:chr(0x3040+i) for i in range(1, 87)} katakana_to_hiragana_dict[ord('*')] = ' ' kana_merge_previous_syllable = {k for k in 'ゃゅょぁぃぇぉぅゎんャュョァィェォゥヮン'} kana_merge_next_syllable = {k for k in 'っッ'} def kana_to_syllable_string(kana: str): syl = '' for k in (it := iter(kana)): if k in kana_merge_next_syllable: k += next(it) syl += '|' + k continue # Else if k not in kana_merge_previous_syllable: syl += '|' syl += k return syl def kana_to_syllable_list(kana: str) -> list[str]: # Kinda inefficient to work it as a string and then split it afterwards, but elegant return kana_to_syllable_string(kana)[1:].split('|') tagger = Tagger('-Owakati') def parse_jp_text(text: str) -> list[tuple[str, str]]: return [(str(token), token.feature.kana) for token in tagger(text)] # Our custom word overrides have two levels: # - One is a simple search-replace to turn matches into manual furigana "{kanji|furi}" format. This could have false hits on short words. # - The latter is to override a word's kana post-tokenization, which requires it to be a dictionary word with multiple readings. word_overrides = {'主': 'しゅ'} re_manual_furi = re.compile(r'{(.+?)\|(.+?)}') def manual_furi_string_to_blocks(line: str) -> list[FuriBlock]: last_unmatched_pos = 0 blocks = [] for match in re_manual_furi.finditer(line): start = match.start() end = match.end() if start > last_unmatched_pos: blocks.append(FuriBlock(line[last_unmatched_pos:start], '')) blocks.append(FuriBlock(*match.groups())) last_unmatched_pos = end if remainder := line[last_unmatched_pos:]: blocks.append(FuriBlock(remainder, '')) return blocks # For debugging def furi_blocks_reconstruction(blocks: list[FuriBlock]): kanji = ''.join([b.kanji for b in blocks]) kana = ''.join([b.furi if b.furi else b.kanji for b in blocks]) return kanji, kana def debug_parse_manual_furi_line(line: str): blocks = manual_furi_string_to_blocks(line) kanji, kana = furi_blocks_reconstruction(blocks) print(kanji) print(kana) syllables = kana_to_syllable_string(kana) print(syllables) romaji_syllables = '|'.join([to_romaji(syl) for syl in syllables.split('|')]) print(romaji_syllables) re_hiragana = re.compile(r'[\u3041-\u309f]+') def word_to_furi_blocks(kanji, hiragana) -> list[FuriBlock]: # On longer words this may incorrectly split and may need manual furi, which our pipeline is not set up for # The problem is okurigana matching to the hiragana # In words with multiple kanji split by okurigana, this is a hard problem. # Naive approach: pick out all the kana and make a regex # e.g. turn '打ち合わせ' into r'(.*)ち(.*)わせ' furi_blocks = [] kanji_block_indices = [] last_unmatched_pos = 0 furi_regex_pattern = '' for match in re_hiragana.finditer(kanji): start = match.start() if start > last_unmatched_pos: furi_regex_pattern += '(.+)' kanji_block_indices.append(len(furi_blocks)) furi_blocks.append(kanji[last_unmatched_pos:start]) # str not FuriBlock! furi = match.group(0) furi_regex_pattern += furi last_unmatched_pos = match.end() furi_blocks.append(FuriBlock(furi, '')) if remainder := kanji[last_unmatched_pos:]: furi_regex_pattern += '(.+)' kanji_block_indices.append(len(furi_blocks)) furi_blocks.append(remainder) # str not FuriBlock! furi_groups = re.match(furi_regex_pattern, hiragana).groups() # This could be ambiguous! for idx, furi in zip(kanji_block_indices, furi_groups): furi_blocks[idx] = FuriBlock(furi_blocks[idx], furi) # str -> FuriBlock return furi_blocks def parse_japanese_line(line: str): # Split line into plaintext segments to be tokenized, and manual furigana segments last_unmatched_pos = 0 word_tokens = [] for match in re_manual_furi.finditer(line): start = match.start() end = match.end() if start > last_unmatched_pos: word_tokens += parse_jp_text(line[last_unmatched_pos:start]) # Process japanese plaintext word_tokens.append(match.groups()) # Already sorted into (kanji, hiragana) last_unmatched_pos = end if remainder := line[last_unmatched_pos:]: word_tokens += parse_jp_text(remainder) # We now have a series of word tokens that have had all the manual furigana handled output = {} # Process the tokens output['word_pairs'] = [] for token in (it := iter(word_tokens)): word, katakana = token if word[-1] == 'っ': # MeCab splits ?って into ?っ, て so we merge it back try: next_word, next_katakana = next(it) word += next_word katakana += next_katakana except StopIteration: pass hiragana = katakana.translate(katakana_to_hiragana_dict) if word in word_overrides: # Note that most word replacements will instead need to be handled BEFORE tokenization! hiragana = word_overrides[word] output['word_pairs'].append((word, hiragana)) # Process our (kanji, hiragana) word pairs into furigana blocks output['furi_blocks'] = [] # Must be iterated for timing for kanji, hiragana in output['word_pairs']: output['furi_blocks'] += word_to_furi_blocks(kanji, hiragana) # Create word-spaced romaji syllables output['romaji_syllables'] = [] # Will have spaces mixed in so must be iterated for timing for _, hiragana in output['word_pairs']: output['romaji_syllables'] += [to_romaji(s) for syl in kana_to_syllable_list(hiragana) if (s:= syl.strip())] if output['romaji_syllables'][-1] != ' ': output['romaji_syllables'].append(' ') return output