From 01e239759443844f5f824b8d6e5d8296c3d3a473 Mon Sep 17 00:00:00 2001
From: Luke Hubmayer-Werner <mongoose41@gmail.com>
Date: Fri, 20 Dec 2024 02:08:51 +1030
Subject: [PATCH] Working timed subtitle generation!!!

---
 lyrics/static/lyrics/input.js              | 68 ++++++++++++++++------
 lyrics/static/lyrics/subtitle_generator.js |  2 +-
 2 files changed, 52 insertions(+), 18 deletions(-)

diff --git a/lyrics/static/lyrics/input.js b/lyrics/static/lyrics/input.js
index b2f8e1c..97c1160 100644
--- a/lyrics/static/lyrics/input.js
+++ b/lyrics/static/lyrics/input.js
@@ -90,23 +90,28 @@ function on_receive_tokenized_lyrics(data) {
 function update_arrangement_output() {
 	console.log('Updating arrangement output');
 	arrangement_line_numbers = []
-	arrangement_input_element.value
-		.split(',')
-		.map(s=>lyric_section_ranges[s.trim()])
-		.filter(x=>x)
-		.forEach(([i0, i1])=>{
+	arrangement_input_element.value.split(',').map(s=>s.trim()).forEach(s => {
+		const range = lyric_section_ranges[s];
+		if (range) {
+			const [i0, i1] = range;
 			for (let i=i0; i<i1; i++) arrangement_line_numbers.push(i);
+		} else {
+			const i = parseInt(s);
+			if (i>0) {
+				arrangement_line_numbers.push(-i);  // Hacky approach - intersperse gap beats as negative line numbers
+			}
+		}
 	});
 	
 	const tl_lines = lyrics_tl_input_element.value.split('\n');
 	var html = '';
-	for (const i of arrangement_line_numbers) {
+	arrangement_line_numbers.filter(x=>x>=0).forEach(i => {
 		const tokenized_line = tokenized_lyric_lines[i];
 		if ((typeof tokenized_line) != "string" && i < tl_lines.length) {
 			html += `<span class="lyrics-tl">${tl_lines[i]}<br></span>`;
 		}
 		html += format_parsed_line(tokenized_line);
-	}
+	});
 	arrangement_output_element.innerHTML = html;
 }
 
@@ -261,22 +266,51 @@ import generate_subtitles_from_data from './subtitle_generator.js'
 function generate_subtitles() {
 	console.log('Attempting to generate subtitles');
 
-	let lines = [];
-	let t0 = 72.0;  // TODO: add line timing data
+	// Grab manual line timestamps
+	const manual_line_timecodes = line_timer_element.value.split('\n').filter(s=>s.trim()).map(s=>parseFloat(s.split('\t')[0]));
+	const placeholder_t0 = 72.0;
 	const seconds_per_line = 10.0;
+	const placeholder_line_timecodes = Array(arrangement_line_numbers.length+1).fill().map((_,i) => Math.floor(i*seconds_per_line + placeholder_t0));
+	// Grab syllable beat counts
+	const line_syllable_timings = lyrics_timing_input_element.value.split('\n').map(l => l.split(' ').filter(c=>c).map(c=>parseInt(c)));
+	const line_syllable_total_beats = line_syllable_timings.map(l=>l.reduce((acc,x)=>acc+x));
+
+	let lines = [];
+	// let t0 = manual_line_timecodes?.[0] ?? placeholder_line_timecodes[0];
 
 	const tl_lines = lyrics_tl_input_element.value.split('\n');
-	for (const i of arrangement_line_numbers) {
-		const tokenized_line = tokenized_lyric_lines[i];
-		if ((typeof tokenized_line) == "string") continue;
-		
+	let out_idx = 0;
+	arrangement_line_numbers.forEach(src_line_idx => {
+		const t0 = manual_line_timecodes?.[out_idx] ?? placeholder_line_timecodes[out_idx];
+		const t1 = manual_line_timecodes?.[out_idx+1] ?? placeholder_line_timecodes[out_idx+1];
+		const duration = t1-t0;
+		const total_beats = (src_line_idx<0) ? (-src_line_idx) : (line_syllable_total_beats?.[src_line_idx] ?? 1);
+		const bpm = 60.0*total_beats/duration;
+		const spb = 60.0/bpm;
+		console.log(src_line_idx, out_idx, t0, t1, total_beats, bpm, spb);
+
+		const tokenized_line = tokenized_lyric_lines[src_line_idx];
+		if ((typeof tokenized_line) == "string") return;  // Label line
+		if (src_line_idx < 0) {  // Rest beats - still increment the output line
+			out_idx++;
+			return;
+		}
+
+		let arr_syllable_cs = [];
+		let acc = 0;
+		line_syllable_timings?.[src_line_idx]?.forEach(b => {
+			acc += b;
+			arr_syllable_cs.push(Math.floor(100*acc*spb));
+		});
+
 		lines.push(Object.assign({
 			t0: t0,
-			t1: t0+seconds_per_line,
-			translated_line: tl_lines?.[i] ?? '',
+			t1: t1,
+			arr_syllables_cs: arr_syllable_cs,
+			translated_line: tl_lines?.[src_line_idx] ?? '',
 		}, tokenized_line));
-		t0 += seconds_per_line;
-	}
+		out_idx++;
+	});
 
 	const subtitles = generate_subtitles_from_data({
 		song_title: document.getElementById('song_title').value,
diff --git a/lyrics/static/lyrics/subtitle_generator.js b/lyrics/static/lyrics/subtitle_generator.js
index aa23d5d..d6736da 100644
--- a/lyrics/static/lyrics/subtitle_generator.js
+++ b/lyrics/static/lyrics/subtitle_generator.js
@@ -19,7 +19,7 @@ function kana_to_syllable_list(kana) {
 			a.push(k);
 		}
 	}
-	console.log(a);
+	// console.log(a);
 	return a;
 }