From a7a6070aae052d0e1b1d20756a6c4f544e8e487b Mon Sep 17 00:00:00 2001
From: Luke Hubmayer-Werner <mongoose41@gmail.com>
Date: Thu, 27 Jun 2024 17:45:39 +0930
Subject: [PATCH] Implement proper unflattening and deep update for building
 AoAoAoA... tables

---
 includes/helpers.py   | 29 +++++++++++++++++++++++++++++
 includes/rom_serde.py |  8 ++------
 2 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/includes/helpers.py b/includes/helpers.py
index 098a683..59128d2 100644
--- a/includes/helpers.py
+++ b/includes/helpers.py
@@ -161,3 +161,32 @@ def load_tsv(filename) -> list:
 		entries.append(unflatten_keys(entry))
 
 	return unflatten_table(headers, entries)
+
+def deep_update(old_container: list|dict, new_container: list|dict, ignore_holes: bool = True):
+	# Recurse through old_container, overwriting all values changed by new_container.
+	# Values of None in new_container will be treated as holes.
+	if isinstance(old_container, list):
+		for i, new in enumerate(new_container):
+			if ignore_holes and new is None:  # Allow holes in the table for values we don't care about overwriting
+				continue
+			if isinstance(new, dict):
+				id = new.get('ID')
+				if not isinstance(id, int):
+					id = i
+				deep_update(old_container[id], new)
+			elif isinstance(new, list):
+				deep_update(old_container[i], new)
+			else:
+				old_container[i] = value
+	elif isinstance(old_container, dict):
+		for key, value in new_container.items():
+			if key == 'ID':
+				continue
+			if ignore_holes and value is None:  # Allow holes in the table for values we don't care about overwriting
+				continue
+			if isinstance(value, dict) or isinstance(value, list):
+				deep_update(old_container[key], value)
+			else:
+				old_container[key] = value
+	else:
+		raise ValueError
diff --git a/includes/rom_serde.py b/includes/rom_serde.py
index b057e24..e9b5d46 100644
--- a/includes/rom_serde.py
+++ b/includes/rom_serde.py
@@ -1,5 +1,5 @@
 from ChocolateBirdData.reference_implementation import get_base_structarraytypes, parse_struct_definitions_from_tsv_filename, get_structarraytype, LeftoverBits, ReadBuffer, WriteBuffer
-from includes.helpers import load_tsv
+from includes.helpers import load_tsv, deep_update
 
 class ROMHandler:
 	offset_key: str
@@ -24,11 +24,7 @@ class ROMHandler:
 	def build_partial(self, table: str, new_data: list[dict], in_buffer, out_buffer):
 		# Safely merge partial data over the existing data, then serialize it.
 		existing_data = self.extract(table, in_buffer)
-		for i, new in enumerate(new_data):
-			id = new.get('ID', i)
-			for k, v in new.items():
-				if k != 'ID' and v is not None:  # Allow holes in the table for values we don't care about overwriting
-					existing_data[id][k] = v
+		deep_update(existing_data, new_data)
 		self.build(table, existing_data, out_buffer)