Add csv2bin in master, preparation to unify with tr_EN

Makefile

@@ -1,10 +1,13 @@
 export LC_CTYPE=C
+export PYTHONIOENCODING=utf-8
 
-# User defined
+#User defined
 TARGET := medarot
 ORIGINAL := baserom.gbc
 TARGET_TYPE := gbc
 SOURCE_TYPE := asm
+DIAG_TYPE := csv
+BIN_TYPE := bin
 TABLE_TYPE := tbl
 TMAP_TYPE := tmap
 SYM_TYPE := sym
@@ -22,6 +25,8 @@ GAME := $(BASE)/game
 SRC := $(GAME)/src
 TEXT := $(BASE)/text
 COMMON := $(SRC)/common
+DIALOG := $(BASE)/text/dialog
+
 LISTS_TEXT := $(TEXT)/lists
 LISTS_OUT := $(BUILD)/lists
 PTRLISTS_TEXT := $(TEXT)/ptrlists
@@ -35,6 +40,7 @@ TILESET_TEXT := $(TEXT)/tilesets
 TILESET_OUT := $(BUILD)/tilesets
 
 MODULES := core gfx story
+TEXT := BattleText Snippet1 Snippet2 Snippet3 Snippet4 Snippet5 StoryText1 StoryText2 StoryText3
 TILEMAPS := $(notdir $(basename $(wildcard $(TILEMAP_TEXT)/*.$(TEXT_TYPE))))
 LISTS := $(notdir $(basename $(wildcard $(LISTS_TEXT)/*.$(TEXT_TYPE))))
 PTRLISTS := $(notdir $(basename $(wildcard $(PTRLISTS_TEXT)/*.$(TEXT_TYPE))))
@@ -47,7 +53,7 @@ LD := rgblink
 LD_ARGS :=
 FIX := rgbfix
 FIX_ARGS := -v -k 9C -l 0x33 -m 0x03 -p 0 -r 3 -t "MEDAROT KABUTO"
-# End User Defined
+#End User Defined
 
 #You shouldn't need to touch anything past this line!
 TARGET_OUT := $(TARGET).$(TARGET_TYPE)
@@ -57,7 +63,9 @@ TARGET_SRC := $(GAME)/$(TARGET).$(SOURCE_TYPE)
 
 INT_TYPE := o
 MODULES_OBJ := $(foreach FILE,$(MODULES),$(BUILD)/$(FILE).$(INT_TYPE))
+DIAG_FILES := $(foreach FILE,$(TEXT),$(DIALOG)/$(FILE).$(DIAG_TYPE))
 COMMON_SRC := $(wildcard $(COMMON)/*.$(SOURCE_TYPE)) $(BUILD)/buffer_constants.$(SOURCE_TYPE)
+BIN_FILE := $(BUILD)/$(word 1, $(TEXT)).$(BIN_TYPE)
 TILEMAP_FILES := $(foreach FILE,$(TILEMAPS),$(TILEMAP_OUT)/$(FILE).$(TMAP_TYPE))
 TILESET_FILES := $(foreach FILE,$(TILESETS),$(TILESET_OUT)/$(FILE).$(TSET_TYPE))
 LISTS_FILES := $(foreach FILE,$(LISTS),$(LISTS_OUT)/$(FILE).$(LIST_TYPE))
@@ -73,8 +81,12 @@ $(TARGET_OUT): $(MODULES_OBJ)
 	$(FIX) $(FIX_ARGS) $@
 	cmp -l $(ORIGINAL) $@
 
+#Make is a stupid spec, this is absurd
+$(BIN_FILE): $(DIAG_FILES)
+	$(PYTHON) scripts/csv2bin.py
+
 .SECONDEXPANSION:
-$(BUILD)/%.$(INT_TYPE): $(SRC)/%.$(SOURCE_TYPE) $(COMMON_SRC) $$(%_ADDITIONAL) $$(wildcard $(SRC)/%/*.$(SOURCE_TYPE)) | $(BUILD)
+$(BUILD)/%.$(INT_TYPE): $(SRC)/%.$(SOURCE_TYPE) $(COMMON_SRC) $$(%_ADDITIONAL) $$(wildcard $(SRC)/%/*.$(SOURCE_TYPE)) $(BIN_FILE) | $(BUILD)
 	$(CC) -o $@ $<
 
 $(TILEMAP_OUT)/tilemap_files.$(SOURCE_TYPE): $(SCRIPT)/res/tilemap_files.$(TABLE_TYPE) $(TILEMAP_FILES)

game/src/story/text_tables.asm

@@ -1,29 +1,38 @@
 SECTION "Snippet1", ROMX[$7e00], BANK[$0c]
 Snippet1:
+  INCBIN "build/Snippet1.bin"
 
 SECTION "Snippet2", ROMX[$7e00], BANK[$0d]
 Snippet2:
+  INCBIN "build/Snippet2.bin"
 
 SECTION "Snippet3", ROMX[$7e00], BANK[$0e]
 Snippet3:
+  INCBIN "build/Snippet3.bin"
 
 SECTION "Snippet4", ROMX[$7e00], BANK[$0f]
 Snippet4:
+  INCBIN "build/Snippet4.bin"
 
 SECTION "Snippet5", ROMX[$7800], BANK[$13]
 Snippet5:
+  INCBIN "build/Snippet5.bin"
 
 SECTION "StoryText1", ROMX[$6000], BANK[$16]
 StoryText1:
+  INCBIN "build/StoryText1.bin"
 
 SECTION "StoryText2", ROMX[$4000], BANK[$18]
 StoryText2:
+  INCBIN "build/StoryText2.bin"
 
 SECTION "StoryText3", ROMX[$4000], BANK[$1a]
-StoryText3:
+StoryText3: ; FIXME: StoryText3 has some data between text that causes trouble when rebuilding... (between 0x682b6 and 0x682b8)
+  ;INCBIN "build/StoryText3.bin"
 
 SECTION "BattleText", ROMX[$4000], BANK[$1d]
 BattleText:
+  INCBIN "build/BattleText.bin"
 
 
 SECTION "Dialog Text Tables", ROM0[$1d3b]

scripts/csv2bin.py

@@ -0,0 +1,132 @@
+#!/bin/python
+# -*- coding: utf-8 -*-
+
+from binascii import unhexlify, hexlify
+from collections import OrderedDict
+from struct import *
+import os
+import csv
+import sys
+
+sys.path.append(os.path.join(os.path.dirname(__file__), 'common'))
+from common import utils
+
+ptr_names = {}
+with open(os.path.join(os.path.dirname(__file__), 'res', 'ptrs.tbl'),"r") as f:
+    for line in f:
+        l = line.split('=')
+        l[1] = "{:X}".format(int(l[1], 16))
+        ptr_names[l[0].strip()] = (int(l[1][2:4], 16), int(l[1][0:2], 16))
+
+def table_convert(txt, tbl):
+    result = bytearray()
+    i = 0
+    endcode = 0x00
+    while i < len(txt):
+        try: 
+            if txt[i] == '<':
+                i += 1
+                special_type = txt[i]
+                i += 1
+                special_data = []
+                while txt[i] != '>':
+                    try:
+                        special_data.append(txt[i])
+                    finally:
+                        i += 1
+                if special_type == '*':
+                    endcode = int(special_data[0])
+                    break
+                elif special_type == 'S':
+                    result.append(0x4D)
+                    result.append(int(''.join(special_data), 16))
+                elif special_type == '&':
+                    result.append(0x4B)
+                    s = ''.join(special_data)
+                    if s in ptr_names:
+                        result.append(ptr_names[s][0])
+                        result.append(ptr_names[s][1])
+                    else:
+                        print("Unable to find ptr name for {0}".format(s))
+                        result.append(int(s[2:4], 16))
+                        result.append(int(s[0:2], 16))
+                elif special_type == '`':
+                    result.append(0x50)
+                elif special_type == '4':
+                    result.append(int( special_type + ''.join(special_data), 16))
+            else: # We don't check, as every character should be mapped
+                result.append(tbl[txt[i]])
+        finally:
+            i += 1
+
+    if len(txt):
+        result.append(0x4F)
+        result.append(endcode)
+    return result
+
+if __name__ == '__main__':
+    # TODO: Set this up to take these as an argument
+    arg1 = 'build' #output directory
+    trans_dir = 'text/dialog'
+    output_dir = 'build'
+    char_table = utils.merge_dicts([
+        utils.read_table("scripts/res/medarot.tbl", reverse=True), # FIXME: There are missing tileset mappings, for now just read medarot.tbl
+        #utils.read_table("scripts/res/tileset_MainDialog.tbl", reverse=True),
+        #utils.read_table("scripts/res/tileset_MainSpecial.tbl", reverse=True),
+        #utils.read_table("scripts/res/dakuten.tbl", reverse=True),
+    ])
+    char_table['\\'] = 0x4A
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    #Reads from the text_tables.asm file to determine banks
+    sections = ['Snippet1', 'Snippet2', 'Snippet3', 'Snippet4', 'Snippet5', 'StoryText1', 'StoryText2', 'StoryText3', 'BattleText']
+    bank_size_max = 0x7FFF
+    ptr_size = 3
+
+    bank_map = {}
+    with open('game/src/story/text_tables.asm', 'r') as txt_file:
+        for line in txt_file:
+            if line.startswith('SECTION'):
+                o = line.lstrip('SECTION ').replace(' ', '').replace('\n','').replace('\r\n','').replace('"','').split(',')
+                #Name ROMX[$OFFSET] BANK[$BANK]
+                if o[0] in sections:
+                    bank_map[o[0]] = { "BANK": int(o[2].replace('BANK','').replace('[','').replace(']','').replace('$','0x'), 16), "OFFSET": int(o[1].replace('ROMX','').replace('[','').replace(']','').replace('$','0x'), 16) }
+        assert(len(bank_map) == len(sections))
+
+    for section in sections:
+        fn = os.path.join(trans_dir, "{}.csv".format(section))
+        of = os.path.join(output_dir, "{}.bin".format(section))
+        print("Starting on {}".format(fn))
+        output_text = OrderedDict()
+        with open(fn, 'r', encoding="utf-8") as f:
+            reader = csv.reader(f, delimiter=',', quotechar='"')
+            next(reader, None) #Skip header
+            for line in reader:
+                if line[0][0] == '#': # Comment
+                    continue
+                # Pointer,Original
+                ptr = line[0]
+                txt = line[1].strip('"')
+                if txt.startswith('='):
+                    output_text[ptr] = int(txt.strip('='), 16)
+                elif txt == "<IGNORED>":
+                    output_text[ptr] = int(0)
+                elif not txt:
+                    output_text[ptr] = bytearray()
+                else:
+                    output_text[ptr] = table_convert(txt, char_table)
+        ptrs_map = {}
+        curr_txt = bank_map[section]["OFFSET"] + 2 * len(output_text) # Start the text after the pointer table
+        with open(of, 'wb') as bin_file:
+            # First pass, write out all the pointers
+            for p in output_text:
+                if isinstance(output_text[p], int) and output_text[p] != 0:
+                    bin_file.write(pack('<H', ptrs_map[output_text[p]]))
+                else:
+                    bin_file.write(pack('<H', curr_txt))
+                    ptrs_map[int(p, 16)] = curr_txt
+                    if output_text[p] != 0: # Ignore it if 0 (no-op), but we want to make sure we take it into account for the pointer table
+                        curr_txt = curr_txt + len(output_text[p])
+            # Second pass, write out text
+            #[bin_file.write(output_text[p]) for p in output_text if not isinstance(output_text[p], int) and len(output_text[p])]

scripts/dump_text.py

@@ -1,5 +1,10 @@
 #!/bin/python
 import os
+import sys
+from collections import OrderedDict
+
+sys.path.append(os.path.join(os.path.dirname(__file__), 'common'))
+from common import utils
 
 rom = open("baserom.gbc", "rb")
 log = open("./scripts/res/ptrs.tbl", "a+")
@@ -24,11 +29,7 @@ def readshort():
 def readbyte():
     return ord(rom.read(1))
 
-table = {}
-for line in open("scripts/res/medarot.tbl", encoding="utf-8").readlines():
-    if line.strip():
-        a, b = line.strip('\n').split("=", 1)
-        table[int(a, 16)] = b.replace("\\n", '\n')
+table = utils.read_table("scripts/res/medarot.tbl")
 
 class Special():
     def __init__(self, symbol, default=0, bts=1, end=False, names=None):
@@ -39,7 +40,7 @@ def __init__(self, symbol, default=0, bts=1, end=False, names=None):
         self.names = names
 
 table[0x4b] = Special("&", bts=2, names=name_table)
-table[0x4d] = Special('S', default=2)
+table[0x4d] = Special('S', default=-1)
 table[0x4f] = Special('*', end=True)
 table[0x50] = Special('`', bts=0, end=True)
 
@@ -65,43 +66,56 @@ def dump_text(addr):
                             else:
                                 if token.names is not None:
                                     n = "BUF{:02X}".format(len(name_table))
-                                    print("{1}={0}".format(hex(param), n), file=log)
                                     name_table[param] = n
                                     text += n
                                 else:
                                     text += hex(param)[2:]
                         text += ">"
                 if token.end:
+                    if not text:
+                        text = "<" + token.symbol + hex(param)[2:] + ">"
                     return text
         else:
             text += "<@{0}>".format(hex(b)[2:])
 
 addrs = [0x33e00, 0x37e00, 0x3be00, 0x3fe00, 0x4f800, 0x5a000, 0x60000, 0x68000, 0x74000]
 filenames = ["Snippet1", "Snippet2", "Snippet3", "Snippet4", "Snippet5", "StoryText1", "StoryText2", "StoryText3" , "BattleText"]
-ptr_range = [1024, 1024, 10, 1024, 1024, 1024, 1024, 1024 , 457] #Specify # of pointers if necessary (Snippet 3 has pointers to graphics as well)
+ptr_range = [1024, 1024, (10, 16), 1024, 1024, 1024, 1024, 1024 , (457, 464)] #Specify # of pointers if necessary (Snippet 3 has pointers to graphics as well)
 
 if not os.path.exists("text/dialog"):
     os.makedirs("text/dialog")
 
 for n, file in enumerate(filenames):
-    texts = {}
+    texts = OrderedDict()
     addr = addrs[n]
-
     bank = addr//0x4000
-    rom.seek(addr)
+    print("Working on {}".format(file))
     pts = {}
-    for i in range(ptr_range[n]):
+    for i in range(ptr_range[n] if isinstance(ptr_range[n], int) else ptr_range[n][1]):
+        rom.seek(addr)
         try:
-            ptr = readpointer(bank)
-            pts[rom.tell()-2] = ptr
+            ptr = readpointer(bank) # actual pointer to text
         except NotPointerException:
             break
-    for ptr, p in pts.items():
-        texts[ptr] = dump_text(p)
+        print(hex(ptr))
+        idx = rom.tell()-2 # position in ptr table
+        text = ""
+        if ptr not in pts:
+            pts[ptr] = idx
+            if not isinstance(ptr_range[n], int) and i >= ptr_range[n][0]:
+                text = "<IGNORED>"
+            else:    
+                text = dump_text(ptr)
+        else:
+            text = "={}".format(hex(pts[ptr]))
+        addr += 2
+        texts[idx] = "{}".format(text.replace('\n\n','<4C>').replace('\n','<4E>').replace('"','""'))
+
     with open("text/dialog/" + file + ".csv", "w", encoding="utf-8") as fp:
         fp.write("Pointer,Original\n")
-        for ptr in sorted(texts):
-            fp.write("{0},{1}\n".format(hex(ptr).rstrip('L'), texts[ptr].replace('\n\n','<4C>').replace('\n','<4E>').replace('"','""')))
+        for idx in texts:
+            fp.write("{},{}\n".format(hex(idx), texts[idx]))
+
 
 log.truncate()
 log.close()

scripts/res/medarot.tbl

@@ -54,7 +54,7 @@
 39=ぺ
 3A=ぽ
 
-4A=\\n
+4A=\\
 4C=\n\n
 4E=\n