ref: da7fd359ebe20e37b3b566c056a7228e8e71d6ca
dir: /extras/analyze_texts.py/
#!/usr/bin/python
#author: Bryan Bishop <kanzure@gmail.com>
#date: 2012-01-06
#analyze texts, how many commands are unknown?
import extract_maps
import analyze_incbins #for asm
try:
from pretty_map_headers import map_name_cleaner, txt_bytes, spacing, constant_abbreviation_bytes
except Exception, exc: pass
from operator import itemgetter
import sys
debug = False #set to True to increase logging output
#how many times is each command byte called?
totals = {}
total_text_commands = 0
should_be_total = 0
def get_text_pointer(texts_pointer, text_id):
if type(texts_pointer) == str: texts_pointer = int(texts_pointer, 16)
if type(text_id) == str: text_id = int(text_id)
byte1 = ord(extract_maps.rom[texts_pointer + ((text_id - 1) * 2)])
byte2 = ord(extract_maps.rom[texts_pointer + ((text_id - 1) * 2) + 1])
pointer = (byte1 + (byte2 << 8))
return pointer
def how_many_until(byte, starting):
index = extract_maps.rom.find(byte, starting)
return index-starting
def print_command_debug_info(command_byte, text_id, text_pointer, map_id):
if debug:
print "byte is " + str(command_byte) + " on text #" + str(text_id) + " at " + hex(text_pointer) + " on map " + str(map_id) + " (" + extract_maps.map_headers[map_id]["name"] + ")"
def add_command_byte_to_totals(byte):
global totals
if not byte in totals.keys(): totals[byte] = 1
else: totals[byte] += 1
def process_00_subcommands(start_address, end_address):
"""split this text up into multiple lines
based on subcommands ending each line"""
lines = {}
subsection = extract_maps.rom[start_address:end_address+1]
line_count = 0
current_line = []
for pbyte in subsection:
byte = ord(pbyte)
current_line.append(byte)
if byte == 0x4f or byte == 0x51 or byte == 0x55:
lines[line_count] = current_line
current_line = []
line_count += 1
#don't forget the last line
lines[line_count] = current_line
line_count += 1
return lines
def parse_text_script(text_pointer, text_id, map_id, txfar=False):
global total_text_commands
offset = text_pointer
commands = {}
command_counter = 0
end = False
while not end:
command = {}
command_byte = ord(extract_maps.rom[offset])
print_command_debug_info(command_byte, text_id, text_pointer, map_id)
if command_byte == 0:
#read until $57, $50 or $58
jump57 = how_many_until(chr(0x57), offset)
jump50 = how_many_until(chr(0x50), offset)
jump58 = how_many_until(chr(0x58), offset)
#whichever command comes first
jump = min([jump57, jump50, jump58])
end_address = offset + jump - 1 #we want the address before $57
command = {"type": command_byte,
"start_address": offset,
"end_address": end_address,
"size": jump,
"lines": process_00_subcommands(offset+1, end_address),
}
offset += jump
elif command_byte == 0x17:
#TX_FAR [pointer][bank]
pointer_byte1 = ord(extract_maps.rom[offset+1])
pointer_byte2 = ord(extract_maps.rom[offset+2])
pointer_bank = ord(extract_maps.rom[offset+3])
pointer = (pointer_byte1 + (pointer_byte2 << 8))
pointer = extract_maps.calculate_pointer(pointer, pointer_bank)
command = {"type": command_byte,
"start_address": offset,
"end_address": offset + 3, #last byte belonging to this command
"pointer": pointer, #parameter
}
offset += 3 + 1
elif command_byte == 0x50 or command_byte == 0x57 or command_byte == 0x58: #end text
command = {"type": command_byte,
"start_address": offset,
"end_address": offset,
}
#this byte simply indicates to end the script
end = True
#this byte simply indicates to end the script
if command_byte == 0x50 and ord(extract_maps.rom[offset+1]) == 0x50: #$50$50 means end completely
end = True
commands[command_counter+1] = command
#also save the next byte, before we quit
commands[command_counter+1]["start_address"] += 1
commands[command_counter+1]["end_address"] += 1
add_command_byte_to_totals(command_byte)
elif command_byte == 0x50: #only end if we started with $0
if len(commands.keys()) > 0:
if commands[0]["type"] == 0x0: end = True
elif command_byte == 0x57 or command_byte == 0x58: #end completely
end = True
offset += 1 #go past this 0x50
elif command_byte == 0x1:
#01 = text from RAM. [01][2-byte pointer]
size = 3 #total size, including the command byte
pointer_byte1 = ord(extract_maps.rom[offset+1])
pointer_byte2 = ord(extract_maps.rom[offset+2])
command = {"type": command_byte,
"start_address": offset+1,
"end_address": offset+2, #last byte belonging to this command
"pointer": [pointer_byte1, pointer_byte2], #RAM pointer
}
#view near these bytes
#subsection = extract_maps.rom[offset:offset+size+1] #peak ahead
#for x in subsection:
# print hex(ord(x))
#print "--"
offset += 2 + 1 #go to the next byte
#use this to look at the surrounding bytes
if debug:
print "next command is: " + hex(ord(extract_maps.rom[offset])) + " ... we are at command number: " + str(command_counter) + " near " + hex(offset) + " on map_id=" + str(map_id) + " for text_id=" + str(text_id) + " and txfar(recursion)=" + str(txfar)
elif command_byte == 0x7:
#07 = shift texts 1 row above (2nd line becomes 1st line); address for next text = 2nd line. [07]
size = 1
command = {"type": command_byte,
"start_address": offset,
"end_address": offset,
}
offset += 1
elif command_byte == 0x3:
#03 = set new address in RAM for text. [03][2-byte RAM address]
size = 3
command = {"type": command_byte, "start_address": offset, "end_address": offset+2}
offset += size
elif command_byte == 0x4: #draw box
#04 = draw box. [04][2-Byte pointer][height Y][width X]
size = 5 #including the command
command = {
"type": command_byte,
"start_address": offset,
"end_address": offset + size,
"pointer_bytes": [ord(extract_maps.rom[offset+1]), ord(extract_maps.rom[offset+2])],
"y": ord(extract_maps.rom[offset+3]),
"x": ord(extract_maps.rom[offset+4]),
}
offset += size + 1
elif command_byte == 0x5:
#05 = write text starting at 2nd line of text-box. [05][text][ending command]
#read until $57, $50 or $58
jump57 = how_many_until(chr(0x57), offset)
jump50 = how_many_until(chr(0x50), offset)
jump58 = how_many_until(chr(0x58), offset)
#whichever command comes first
jump = min([jump57, jump50, jump58])
end_address = offset + jump - 1 #we want the address before $57
command = {"type": command_byte,
"start_address": offset,
"end_address": end_address,
"size": jump,
"lines": process_00_subcommands(offset+1, end_address),
}
offset = end_address + 1
elif command_byte == 0x6:
#06 = wait for keypress A or B (put blinking arrow in textbox). [06]
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0x7:
#07 = shift texts 1 row above (2nd line becomes 1st line); address for next text = 2nd line. [07]
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0x8:
#08 = asm until whenever
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
end = True
elif command_byte == 0x9:
#09 = write hex-to-dec number from RAM to textbox [09][2-byte RAM address][byte bbbbcccc]
# bbbb = how many bytes to read (read number is big-endian)
# cccc = how many digits display (decimal)
#(note: max of decimal digits is 7,i.e. max number correctly displayable is 9999999)
ram_address_byte1 = ord(extract_maps.rom[offset+1])
ram_address_byte2 = ord(extract_maps.rom[offset+2])
read_byte = ord(extract_maps.rom[offset+3])
command = {
"type": command_byte,
"address": [ram_address_byte1, ram_address_byte2],
"read_byte": read_byte, #split this up when we make a macro for this
}
offset += 4
elif command_byte == 0xB:
#0B = sound_86 (Hiro obtains ITEM)[same as 0F]
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0xE:
#0E = sound_91 (learnt something)
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0xF:
#0F = sound_86 (given rare candy)[same as 0B]
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0x10:
#10 = sound_89 (PKMN successfully caught)
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0x11:
#11 = sound_94 (Hiro gives OAK the PARCEL)
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0x12:
#12 = sound_9A (successfully caught)
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0x13:
#13 = sound_98 (song heard when "new data will be added for..")
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0x14:
#14 = MonCry (Nidorina)
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0x15:
#14 = MonCry (Pidgeot)
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0x16:
#14 = MonCry (Dewgong)
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0x19:
#19 = play a 'bump' noise
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0x1F:
#1F = play some pokemon's roar, don't know which..
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0x20:
#20 = oddish roar?
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0x3F:
#3F = some other roar
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0x9D:
#9D = a roar or some other sound, four times in quick succession
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0x76:
#76 = another roar
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0xCA:
#CA = stop music, start this other song that i can't name
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0xF6:
#F6 = play a 'blurp blurp' noise.. like something is increasing
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0xFA:
#FA = change music to champion song?
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0xFE:
#FE = another roar, kinda glitchy?
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
elif command_byte == 0xFF:
#FF = change music to a specific song that i don't know the name of
command = {"type": command_byte, "start_address": offset, "end_address": offset}
offset += 1
else:
#if len(commands) > 0:
# print "Unknown text command " + hex(command_byte) + " at " + hex(offset) + ", script began with " + hex(commands[0]["type"])
if debug:
print "Unknown text command at " + hex(offset) + " - command: " + hex(ord(extract_maps.rom[offset])) + " on map_id=" + str(map_id) + " text_id=" + str(text_id)
#end at the first unknown command
end = True
add_command_byte_to_totals(command_byte)
commands[command_counter] = command
command_counter += 1
total_text_commands += len(commands)
return commands
def analyze_texts():
global should_be_total
texts = {}
for map_id in extract_maps.map_headers:
if map_id in extract_maps.bad_maps: continue #skip
map2 = extract_maps.map_headers[map_id]
map2["texts"] = {}
referenced_texts = map2["referenced_texts"]
should_be_total += len(referenced_texts)
texts_pointer = int(map2["texts_pointer"], 16)
#print "Checking texts on... map_id=" + str(map_id) + " and len(referenced_texts)=" + str(len(referenced_texts))
for text_id in referenced_texts:
text_pointer = get_text_pointer(texts_pointer, text_id)
if 0x4000 <= text_pointer <= 0x7fff: #only care about bank when it's between 4000-7fff
text_pointer = extract_maps.calculate_pointer(text_pointer, int(map2["bank"], 16))
#print "Working on map id=" + str(map2["id"]) + " and text id=" + str(text_id)
#print "for map_id=" + str(map_id) + " texts_pointer=" + hex(texts_pointer) + " text_id=" + str(text_id) + " the pointer=" + hex(text_pointer)
commands = parse_text_script(text_pointer, text_id, map_id)
#process TX_FARs
for command_id in commands:
#skip commands starting with an unknown command byte
if len(commands[command_id]) == 0: continue
if commands[command_id]["type"] == 0x17:
TX_FAR = parse_text_script(commands[command_id]["pointer"], text_id, map_id, txfar=True)
if debug:
if len(TX_FAR.keys()) > 0:
#print "TX_FAR object: " + str(TX_FAR)
print "processing a TX_FAR at " + hex(commands[command_id]["pointer"]) + "... first byte is: " + str(ord(extract_maps.rom[commands[command_id]["pointer"]])) + " .. offset: " + hex(commands[command_id]["pointer"])
##sys.exit(0)
commands[command_id]["TX_FAR"] = TX_FAR
#map2["texts"][text_id][command_id]["TX_FAR"] = parse_text_script(command["pointer"], text_id, map_id)
map2["texts"][text_id] = commands
texts[map_id] = map2["texts"]
extract_maps.map_headers[map_id]["texts"] = map2["texts"]
return texts
def find_missing_08s(all_texts):
"""determines which $08s have yet to be inserted
based on their start addresses"""
missing_08s = 0
for map_id in all_texts.keys():
for text_id in all_texts[map_id].keys():
for line_id in all_texts[map_id][text_id].keys():
if not line_id == 0:
current_line = all_texts[map_id][text_id][line_id]
if "type" in current_line.keys():
if current_line["type"] == 0x8:
missing_08s += 1
print "missing $08 on map_id=" + str(map_id) + " text_id=" + str(text_id) + " line_id=" + str(line_id) + " at " + hex(current_line["start_address"])
return missing_08s
def text_pretty_printer_at(start_address, label="SomeLabel"):
commands = parse_text_script(start_address, None, None)
needs_to_begin_with_0 = True #how should this be determined?
#wanted_command = None
#if needs_to_begin_with_0:
# wanted_command = None
# for command_id in commands:
# command = commands[command_id]
# if command["type"] == 0:
# wanted_command = command_id
#
# if wanted_command == None:
# raise "error: address did not start with a $0 text"
#start with zero please
byte_count = 0
output = ""
had_text_end_byte = False
had_text_end_byte_57_58 = False
had_db_last = False
first_line = True
for this_command in commands.keys():
if not "lines" in commands[this_command].keys():
command = commands[this_command]
if not "type" in command.keys():
print "ERROR in command: " + str(command)
continue #dunno what to do here?
if command["type"] == 0x1: #TX_RAM
if first_line:
output = "\n"
output += label + ": ; " + hex(start_address)
first_line = False
p1 = command["pointer"][0]
p2 = command["pointer"][1]
#remember to account for big endian -> little endian
output += "\n" + spacing + "TX_RAM $%.2x%.2x" %(p2, p1)
byte_count += 3
had_db_last = False
elif command["type"] == 0x17: #TX_FAR
if first_line:
output = "\n"
output += label + ": ; " + hex(start_address)
first_line = False
#p1 = command["pointer"][0]
#p2 = command["pointer"][1]
output += "\n" + spacing + "TX_FAR _" + label + " ; " + hex(command["pointer"])
byte_count += 4 #$17, bank, address word
had_db_last = False
elif command["type"] == 0x9: #TX_RAM_HEX2DEC
if first_line:
output = "\n" + label + ": ; " + hex(start_address)
first_line = False
#address, read_byte
output += "\n" + spacing + "TX_NUM $%.2x%.2x, $%.2x" % (command["address"][1], command["address"][0], command["read_byte"])
had_db_last = False
byte_count += 4
elif command["type"] == 0x50 and not had_text_end_byte:
#had_text_end_byte helps us avoid repeating $50s
if first_line:
output = "\n" + label + ": ; " + hex(start_address)
first_line = False
if had_db_last:
output += ", $50"
else:
output += "\n" + spacing + "db $50"
byte_count += 1
had_db_last = True
elif command["type"] in [0x57, 0x58] and not had_text_end_byte_57_58:
if first_line: #shouldn't happen, really
output = "\n" + label + ": ; " + hex(start_address)
first_line = False
if had_db_last:
output += ", $%.2x" % (command["type"])
else:
output += "\n" + spacing + "db $%.2x" % (command["type"])
byte_count += 1
had_db_last = True
elif command["type"] in [0x57, 0x58] and had_text_end_byte_57_58:
pass #this is ok
elif command["type"] == 0x50 and had_text_end_byte:
pass #this is also ok
elif command["type"] == 0x0b:
if first_line:
output = "\n" + label + ": ; " + hex(start_address)
first_line = False
if had_db_last:
output += ", $0b"
else:
output += "\n" + spacing + "db $0B"
byte_count += 1
had_db_last = True
elif command["type"] == 0x11:
if first_line:
output = "\n" + label + ": ; " + hex(start_address)
first_line = False
if had_db_last:
output += ", $11"
else:
output += "\n" + spacing + "db $11"
byte_count += 1
had_db_last = True
elif command["type"] == 0x6: #wait for keypress
if first_line:
output = "\n" + label + ": ; " + hex(start_address)
first_line = False
if had_db_last:
output += ", $6"
else:
output += "\n" + spacing + "db $6"
byte_count += 1
had_db_last = True
else:
print "ERROR in command: " + hex(command["type"])
had_db_last = False
#everything else is for $0s, really
continue
lines = commands[this_command]["lines"]
#reset this in case we have non-$0s later
had_db_last = False
#add the ending byte to the last line- always seems $57
#this should already be in there, but it's not because of a bug in the text parser
lines[len(lines.keys())-1].append(commands[len(commands.keys())-1]["type"])
if first_line:
output = "\n"
output += label + ": ; " + hex(start_address) + "\n"
first_line = False
else:
output += "\n"
first = True #first byte
for line_id in lines:
line = lines[line_id]
output += spacing + "db "
if first and needs_to_begin_with_0:
output += "$0, "
first = False
byte_count += 1
quotes_open = False
first_byte = True
was_byte = False
for byte in line:
if byte == 0x50:
had_text_end_byte = True #don't repeat it
if byte in [0x58, 0x57]:
had_text_end_byte_57_58 = True
if byte in txt_bytes:
if not quotes_open and not first_byte: #start text
output += ", \""
quotes_open = True
first_byte = False
if not quotes_open and first_byte: #start text
output += "\""
quotes_open = True
output += txt_bytes[byte]
elif byte in constant_abbreviation_bytes:
if quotes_open:
output += "\""
quotes_open = False
if not first_byte:
output += ", "
output += constant_abbreviation_bytes[byte]
else:
if quotes_open:
output += "\""
quotes_open = False
#if you want the ending byte on the last line
#if not (byte == 0x57 or byte == 0x50 or byte == 0x58):
if not first_byte:
output += ", "
output += "$" + hex(byte)[2:]
was_byte = True
#add a comma unless it's the end of the line
#if byte_count+1 != len(line):
# output += ", "
first_byte = False
byte_count += 1
#close final quotes
if quotes_open:
output += "\""
quotes_open = False
output += "\n"
include_newline = "\n"
if len(output)!=0 and output[-1] == "\n":
include_newline = ""
output += include_newline + "; " + hex(start_address) + " + " + str(byte_count) + " bytes = " + hex(start_address + byte_count)
print output
return (output, byte_count)
def is_label_in_asm(label):
for line in analyze_incbins.asm:
if label in line:
if line[0:len(label)] == label:
return True
return False
def find_undone_texts():
usable_table = {}
if analyze_incbins.asm == None:
analyze_incbins.load_asm()
for map_id in extract_maps.map_headers:
#skip bad maps
if map_id in extract_maps.bad_maps: continue
map2 = extract_maps.map_headers[map_id]
name = map_name_cleaner(map2["name"], None)[:-2] + "Text"
for text_id in map2["referenced_texts"]:
label = name + str(text_id)
if len(extract_maps.map_headers[map_id]["texts"][text_id].keys()) == 0: continue
if len(extract_maps.map_headers[map_id]["texts"][text_id][0].keys()) == 0: continue
try:
address = extract_maps.map_headers[map_id]["texts"][text_id][0]["start_address"]
except:
address = extract_maps.map_headers[map_id]["texts"][text_id][1]["start_address"]
if not is_label_in_asm(label):
print label + " map_id=" + str(map_id) + " text_id=" + str(text_id) + " at " + hex(address) + " byte is: " + hex(ord(extract_maps.rom[address]))
if not address in usable_table.keys():
usable_table[address] = 1
else:
usable_table[address] += 1
print "\n\n which ones are priority?"
sorted_results = sorted(usable_table.iteritems(), key=itemgetter(1), reverse=True)
for result in sorted_results:
print str(result[1]) + " times: " + hex(result[0])
def scan_rom_for_tx_fars(printer=True):
"""find TX_FARs
search only addresses that are INCBINed
keep only TX_FARs that are valid
returns a list of [TX_FAR target address, TX_FAR address]"""
rom = extract_maps.rom
analyze_incbins.load_asm()
analyze_incbins.isolate_incbins()
analyze_incbins.process_incbins()
possible_tx_fars = []
possible_tx_far_targets = []
for incbin_line_number in analyze_incbins.processed_incbins.keys():
incbin = analyze_incbins.processed_incbins[incbin_line_number]
start_address = incbin["start"]
end_address = incbin["end"]
if incbin["interval"] == 0: continue #skip this one
subrom = rom[start_address:end_address]
for address in range(start_address, end_address):
current_byte = ord(rom[address])
if current_byte == 0x17:
if ord(rom[address+4]) == 0x50:
byte1 = ord(rom[address+1])
byte2 = ord(rom[address+2])
address2 = byte1 + (byte2 << 8)
if address2 > 0x3fff:
address2 = extract_maps.calculate_pointer(address2, ord(rom[address+3]))
#print "possible TX_FAR at " + hex(address) + " to " + hex(address2)
possible_tx_fars.append(address)
possible_tx_far_targets.append([address2, address])
if printer:
pre_handled = []
#address_bundle is [TX_FAR target address, TX_FAR address]
for address_bundle in possible_tx_far_targets:
if address_bundle[0] in [0xa82f8, 0xa8315]:
continue #bad
if address_bundle[0] in pre_handled:
continue #already did this
print "-------"
print "TX_FAR is at: " + hex(address_bundle[1])
#let's try printing out the TX_FAR?
text_pretty_printer_at(address_bundle[1], "blah")
text_pretty_printer_at(address_bundle[0], "_blah")
print "-------"
pre_handled.append(address_bundle[0])
return possible_tx_far_targets
if __name__ == "__main__":
extract_maps.load_rom()
extract_maps.load_map_pointers()
extract_maps.read_all_map_headers()
#text_output = analyze_texts()
#print text_output
#these aren't really "missing", just a certain type that was
#known to be missed on a first pass.
#missing_08s = find_missing_08s(text_output)
#print "\n\n---- stats ----\n\n"
#popular_text_commands = sorted(totals.iteritems(), key=itemgetter(1), reverse=True)
#convert the first values (command byte) to hex
#for popular_item in popular_text_commands:
# print hex(popular_item[0]) + " was used " + str(popular_item[1]) + " times."
#print "popular text commands: " + str(popular_text_commands)
#print "total text commands: " + str(total_text_commands)
#print "total text scripts: " + str(should_be_total)
#print "missing 08s: " + str(missing_08s)
#print "\n\n"
#text_pretty_printer_at(0x800b1)
#find_undone_texts()
scan_rom_for_tx_fars()