Tokenize a given line as provided by IDA

tokenize_line.py

import ida_lines
import ida_kernwin
from enum import Enum


def tokenize_line(line):
    COLOR_ON = "\x01"
    COLOR_OFF = "\x02"
    blacklist = ["SCOLOR_ON", "SCOLOR_OFF", "SCOLOR_ESC", "SCOLOR_INV", "SCOLOR_UTF8", "SCOLOR_FG_MAX"]
    tag_mapping = Enum("TagMapping", {x: getattr(ida_lines, x) for x in dir(ida_lines) if (x.startswith("SCOLOR_") and x not in blacklist)})

    data = []

    len_s = len(line)
    i = 0
    opened = False
    prev = None

    while i < len_s:
        n = ida_lines.tag_skipcode(line[i:])
        if n:
            for j in range(n):
                code = line[i+j]
                if code == COLOR_ON:
                    opened = True
                elif code == COLOR_OFF and prev != COLOR_ON and opened:  # We are on a true COLOR_OFF (otherwise comment)
                    opened = False
                else:
                    data.append(tag_mapping(code))
                prev = code
            i += n
        else:
            if isinstance(data[-1], str):
                data[-1] += line[i]
            else:
                data.append(line[i])
            i += 1

    return data


p0 = ida_kernwin.twinpos_t()
p1 = ida_kernwin.twinpos_t()
view = ida_kernwin.get_current_viewer()
ida_kernwin.read_selection(view, p0, p1)