प्रोग्राम की सुविधाएँ · LZ78 वाक्यांश टोकन

def lz78_tokens(data):
    if type(data) is not bytes or len(data) > 4096:
        raise ValueError("lz78.input")
    dictionary, phrase, output = {b"": 0}, b"", []
    for byte in data:
        extended = phrase + bytes([byte])
        if extended in dictionary:
            phrase = extended
        else:
            output.append((dictionary[phrase], byte))
            dictionary[extended] = len(dictionary)
            phrase = b""
    if phrase:
        output.append((dictionary[phrase], None))
    return output