Structured JSON-style python form converter

structured_form.py

""" 
The goal of this code is to allow submitting data using the 
naming convention specified in the un-implemented WC3 JSON 
form proposal.

https://www.w3.org/TR/html-json-forms/

This would allow forms to better submit more structured data.
This will differ from the W3C proposal in a few important ways:

 * Takes place server-side instead of client-side
 * Does not support files
 * Supports negative indexing
 * Supports empty array append keys mid-path, e.g. this[is][][perfectly valid]
 * Supports comments/tags that will not be included in the output path: real[path]#this-is-not-included
    (This is useful to differientiate radio groups that might need to have the same name)

This is especially useful as a layer sitting between a web framework such as Flask
and a deserializtion library such as marshmallow.
"""
from ast import literal_eval
from collections import namedtuple
from enum import Enum

class KeyType(Enum):
    object = 'object'
    array = 'array'

PathSegment = namedtuple('PathSegment', ('key', 'type'))

undefined = type('undefined', (), {})()


def structured_form(kvpairs, parse_literals=False, datatype=dict):
    """
    Takes an interable of key-value pairs where the key is
    the data path and the value is the value to set.
    Returns the data as a structured dirctionary.

    If parse_literals is true, the function will attempt
    to parse values as Python literal values, defaulting
    to strings if the values are not valid literals.

    Pass the optional datatype parameter to use another mapping
    datatype, such as and ordered dict
    """
    data = datatype()
    for key, value in kvpairs:
        if parse_literals:
            try:
                value = literal_eval(value)
            except:
                pass # Assume it's a string
        if value == '':
            value = None
        steps = parse_path(key)
        data = set_value(data, steps, value, map_datatype=datatype)
    return undefined_to_none(data)


def parse_path(path: str):
    """
    Takes a string-based JSON path, and parses it into a list of
    PathSegment steps.

    Equvilent to the "steps to parse a JSON encoding path" 
    algorithm detailed in the W3C Proposal.
    """
    original = path
    steps = []
    if '[' not in path:
        steps.append(PathSegment(path, KeyType.object))
        return steps
    first_key, path = path.split('[', 1)
    steps.append(PathSegment(first_key, KeyType.object))
    while path:
        if path[0] == '#':
            # This is outside the original scope, but is essensially a comment.
            # The use case for this is to allow a way to diffrientiate radio
            # button groups when -1 indexing is used
            break
        if path[0] == '[':
            # Technically this doesn't perfectly match the original algorithm,
            # it would parse in invalid path such as this[is]invaid] the same as
            # this[is][valid].
            path = path[1:]
        if path[0] == ']':
            # This is also outside the original spec. It would allow empty indexes
            # mid-path
            path = path[1:]
            steps.append(PathSegment(None, KeyType.array))
        elif ']' in path:
            key, path = path.split(']', 1)
            try:
                key = int(key)
                steps.append(PathSegment(key, KeyType.array))
            except ValueError:
                steps.append(PathSegment(key, KeyType.object))
        else:
            # Malformed path, failure
            return [PathSegment(original, KeyType.object)]
    return steps




    

def set_value(current, steps, value, map_datatype = dict):
    """
    Takes a list of steps, the value to set, and the current value. 
    Returns a new value to replace current, with value set approriately 
    on it.

    Equivilent to the "steps to set a JSON encoding value" algorithm
    in the W3C proposal.
    """
    if not steps:
        # When we reach the end of recursion
        if current is undefined:
            return value
        elif isinstance(current, list):
            return current.append(value)
        elif isinstance(current, dict):
            return set_value(current, [PathSegment('', KeyType.object)], value, map_datatype=map_datatype)
        else:
            return [current, value]
    step = steps[0]

    if current is undefined:
        current = [] if step.type is KeyType.array else map_datatype()

    if isinstance(current, list):
        if step.type is KeyType.array:
            if step.key is None:
                current.append(undefined)
            elif step.key >= len(current):
                current.extend([undefined] * (step.key + 1 - len(current)))
            key = -1 if step.key is None else step.key
            current[key] = set_value(current[key], steps[1:], value)
        elif step.type is KeyType.object:
            # Current is a list but we're setting an object key
            # Convert current to a dict
            current = map_datatype([
                        (index, value)
                        for index, value in enumerate(current) 
                        if value is not undefined
                    ])
            current[step.key] = set_value(current.get(step.key, undefined), steps[1:], value, map_datatype=map_datatype)
        else:
            raise ValueError(f'Invalid key step type: {step.type}')
    elif isinstance(current, dict):
        current[step.key] = set_value(current.get(step.key, undefined), steps[1:], value, map_datatype=map_datatype)
    else:
        # There is a value currently here we need to replace with a dict
        current = {'': current}
        current[step.key] = set_value(current.get(step.key, undefined), steps[1:], value, map_datatype=map_datatype)

    return current


def undefined_to_none(data):
    """
    Converts undefined placeholder values to None.
    """
    if isinstance(data, dict):
        iterable = data.items()
    elif isinstance(data, list):
        iterable = enumerate(data)
    else:
        return data
    for key, value in iterable:
        if value is undefined:
            data[key] = None
        elif isinstance(value, (dict, list)):
            undefined_to_none(value)
    return data


def test():
    """
    Test the the library works as expected.
    """
    kvpairs = [
        ('justme', 'test'),
        ('array_append[]', 'one'),
        ('array_append[]', 'two'),
        ('array_append[]', 'three'),
        ('array_append[]', 'four'),
        ('obj[k1]', '1'),
        ('obj[k2]', '2'),
        ('obj[k3]', '3'),
        ('obj[k4]', '4'),
        ('obj[level2][k1]', '21'),
        ('obj[level2][k2]', '22'),
        ('twice', 'a'),
        ('twice', 'b'),
        ('overwrite_obj', 'original'),
        ('overwrite_obj[kiddie]', 'kitty'),
        ('overwrite_backward[kiddie]', 'kitty'),
        ('overwrite_backward', 'real'),
        ('wow[such][deep][3][much][power][!]', 'Amaze'),
        ('negative[][name]', 'John'),
        ('negative[-1][gender]', 'Male'),
        ('negative[][name]', 'Jane'),
        ('negative[-1][gender]', 'Female'),
        ('out_of_order[3]', 'ATM'),
        ('out_of_order[1]', 'Toilet'),
        ('out_of_order[0]', 'Arcade Game'),
        ('out_of_order[4]', 'Vending Machine'),
        ('out_of_order[2]', 'My Brain'),
    ]
    data = structured_form(kvpairs)
    from pprint import pprint
    pprint(data)
    expected = {
        'justme': 'test',
        'array_append': ['one', 'two', 'three', 'four'],
        'obj': {
            'k1': '1',
            'k2': '2',
            'k3': '3',
            'k4': '4',
            'level2':{
                'k1': '21',
                'k2': '22',
            }
        },
        'twice': ['a', 'b'],
        'overwrite_obj': {
            '': 'original',
            'kiddie': 'kitty'
        },
        'overwrite_backward': {
            '': 'real',
            'kiddie': 'kitty'
        },
        'wow':{'such':{'deep':[
            None,
            None,
            None,
            {'much':{'power':{'!': 'Amaze'}}}
        ]}},
        'negative':[
            {
                'name': 'John',
                'gender': 'Male',
            },
            {
                'name': 'Jane',
                'gender': 'Female',
            }
        ],
        'out_of_order':[
            'Arcade Game',
            'Toilet',
            'My Brain',
            'ATM',
            'Vending Machine',
        ]
    }
    assert(data == expected)


if __name__ == '__main__':
    test()