deduplicate

deduplicate.py

def deduplicate(ls: list, selector=lambda x: x) -> list:
    seen = set()
    return [x for x in ls if (e := selector(x)) not in seen and (seen.add(e) or True)]

test.py

def test_deduplicate():
    # 普通 list 去重
    assert deduplicate([1, 2, 2, 3]) == [1, 2, 3]
    
    data = [{"id": 1, "name": "a"}, {"id": 1, "name": "b"}]
    # 整个 dict 比较
    assert deduplicate(data, lambda x: tuple(x.items())) == data
    # 某个 key 作为唯一键比较
    assert deduplicate(data, lambda x: x["id"]) == [{"id": 1, "name": "a"}]