Dedupe by Key With Last-Write-Wins (JS)

// Last-write-wins dedupe: a Map keyed by the dedupe field.
// Iteration order on a JS Map is insertion order, so this preserves arrival order
// while keeping the most recent value for each key.

function dedupeBy(rows, keyOf) {
    const pickKey = typeof keyOf === 'function' ? keyOf : (r) => r[keyOf];
    const out = new Map();
    for (const row of rows) out.set(pickKey(row), row);
    return [...out.values()];
}

const events = [
    { id: 'u1', name: 'Alice',  ts: 100 },
    { id: 'u2', name: 'Bob',    ts: 110 },
    { id: 'u1', name: 'Alicia', ts: 130 },  // newer write for u1
    { id: 'u3', name: 'Carol',  ts: 140 },
];

console.log(dedupeBy(events, 'id'));
// [{id:'u1',name:'Alicia',ts:130}, {id:'u2',name:'Bob',ts:110}, {id:'u3',name:'Carol',ts:140}]

// When 'last write wins' is wrong: late events have only some fields.
// Pass a merge resolver to combine the existing row with the new one.

function dedupeBy(rows, keyOf, merge) {
    const pickKey = typeof keyOf === 'function' ? keyOf : (r) => r[keyOf];
    const out = new Map();
    for (const row of rows) {
        const k = pickKey(row);
        const prev = out.get(k);
        out.set(k, prev === undefined ? row : merge(prev, row));
    }
    return [...out.values()];
}

// Real use: a backfill emits partial profile updates. Drop nulls, prefer newer ts.
function mergeProfiles(prev, next) {
    const merged = { ...prev };
    for (const [k, v] of Object.entries(next)) {
        if (v !== null && v !== undefined) merged[k] = v;
    }
    merged.ts = Math.max(prev.ts ?? 0, next.ts ?? 0);
    return merged;
}

const events = [
    { id: 'u1', name: 'Alice', email: '[email protected]',  ts: 100 },
    { id: 'u1', name: null,    email: 'alice@x',  ts: 130 },  // backfill: only email
    { id: 'u2', name: 'Bob',   email: null,       ts: 110 },
    { id: 'u2', name: 'Bob',   email: '[email protected]',  ts: 150 },
];

console.log(dedupeBy(events, 'id', mergeProfiles));

// When the input is millions of rows from a stream, you do not want to
// materialize the full array first. Yield as keys 'finalize'.
// Strategy: sort the stream by key (or assume already-sorted), then emit
// one record per run.

function* dedupeByStream(iter, keyOf, merge) {
    const pickKey = typeof keyOf === 'function' ? keyOf : (r) => r[keyOf];
    let currentKey;
    let current;
    let started = false;
    for (const row of iter) {
        const k = pickKey(row);
        if (!started) {
            currentKey = k;
            current = row;
            started = true;
            continue;
        }
        if (k === currentKey) {
            current = merge ? merge(current, row) : row;
        } else {
            yield current;
            currentKey = k;
            current = row;
        }
    }
    if (started) yield current;
}

// Caller is responsible for sorting; here we mock a sorted stream.
function* sortedStream(rows, keyOf) {
    const pick = (r) => r[keyOf];
    const sorted = [...rows].sort((a, b) => (pick(a) < pick(b) ? -1 : pick(a) > pick(b) ? 1 : 0));
    for (const r of sorted) yield r;
}

const events = [
    { id: 'u1', name: 'Alice',  ts: 100 },
    { id: 'u3', name: 'Carol',  ts: 140 },
    { id: 'u1', name: 'Alicia', ts: 130 },
    { id: 'u2', name: 'Bob',    ts: 110 },
    { id: 'u1', name: 'Al',     ts: 160 },
];

const sorted = sortedStream(events, 'id');
for (const row of dedupeByStream(sorted, 'id')) {
    console.log(row);
}