Skip to content

Commit

Permalink
mergeTree: implement sided obliterate (microsoft#22643)
Browse files Browse the repository at this point in the history 
This PR implements the new OBLITERATE_SIDED op type introduced by
microsoft#22596.
This allows obliteration ranges to be defined using either inclusive or
exclusive bounds on each endpoint. Using an exclusive bound will allow
the obliterated range to "grow" to include concurrently inserted
segments adjacent to that endpoint.
  • Loading branch information
@titrindl
titrindl authored Sep 30, 2024
1 parent 5163b54 commit dd433df
Show file tree
Hide file tree
Showing 5 changed files with 262 additions and 77 deletions.
108 changes: 91 additions & 17 deletions packages/dds/merge-tree/src/client.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ import {
} from "./ops.js";
import { PropertySet } from "./properties.js";
import { DetachedReferencePosition, ReferencePosition } from "./referencePositions.js";
import { type InteriorSequencePlace } from "./sequencePlace.js";
import { Side, type InteriorSequencePlace } from "./sequencePlace.js";
import { SnapshotLoader } from "./snapshotLoader.js";
import { SnapshotV1 } from "./snapshotV1.js";
import { SnapshotLegacy } from "./snapshotlegacy.js";
Expand Down Expand Up @@ -494,17 +494,16 @@ export class Client extends TypedEventEmitter<IClientEvents> {
const op = opArgs.op;
const clientArgs = this.getClientSequenceArgs(opArgs);
if (this._mergeTree.options?.mergeTreeEnableSidedObliterate) {
/*
const _start: InteriorSequencePlace =
typeof op.pos1 === "object"
? { pos: op.pos1.pos, side: op.pos1.before ? Side.Before : Side.After }
: { pos: op.pos1, side: Side.Before };
const _end: InteriorSequencePlace =
typeof op.pos2 === "object"
? { pos: op.pos2.pos, side: op.pos2.before ? Side.Before : Side.After }
: { pos: op.pos2 - 1, side: Side.After };
*/
assert(false, "TODO: sided obliterate will come in a follow-up PR shortly.");
const { start, end } = this.getValidSidedRange(op, clientArgs);
this._mergeTree.obliterateRange(
start,
end,
clientArgs.referenceSequenceNumber,
clientArgs.clientId,
clientArgs.sequenceNumber,
false,
opArgs,
);
} else {
assert(
op.type === MergeTreeDeltaType.OBLITERATE,
Expand Down Expand Up @@ -598,6 +597,82 @@ export class Client extends TypedEventEmitter<IClientEvents> {
);
}

/**
* Returns a valid range for the op, or throws if the range is invalid
* @param op - The op to generate the range for
* @param clientArgs - The client args for the op
* @throws LoggingError if the range is invalid
*/
private getValidSidedRange(
// eslint-disable-next-line import/no-deprecated
op: IMergeTreeObliterateSidedMsg | IMergeTreeObliterateMsg,
clientArgs: IMergeTreeClientSequenceArgs,
): {
start: InteriorSequencePlace;
end: InteriorSequencePlace;
} {
const invalidPositions: string[] = [];
let start: InteriorSequencePlace | undefined;
let end: InteriorSequencePlace | undefined;
if (op.pos1 === undefined) {
invalidPositions.push("start");
} else {
start =
typeof op.pos1 === "object"
? { pos: op.pos1.pos, side: op.pos1.before ? Side.Before : Side.After }
: { pos: op.pos1, side: Side.Before };
}
if (op.pos2 === undefined) {
invalidPositions.push("end");
} else {
end =
typeof op.pos2 === "object"
? { pos: op.pos2.pos, side: op.pos2.before ? Side.Before : Side.After }
: { pos: op.pos2 - 1, side: Side.After };
}

// Validate if local op
if (clientArgs.clientId === this.getClientId()) {
const length = this._mergeTree.getLength(
this.getCollabWindow().currentSeq,
this.getClientId(),
);
if (start !== undefined && (start.pos >= length || start.pos < 0)) {
// start out of bounds
invalidPositions.push("start");
}
if (end !== undefined && (end.pos >= length || end.pos < 0)) {
invalidPositions.push("end");
}
if (
start !== undefined &&
end !== undefined &&
(start.pos > end.pos ||
(start.pos === end.pos && start.side !== end.side && start.side === Side.After))
) {
// end is before start
invalidPositions.push("inverted");
}
if (invalidPositions.length > 0) {
throw new LoggingError("InvalidRange", {
usageError: true,
invalidPositions: invalidPositions.toString(),
length,
opType: op.type,
opPos1Relative: op.relativePos1 !== undefined,
opPos2Relative: op.relativePos2 !== undefined,
opPos1: JSON.stringify(op.pos1),
opPos2: JSON.stringify(op.pos2),
start: JSON.stringify(start),
end: JSON.stringify(end),
});
}
}

assert(start !== undefined && end !== undefined, "Missing start or end of range");
return { start, end };
}

/**
* Returns a valid range for the op, or undefined
* @param op - The op to generate the range for
Expand Down Expand Up @@ -976,7 +1051,8 @@ export class Client extends TypedEventEmitter<IClientEvents> {
this.applyAnnotateRangeOp(opArgs);
break;
}
case MergeTreeDeltaType.OBLITERATE: {
case MergeTreeDeltaType.OBLITERATE:
case MergeTreeDeltaType.OBLITERATE_SIDED: {
this.applyObliterateRangeOp(opArgs);
break;
}
Expand Down Expand Up @@ -1010,14 +1086,11 @@ export class Client extends TypedEventEmitter<IClientEvents> {
this.applyAnnotateRangeOp({ op });
break;
}
case MergeTreeDeltaType.OBLITERATE_SIDED:
case MergeTreeDeltaType.OBLITERATE: {
this.applyObliterateRangeOp({ op });
break;
}
case MergeTreeDeltaType.OBLITERATE_SIDED: {
assert(false, "TODO: sided obliterate will come in a follow-up PR shortly.");
break;
}
case MergeTreeDeltaType.GROUP: {
op.ops.map((o) => this.applyStashedOp(o));
break;
Expand Down Expand Up @@ -1246,6 +1319,7 @@ export class Client extends TypedEventEmitter<IClientEvents> {
this.applyRemoveRangeOp(opArgs);
break;
}
case MergeTreeDeltaType.OBLITERATE_SIDED:
case MergeTreeDeltaType.OBLITERATE: {
this.applyObliterateRangeOp(opArgs);
break;
Expand Down
114 changes: 93 additions & 21 deletions packages/dds/merge-tree/src/mergeTree.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ import {
} from "./referencePositions.js";
// eslint-disable-next-line import/no-deprecated
import { PropertiesRollback } from "./segmentPropertiesManager.js";
import { Side, type InteriorSequencePlace } from "./sequencePlace.js";
import { SortedSegmentSet } from "./sortedSegmentSet.js";
import { zamboniSegments } from "./zamboni.js";

Expand Down Expand Up @@ -1247,15 +1248,19 @@ export class MergeTree {
});
});

if (opArgs.op.type === MergeTreeDeltaType.OBLITERATE) {
if (
opArgs.op.type === MergeTreeDeltaType.OBLITERATE ||
opArgs.op.type === MergeTreeDeltaType.OBLITERATE_SIDED
) {
this.obliterates.addOrUpdate(pendingSegmentGroup.obliterateInfo!);
}

// Perform slides after all segments have been acked, so that
// positions after slide are final
if (
opArgs.op.type === MergeTreeDeltaType.REMOVE ||
opArgs.op.type === MergeTreeDeltaType.OBLITERATE
opArgs.op.type === MergeTreeDeltaType.OBLITERATE ||
opArgs.op.type === MergeTreeDeltaType.OBLITERATE_SIDED
) {
this.slideAckedRemovedSegmentReferences(pendingSegmentGroup.segments);
}
Expand Down Expand Up @@ -1555,13 +1560,13 @@ export class MergeTree {
movedClientIds.unshift(ob.clientId);
movedSeqs.unshift(ob.seq);
} else {
if (newest === undefined || normalizedNewestSeq < normalizedObSeq) {
normalizedNewestSeq = normalizedObSeq;
newest = ob;
}
movedClientIds.push(ob.clientId);
movedSeqs.push(ob.seq);
}
if (newest === undefined || normalizedNewestSeq < normalizedObSeq) {
normalizedNewestSeq = normalizedObSeq;
newest = ob;
}
}
}

Expand All @@ -1588,7 +1593,10 @@ export class MergeTree {
if (newSegment.parent) {
this.blockUpdatePathLengths(newSegment.parent, seq, clientId);
}
} else if (oldest && newest?.clientId === clientId) {
newSegment.prevObliterateByInserter = newest;
}

saveIfLocal(newSegment);
}
}
Expand Down Expand Up @@ -1880,19 +1888,20 @@ export class MergeTree {
}
}

public obliterateRange(
start: number,
end: number,
private obliterateRangeSided(
start: InteriorSequencePlace,
end: InteriorSequencePlace,
refSeq: number,
clientId: number,
seq: number,
overwrite: boolean = false,
opArgs: IMergeTreeDeltaOpArgs,
): void {
errorIfOptionNotTrue(this.options, "mergeTreeEnableObliterate");
const startPos = start.side === Side.Before ? start.pos : start.pos + 1;
const endPos = end.side === Side.Before ? end.pos : end.pos + 1;

this.ensureIntervalBoundary(start, refSeq, clientId);
this.ensureIntervalBoundary(end, refSeq, clientId);
this.ensureIntervalBoundary(startPos, refSeq, clientId);
this.ensureIntervalBoundary(endPos, refSeq, clientId);

let _overwrite = overwrite;
const localOverlapWithRefs: ISegment[] = [];
Expand All @@ -1910,16 +1919,16 @@ export class MergeTree {
segmentGroup: undefined,
};

const { segment: startSeg } = this.getContainingSegment(start, refSeq, clientId);
const { segment: endSeg } = this.getContainingSegment(end - 1, refSeq, clientId);
const { segment: startSeg } = this.getContainingSegment(start.pos, refSeq, clientId);
const { segment: endSeg } = this.getContainingSegment(end.pos, refSeq, clientId);
assert(
startSeg !== undefined && endSeg !== undefined,
0xa3f /* segments cannot be undefined */,
);

obliterate.start = this.createLocalReferencePosition(
startSeg,
0,
start.side === Side.Before ? 0 : Math.max(startSeg.cachedLength - 1, 0),
ReferenceType.StayOnRemove,
{
obliterate,
Expand All @@ -1928,20 +1937,53 @@ export class MergeTree {

obliterate.end = this.createLocalReferencePosition(
endSeg,
endSeg.cachedLength - 1,
end.side === Side.Before ? 0 : Math.max(endSeg.cachedLength - 1, 0),
ReferenceType.StayOnRemove,
{
obliterate,
},
);

// Always create a segment group for obliterate,
// even if there are no segments currently in the obliteration range.
// Segments may be concurrently inserted into the obliteration range,
// at which point they are added to the segment group.
obliterate.segmentGroup = {
segments: [],
localSeq,
refSeq: this.collabWindow.currentSeq,
obliterateInfo: obliterate,
};
if (this.collabWindow.collaborating && clientId === this.collabWindow.clientId) {
this.pendingSegments.push(obliterate.segmentGroup);
}
this.obliterates.addOrUpdate(obliterate);

const markMoved = (
segment: ISegment,
pos: number,
_start: number,
_end: number,
): boolean => {
if (
(start.side === Side.After && startPos === pos + segment.cachedLength) || // exclusive start segment
(end.side === Side.Before &&
endPos === pos &&
isSegmentPresent(segment, { refSeq, localSeq })) // exclusive end segment
) {
// We walk these segments because we want to also walk any concurrently inserted segments between here and the obliterated segments.
// These segments are outside of the obliteration range though, so return true to keep walking.
return true;
}
const existingMoveInfo = toMoveInfo(segment);

if (segment.prevObliterateByInserter?.seq === UnassignedSequenceNumber) {
// We chose to not obliterate this segment because we are aware of an unacked local obliteration.
// The local obliterate has not been sequenced yet, so it is still the newest obliterate we are aware of.
// Other clients will also choose not to obliterate this segment because the most recent obliteration has the same clientId
return true;
}

if (
clientId !== segment.clientId &&
segment.seq !== undefined &&
Expand Down Expand Up @@ -1992,7 +2034,6 @@ export class MergeTree {
obliterate.segmentGroup,
localSeq,
);
obliterate.segmentGroup.obliterateInfo ??= obliterate;
} else {
if (MergeTree.options.zamboniSegments) {
this.addToLRUSet(segment, seq);
Expand Down Expand Up @@ -2022,14 +2063,12 @@ export class MergeTree {
markMoved,
undefined,
afterMarkMoved,
start,
end,
start.pos,
end.pos + 1, // include the segment containing the end reference
undefined,
seq === UnassignedSequenceNumber ? undefined : seq,
);

this.obliterates.addOrUpdate(obliterate);

this.slideAckedRemovedSegmentReferences(localOverlapWithRefs);
// opArgs == undefined => test code
if (movedSegments.length > 0) {
Expand All @@ -2055,6 +2094,39 @@ export class MergeTree {
}
}

public obliterateRange(
start: number | InteriorSequencePlace,
end: number | InteriorSequencePlace,
refSeq: number,
clientId: number,
seq: number,
overwrite: boolean = false,
opArgs: IMergeTreeDeltaOpArgs,
): void {
errorIfOptionNotTrue(this.options, "mergeTreeEnableObliterate");
if (this.options?.mergeTreeEnableSidedObliterate) {
assert(
typeof start === "object" && typeof end === "object",
"Start and end must be of type InteriorSequencePlace if mergeTreeEnableSidedObliterate is enabled.",
);
this.obliterateRangeSided(start, end, refSeq, clientId, seq, overwrite, opArgs);
} else {
assert(
typeof start === "number" && typeof end === "number",
"Start and end must be numbers if mergeTreeEnableSidedObliterate is not enabled.",
);
this.obliterateRangeSided(
{ pos: start, side: Side.Before },
{ pos: end - 1, side: Side.After },
refSeq,
clientId,
seq,
overwrite,
opArgs,
);
}
}

public markRangeRemoved(
start: number,
end: number,
Expand Down
3 changes: 2 additions & 1 deletion packages/dds/merge-tree/src/mergeTreeNodes.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -668,7 +668,8 @@ export abstract class BaseSegment implements ISegment {
return false;
}

case MergeTreeDeltaType.OBLITERATE: {
case MergeTreeDeltaType.OBLITERATE:
case MergeTreeDeltaType.OBLITERATE_SIDED: {
const moveInfo: IMoveInfo | undefined = toMoveInfo(this);
assert(moveInfo !== undefined, 0x86e /* On obliterate ack, missing move info! */);
const obliterateInfo = segmentGroup.obliterateInfo;
Expand Down
Loading

0 comments on commit dd433df

Please sign in to comment.