Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

BREAKING: Indexing Sequence Number #887

Closed
wants to merge 5 commits into from
Closed

BREAKING: Indexing Sequence Number #887

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
13a0a6c
initial sequence numbers patch
Jeevananthan-23 Nov 2, 2023
281a839
completely cutover TrackingIndexWriter and all IW change to return seqNo
Jeevananthan-23 Nov 3, 2023
487597c
seqno: adding more test and bug fix in concurrency issues
Jeevananthan-23 Nov 4, 2023
255ca90
seqno: fix text bug
Jeevananthan-23 Nov 8, 2023
00d3942
seqno: fix bug in IW testpreparecommit
Jeevananthan-23 Nov 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions src/Lucene.Net.Facet/Taxonomy/Directory/DirectoryTaxonomyWriter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -745,7 +745,7 @@ private void RefreshReaderManager()
}
}

public virtual void Commit()
public virtual long Commit()
{
UninterruptableMonitor.Enter(syncLock);
try
Expand All @@ -758,7 +758,7 @@ public virtual void Commit()
{
indexWriter.SetCommitData(CombinedCommitData(indexWriter.CommitData));
}
indexWriter.Commit();
return indexWriter.Commit();
}
finally
{
Expand Down Expand Up @@ -790,7 +790,7 @@ public virtual void SetCommitData(IDictionary<string, string> commitUserData)
/// prepare most of the work needed for a two-phase commit.
/// See <see cref="IndexWriter.PrepareCommit"/>.
/// </summary>
public virtual void PrepareCommit()
public virtual long PrepareCommit()
{
UninterruptableMonitor.Enter(syncLock);
try
Expand All @@ -803,7 +803,7 @@ public virtual void PrepareCommit()
{
indexWriter.SetCommitData(CombinedCommitData(indexWriter.CommitData));
}
indexWriter.PrepareCommit();
return indexWriter.PrepareCommit();
}
finally
{
Expand Down
46 changes: 26 additions & 20 deletions src/Lucene.Net.TestFramework/Index/RandomIndexWriter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,27 +125,29 @@ public RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c)
/// <summary>
/// Adds a Document. </summary>
/// <seealso cref="IndexWriter.AddDocument(IEnumerable{IIndexableField})"/>
public virtual void AddDocument(IEnumerable<IIndexableField> doc)
public virtual long AddDocument(IEnumerable<IIndexableField> doc)
{
AddDocument(doc, IndexWriter.Analyzer);
return AddDocument(doc, IndexWriter.Analyzer);
}

public virtual void AddDocument(IEnumerable<IIndexableField> doc, Analyzer a)
public virtual long AddDocument(IEnumerable<IIndexableField> doc, Analyzer a)
{
long seqNo;
if (r.Next(5) == 3)
{
// TODO: maybe, we should simply buffer up added docs
// (but we need to clone them), and only when
// getReader, commit, etc. are called, we do an
// addDocuments? Would be better testing.
IndexWriter.AddDocuments(new EnumerableAnonymousClass<IIndexableField>(doc), a);
seqNo = IndexWriter.AddDocuments(new EnumerableAnonymousClass<IIndexableField>(doc), a);
}
else
{
IndexWriter.AddDocument(doc, a);
seqNo = IndexWriter.AddDocument(doc, a);
}

MaybeCommit();
return seqNo;
}

private sealed class EnumerableAnonymousClass<IndexableField> : IEnumerable<IEnumerable<IndexableField>>
Expand Down Expand Up @@ -223,32 +225,36 @@ private void MaybeCommit()
}
}

public virtual void AddDocuments(IEnumerable<IEnumerable<IIndexableField>> docs)
public virtual long AddDocuments(IEnumerable<IEnumerable<IIndexableField>> docs)
{
IndexWriter.AddDocuments(docs);
long seqNo = IndexWriter.AddDocuments(docs);
MaybeCommit();
return seqNo;
}

public virtual void UpdateDocuments(Term delTerm, IEnumerable<IEnumerable<IIndexableField>> docs)
public virtual long UpdateDocuments(Term delTerm, IEnumerable<IEnumerable<IIndexableField>> docs)
{
IndexWriter.UpdateDocuments(delTerm, docs);
long seqNo = IndexWriter.UpdateDocuments(delTerm, docs);
MaybeCommit();
return seqNo;
}

/// <summary>
/// Updates a document. </summary>
/// <see cref="IndexWriter.UpdateDocument(Term, IEnumerable{IIndexableField})"/>
public virtual void UpdateDocument(Term t, IEnumerable<IIndexableField> doc)
public virtual long UpdateDocument(Term t, IEnumerable<IIndexableField> doc)
{
long seqNo;
if (r.Next(5) == 3)
{
IndexWriter.UpdateDocuments(t, new EnumerableAnonymousClass2(doc));
seqNo = IndexWriter.UpdateDocuments(t, new EnumerableAnonymousClass2(doc));
}
else
{
IndexWriter.UpdateDocument(t, doc);
seqNo = IndexWriter.UpdateDocument(t, doc);
}
MaybeCommit();
return seqNo;
}

private sealed class EnumerableAnonymousClass2 : IEnumerable<IEnumerable<IIndexableField>>
Expand Down Expand Up @@ -306,25 +312,25 @@ public void Dispose()
}
}

public virtual void AddIndexes(params Directory[] dirs)
public virtual long AddIndexes(params Directory[] dirs)
=> IndexWriter.AddIndexes(dirs);

public virtual void AddIndexes(params IndexReader[] readers)
public virtual long AddIndexes(params IndexReader[] readers)
=> IndexWriter.AddIndexes(readers);

public virtual void UpdateNumericDocValue(Term term, string field, long? value)
public virtual long UpdateNumericDocValue(Term term, string field, long? value)
=> IndexWriter.UpdateNumericDocValue(term, field, value);

public virtual void UpdateBinaryDocValue(Term term, string field, BytesRef value)
public virtual long UpdateBinaryDocValue(Term term, string field, BytesRef value)
=> IndexWriter.UpdateBinaryDocValue(term, field, value);

public virtual void DeleteDocuments(Term term)
public virtual long DeleteDocuments(Term term)
=> IndexWriter.DeleteDocuments(term);

public virtual void DeleteDocuments(Query q)
public virtual long DeleteDocuments(Query q)
=> IndexWriter.DeleteDocuments(q);

public virtual void Commit()
public virtual long Commit()
=> IndexWriter.Commit();

public virtual int NumDocs
Expand All @@ -333,7 +339,7 @@ public virtual int NumDocs
public virtual int MaxDoc
=> IndexWriter.MaxDoc;

public virtual void DeleteAll()
public virtual long DeleteAll()
=> IndexWriter.DeleteAll();

public virtual DirectoryReader GetReader()
Expand Down
6 changes: 3 additions & 3 deletions src/Lucene.Net.Tests/Index/TestDocumentsWriterDeleteQueue.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ public virtual void TestUpdateDelteSlices()
}
DeleteSlice slice1 = queue.NewSlice();
DeleteSlice slice2 = queue.NewSlice();
BufferedUpdates bd1 = new BufferedUpdates();
BufferedUpdates bd2 = new BufferedUpdates();
BufferedUpdates bd1 = new BufferedUpdates("bd1");
BufferedUpdates bd2 = new BufferedUpdates("bd2");
int last1 = 0;
int last2 = 0;
ISet<Term> uniqueValues = new JCG.HashSet<Term>();
Expand Down Expand Up @@ -312,7 +312,7 @@ protected internal UpdateThread(DocumentsWriterDeleteQueue queue, AtomicInt32 in
this.index = index;
this.ids = ids;
this.slice = queue.NewSlice();
deletes = new BufferedUpdates();
deletes = new BufferedUpdates("deletes");
this.latch = latch;
}

Expand Down
2 changes: 1 addition & 1 deletion src/Lucene.Net.Tests/Index/TestIndexWriterConfig.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ public virtual void TestDefaults()
getters.Add("getIndexingChain");
getters.Add("getMergedSegmentWarmer");
getters.Add("getMergePolicy");
getters.Add("getMaxThreadStates");
getters.Add("getMaxThreadStates");//why should removed?
getters.Add("getReaderPooling");
getters.Add("getIndexerThreadPool");
getters.Add("getReaderTermsIndexDivisor");
Expand Down
4 changes: 2 additions & 2 deletions src/Lucene.Net.Tests/Index/TestIndexWriterDelete.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1429,8 +1429,8 @@ public virtual void TestTryDeleteDocument()
iwc.SetOpenMode(OpenMode.APPEND);
w = new IndexWriter(d, iwc);
IndexReader r = DirectoryReader.Open(w, false);
Assert.IsTrue(w.TryDeleteDocument(r, 1));
Assert.IsTrue(w.TryDeleteDocument(r.Leaves[0].Reader, 0));
Assert.IsTrue(w.TryDeleteDocument(r, 1) != -1);
Assert.IsTrue(w.TryDeleteDocument(r.Leaves[0].Reader, 0) != -1);
r.Dispose();
w.Dispose();

Expand Down
Loading