Skip to content

Commit

Permalink
Merge pull request #5 from qiuhaotc/SupportMultiCodeIndexes
Browse files Browse the repository at this point in the history
  • Loading branch information
@qiuhaotc
qiuhaotc authored Dec 9, 2020
2 parents 3709381 + 034b62d commit 63d7731
Show file tree
Hide file tree
Showing 6 changed files with 139 additions and 38 deletions.
8 changes: 4 additions & 4 deletions src/CodeIndex.IndexBuilder/CodeIndex.IndexBuilder.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Lucene.Net" Version="4.8.0-beta00013" />
<PackageReference Include="Lucene.Net.Analysis.Common" Version="4.8.0-beta00013" />
<PackageReference Include="Lucene.Net.ICU" Version="4.8.0-beta00013" />
<PackageReference Include="Lucene.Net.QueryParser" Version="4.8.0-beta00013" />
<PackageReference Include="Lucene.Net" Version="4.8.0-beta00007" />
<PackageReference Include="Lucene.Net.Analysis.Common" Version="4.8.0-beta00007" />
<PackageReference Include="Lucene.Net.ICU" Version="4.8.0-beta00007" />
<PackageReference Include="Lucene.Net.QueryParser" Version="4.8.0-beta00007" />
</ItemGroup>

<ItemGroup>
Expand Down
53 changes: 47 additions & 6 deletions src/CodeIndex.IndexBuilder/CodeIndexBuilderLight.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,13 +50,14 @@ public void InitIndexFolderIfNeeded()

public ConcurrentBag<FileInfo> BuildIndexByBatch(IEnumerable<FileInfo> fileInfos, bool needCommit, bool triggerMerge, bool applyAllDeletes, CancellationToken cancellationToken, int batchSize = 10000)
{
cancellationToken.ThrowIfCancellationRequested();
fileInfos.RequireNotNull(nameof(fileInfos));
batchSize.RequireRange(nameof(batchSize), int.MaxValue, 50);

var codeDocuments = new ConcurrentBag<Document>();
var hintWords = new ConcurrentDictionary<string, int>();
var failedIndexFiles = new ConcurrentBag<FileInfo>();
var readWriteSlimLock = new ReaderWriterLockSlim();
using var readWriteSlimLock = new ReaderWriterLockSlim();

Parallel.ForEach(fileInfos, fileInfo =>
{
Expand Down Expand Up @@ -145,6 +146,47 @@ public void DeleteAllIndex()
return CodeIndexPool.Search(new MatchAllDocsQuery(), int.MaxValue).Select(u => (u.Get(nameof(CodeSource.FilePath)), new DateTime(long.Parse(u.Get(nameof(CodeSource.LastWriteTimeUtc)))))).ToList();
}

public bool CreateIndex(FileInfo fileInfo)
{
try
{
if (fileInfo.Exists)
{
var source = CodeSource.GetCodeSource(fileInfo, FilesContentHelper.ReadAllText(fileInfo.FullName));

var words = new HashSet<string>();
AddHintWords(words, source.Content);

var doc = CodeIndexBuilder.GetDocumentFromSource(source);
CodeIndexPool.BuildIndex(new[] { doc }, false);

foreach (var word in words)
{
HintIndexPool.UpdateIndex(new Term(nameof(CodeWord.Word), word), new Document
{
new StringField(nameof(CodeWord.Word), word, Field.Store.YES),
new StringField(nameof(CodeWord.WordLower), word.ToLowerInvariant(), Field.Store.YES)
});
}

Log.Info($"{Name}: Update index For {source.FilePath} finished");
}

return true;
}
catch (Exception ex)
{
Log.Error($"{Name}: Update index for {fileInfo.FullName} failed, exception: " + ex);

if (ex is OperationCanceledException)
{
throw;
}

return false;
}
}

void BuildIndex(bool needCommit, bool triggerMerge, bool applyAllDeletes, ConcurrentBag<Document> codeDocuments, ConcurrentDictionary<string, int> words, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
Expand All @@ -157,8 +199,6 @@ void BuildIndex(bool needCommit, bool triggerMerge, bool applyAllDeletes, Concur

foreach (var word in words)
{
cancellationToken.ThrowIfCancellationRequested();

HintIndexPool.UpdateIndex(new Term(nameof(CodeWord.Word), word.Key), new Document
{
new StringField(nameof(CodeWord.Word), word.Key, Field.Store.YES),
Expand All @@ -174,14 +214,15 @@ void BuildIndex(bool needCommit, bool triggerMerge, bool applyAllDeletes, Concur
Log.Info($"{Name}: Build hint index finished");
}

public bool RenameFolderIndexes(string oldFolderPath, string nowFolderPath)
public bool RenameFolderIndexes(string oldFolderPath, string nowFolderPath, CancellationToken cancellationToken)
{
try
{
var documents = CodeIndexPool.Search(new TermQuery(GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), oldFolderPath)), 1);

foreach (var document in documents)
{
cancellationToken.ThrowIfCancellationRequested();
RenameIndex(document, oldFolderPath, nowFolderPath);
}

Expand Down Expand Up @@ -249,6 +290,8 @@ public bool UpdateIndex(FileInfo fileInfo, CancellationToken cancellationToken)
{
if (fileInfo.Exists)
{
cancellationToken.ThrowIfCancellationRequested();

var source = CodeSource.GetCodeSource(fileInfo, FilesContentHelper.ReadAllText(fileInfo.FullName));

var words = new HashSet<string>();
Expand All @@ -259,8 +302,6 @@ public bool UpdateIndex(FileInfo fileInfo, CancellationToken cancellationToken)

foreach (var word in words)
{
cancellationToken.ThrowIfCancellationRequested();

// TODO: Delete And Add Hint Words

HintIndexPool.UpdateIndex(new Term(nameof(CodeWord.Word), word), new Document
Expand Down
2 changes: 1 addition & 1 deletion src/CodeIndex.IndexBuilder/CodeTokenUtils/CodeTokenizer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ namespace CodeIndex.IndexBuilder
/// <summary>
/// Reference the SmartCn Tokenizer
/// </summary>
internal sealed class CodeTokenizer : SegmentingTokenizerBase
internal class CodeTokenizer : SegmentingTokenizerBase
{
static readonly BreakIterator sentenceProto = BreakIterator.GetSentenceInstance(CultureInfo.InvariantCulture);
readonly WordSegmenter wordSegmenter = new WordSegmenter();
Expand Down
74 changes: 48 additions & 26 deletions src/CodeIndex.MaintainIndex/IndexMaintainer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,10 @@ public IndexMaintainer(IndexConfig indexConfig, CodeIndexConfiguration codeIndex
ExcludedExtensions = indexConfig.ExcludedExtensionsArray.Select(u => u.ToUpperInvariant()).ToArray();
ExcludedPaths = FilePathHelper.GetPaths(indexConfig.ExcludedPathsArray, codeIndexConfiguration.IsInLinux);
IncludedExtensions = indexConfig.IncludedExtensionsArray?.Select(u => u.ToUpperInvariant()).ToArray() ?? Array.Empty<string>();
TokenSource = new CancellationTokenSource();
}

public async Task InitializeIndex(bool forceRebuild, CancellationToken cancellationToken)
public async Task InitializeIndex(bool forceRebuild)
{
if (Status != IndexStatus.Idle)
{
Expand All @@ -45,8 +46,8 @@ public async Task InitializeIndex(bool forceRebuild, CancellationToken cancellat
{
await Task.Run(() =>
{
InitializeIndexCore(forceRebuild, cancellationToken);
}, cancellationToken);
InitializeIndexCore(forceRebuild);
}, TokenSource.Token);

Status = IndexStatus.Initialized;
}
Expand All @@ -65,21 +66,21 @@ await Task.Run(() =>
}
}

public async Task MaintainIndexes(CancellationToken cancellationToken)
public async Task MaintainIndexes()
{
if (Status != IndexStatus.Initialized)
{
return;
}

await MaintainIndexesCore(cancellationToken);
await MaintainIndexesCore();
}

async Task MaintainIndexesCore(CancellationToken cancellationToken)
async Task MaintainIndexesCore()
{
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
TokenSource.Token.ThrowIfCancellationRequested();

var fetchEndDate = DateTime.Now.AddSeconds(-6);
var notChangedDuring = fetchEndDate.AddSeconds(3);
Expand All @@ -95,16 +96,16 @@ async Task MaintainIndexesCore(CancellationToken cancellationToken)

ProcessingChanges(orderedNeedProcessingChanges);

// TODO: Commit Changes If Needed
IndexBuilderLight.Commit();
}
}

if (cancellationToken.IsCancellationRequested)
if (TokenSource.Token.IsCancellationRequested)
{
throw new OperationCanceledException();
}

await Task.Delay(3000, cancellationToken);
await Task.Delay(3000, TokenSource.Token);
}
}

Expand All @@ -114,6 +115,8 @@ void ProcessingChanges(List<ChangedSource> orderedNeedProcessingChanges)

foreach (var changes in orderedNeedProcessingChanges)
{
TokenSource.Token.ThrowIfCancellationRequested();

switch (changes.ChangesType)
{
case WatcherChangeTypes.Changed:
Expand All @@ -139,24 +142,30 @@ void ProcessingChanges(List<ChangedSource> orderedNeedProcessingChanges)

void CreateIndex(ChangedSource changes)
{
throw new NotImplementedException();
if (IsFile(changes.FilePath))
{
IndexBuilderLight.CreateIndex(new FileInfo(changes.FilePath));
}
}

void RenameIndex(ChangedSource changes)
{
if (IsExcludedFromIndex(changes.FilePath))
{
IndexBuilderLight.DeleteIndex(changes.OldPath);
if (!IsExcludedFromIndex(changes.OldPath))
{
IndexBuilderLight.DeleteIndex(changes.OldPath);
}
}
else
{
if(IsFile(changes.FilePath))
if (IsFile(changes.FilePath))
{
IndexBuilderLight.RenameFileIndex(changes.OldPath, changes.FilePath);
}
else if (IsDirectory(changes.FilePath))
{
IndexBuilderLight.RenameFolderIndexes(changes.OldPath, changes.FilePath);
IndexBuilderLight.RenameFolderIndexes(changes.OldPath, changes.FilePath, TokenSource.Token);
}
}
}
Expand All @@ -178,10 +187,13 @@ void DeleteIndex(ChangedSource changes)

void UpdateIndex(ChangedSource changes)
{
throw new NotImplementedException();
if (IsFile(changes.FilePath))
{
IndexBuilderLight.UpdateIndex(new FileInfo(changes.FilePath), TokenSource.Token);
}
}

void InitializeIndexCore(bool forceRebuild, CancellationToken cancellationToken)
void InitializeIndexCore(bool forceRebuild)
{
var folders = IndexConfig.GetFolders(CodeIndexConfiguration.LuceneIndex);

Expand Down Expand Up @@ -220,14 +232,14 @@ void InitializeIndexCore(bool forceRebuild, CancellationToken cancellationToken)

foreach (var codeSource in allCodeSource)
{
cancellationToken.ThrowIfCancellationRequested();
TokenSource.Token.ThrowIfCancellationRequested();

if (allFilesDictionary.TryGetValue(codeSource.FilePath, out var fileInfo))
{
if (fileInfo.LastWriteTimeUtc != codeSource.LastWriteTimeUtc)
{
Log.Info($"{IndexConfig.IndexName}: File {fileInfo.FullName} modified");
if (!IndexBuilderLight.UpdateIndex(fileInfo, cancellationToken))
if (!IndexBuilderLight.UpdateIndex(fileInfo, TokenSource.Token))
{
failedUpdateOrDeleteFiles.Add(codeSource.FilePath);
}
Expand All @@ -253,7 +265,7 @@ void InitializeIndexCore(bool forceRebuild, CancellationToken cancellationToken)
}
}

AddNewIndexFiles(needToBuildIndex ?? allFiles, out var failedIndexFiles, cancellationToken);
AddNewIndexFiles(needToBuildIndex ?? allFiles, out var failedIndexFiles);

IndexBuilderLight.Commit();

Expand All @@ -267,34 +279,41 @@ void InitializeIndexCore(bool forceRebuild, CancellationToken cancellationToken)
}
}

void AddNewIndexFiles(IEnumerable<FileInfo> needToBuildIndex, out ConcurrentBag<FileInfo> failedIndexFiles, CancellationToken cancellationToken)
void AddNewIndexFiles(IEnumerable<FileInfo> needToBuildIndex, out ConcurrentBag<FileInfo> failedIndexFiles)
{
failedIndexFiles = IndexBuilderLight.BuildIndexByBatch(needToBuildIndex, true, false, false, cancellationToken);
failedIndexFiles = IndexBuilderLight.BuildIndexByBatch(needToBuildIndex, true, false, false, TokenSource.Token);

if (failedIndexFiles.Count > 0)
{
Log.Info($"{IndexConfig.IndexName}: Retry failed build indexes files, files count {failedIndexFiles.Count}");
failedIndexFiles = IndexBuilderLight.BuildIndexByBatch(failedIndexFiles, true, false, false, cancellationToken);
failedIndexFiles = IndexBuilderLight.BuildIndexByBatch(failedIndexFiles, true, false, false, TokenSource.Token);
}
}

void OnChange(object sender, FileSystemEventArgs e)
{
ChangedSources.Enqueue(new ChangedSource
var changeSource = new ChangedSource
{
ChangesType = e.ChangeType,
FilePath = e.FullPath
});
};

if (!IsExcludedFromIndex(changeSource.FilePath))
{
ChangedSources.Enqueue(changeSource);
}
}

void OnRename(object sender, RenamedEventArgs e)
{
ChangedSources.Enqueue(new ChangedSource
var changeSource = new ChangedSource
{
ChangesType = e.ChangeType,
FilePath = e.FullPath,
OldPath = e.OldFullPath
});
};

ChangedSources.Enqueue(changeSource);
}

public IndexConfig IndexConfig { get; }
Expand All @@ -305,6 +324,7 @@ void OnRename(object sender, RenamedEventArgs e)
public string Description { get; set; }
public bool IsDisposing { get; private set; }
FileSystemWatcher FilesWatcher { get; set; }
public CancellationTokenSource TokenSource { get; }
ConcurrentQueue<ChangedSource> ChangedSources { get; set; }
ConcurrentQueue<PendingRetrySource> PendingRetryCodeSources { get; set; }
string[] ExcludedExtensions { get; }
Expand All @@ -316,6 +336,8 @@ public void Dispose()
if (!IsDisposing)
{
IsDisposing = true;
TokenSource.Cancel();
TokenSource.Dispose();
FilesWatcher?.Dispose();
IndexBuilderLight?.Dispose();
}
Expand Down
Loading

0 comments on commit 63d7731

Please sign in to comment.