From 034b62df9fe9e6efe58c3850988f29d820e1a4b5 Mon Sep 17 00:00:00 2001 From: "QH-PC-NEW\\qiuhao" Date: Wed, 9 Dec 2020 23:10:12 +0800 Subject: [PATCH] commit --- .../CodeIndex.IndexBuilder.csproj | 8 +- .../CodeIndexBuilderLight.cs | 53 +++++++++++-- .../CodeTokenUtils/CodeTokenizer.cs | 2 +- .../IndexMaintainer.cs | 74 ++++++++++++------- .../Search/CodeIndexSearcherTest.cs | 3 +- 5 files changed, 102 insertions(+), 38 deletions(-) diff --git a/src/CodeIndex.IndexBuilder/CodeIndex.IndexBuilder.csproj b/src/CodeIndex.IndexBuilder/CodeIndex.IndexBuilder.csproj index d7fcea1..ffb1252 100644 --- a/src/CodeIndex.IndexBuilder/CodeIndex.IndexBuilder.csproj +++ b/src/CodeIndex.IndexBuilder/CodeIndex.IndexBuilder.csproj @@ -5,10 +5,10 @@ - - - - + + + + diff --git a/src/CodeIndex.IndexBuilder/CodeIndexBuilderLight.cs b/src/CodeIndex.IndexBuilder/CodeIndexBuilderLight.cs index 0968cb4..802fc23 100644 --- a/src/CodeIndex.IndexBuilder/CodeIndexBuilderLight.cs +++ b/src/CodeIndex.IndexBuilder/CodeIndexBuilderLight.cs @@ -50,13 +50,14 @@ public void InitIndexFolderIfNeeded() public ConcurrentBag BuildIndexByBatch(IEnumerable fileInfos, bool needCommit, bool triggerMerge, bool applyAllDeletes, CancellationToken cancellationToken, int batchSize = 10000) { + cancellationToken.ThrowIfCancellationRequested(); fileInfos.RequireNotNull(nameof(fileInfos)); batchSize.RequireRange(nameof(batchSize), int.MaxValue, 50); var codeDocuments = new ConcurrentBag(); var hintWords = new ConcurrentDictionary(); var failedIndexFiles = new ConcurrentBag(); - var readWriteSlimLock = new ReaderWriterLockSlim(); + using var readWriteSlimLock = new ReaderWriterLockSlim(); Parallel.ForEach(fileInfos, fileInfo => { @@ -145,6 +146,47 @@ public void DeleteAllIndex() return CodeIndexPool.Search(new MatchAllDocsQuery(), int.MaxValue).Select(u => (u.Get(nameof(CodeSource.FilePath)), new DateTime(long.Parse(u.Get(nameof(CodeSource.LastWriteTimeUtc)))))).ToList(); } + public bool CreateIndex(FileInfo fileInfo) + { + try + { + if (fileInfo.Exists) + { + var source = CodeSource.GetCodeSource(fileInfo, FilesContentHelper.ReadAllText(fileInfo.FullName)); + + var words = new HashSet(); + AddHintWords(words, source.Content); + + var doc = CodeIndexBuilder.GetDocumentFromSource(source); + CodeIndexPool.BuildIndex(new[] { doc }, false); + + foreach (var word in words) + { + HintIndexPool.UpdateIndex(new Term(nameof(CodeWord.Word), word), new Document + { + new StringField(nameof(CodeWord.Word), word, Field.Store.YES), + new StringField(nameof(CodeWord.WordLower), word.ToLowerInvariant(), Field.Store.YES) + }); + } + + Log.Info($"{Name}: Update index For {source.FilePath} finished"); + } + + return true; + } + catch (Exception ex) + { + Log.Error($"{Name}: Update index for {fileInfo.FullName} failed, exception: " + ex); + + if (ex is OperationCanceledException) + { + throw; + } + + return false; + } + } + void BuildIndex(bool needCommit, bool triggerMerge, bool applyAllDeletes, ConcurrentBag codeDocuments, ConcurrentDictionary words, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -157,8 +199,6 @@ void BuildIndex(bool needCommit, bool triggerMerge, bool applyAllDeletes, Concur foreach (var word in words) { - cancellationToken.ThrowIfCancellationRequested(); - HintIndexPool.UpdateIndex(new Term(nameof(CodeWord.Word), word.Key), new Document { new StringField(nameof(CodeWord.Word), word.Key, Field.Store.YES), @@ -174,7 +214,7 @@ void BuildIndex(bool needCommit, bool triggerMerge, bool applyAllDeletes, Concur Log.Info($"{Name}: Build hint index finished"); } - public bool RenameFolderIndexes(string oldFolderPath, string nowFolderPath) + public bool RenameFolderIndexes(string oldFolderPath, string nowFolderPath, CancellationToken cancellationToken) { try { @@ -182,6 +222,7 @@ public bool RenameFolderIndexes(string oldFolderPath, string nowFolderPath) foreach (var document in documents) { + cancellationToken.ThrowIfCancellationRequested(); RenameIndex(document, oldFolderPath, nowFolderPath); } @@ -249,6 +290,8 @@ public bool UpdateIndex(FileInfo fileInfo, CancellationToken cancellationToken) { if (fileInfo.Exists) { + cancellationToken.ThrowIfCancellationRequested(); + var source = CodeSource.GetCodeSource(fileInfo, FilesContentHelper.ReadAllText(fileInfo.FullName)); var words = new HashSet(); @@ -259,8 +302,6 @@ public bool UpdateIndex(FileInfo fileInfo, CancellationToken cancellationToken) foreach (var word in words) { - cancellationToken.ThrowIfCancellationRequested(); - // TODO: Delete And Add Hint Words HintIndexPool.UpdateIndex(new Term(nameof(CodeWord.Word), word), new Document diff --git a/src/CodeIndex.IndexBuilder/CodeTokenUtils/CodeTokenizer.cs b/src/CodeIndex.IndexBuilder/CodeTokenUtils/CodeTokenizer.cs index 808c9c2..2c4aaaa 100644 --- a/src/CodeIndex.IndexBuilder/CodeTokenUtils/CodeTokenizer.cs +++ b/src/CodeIndex.IndexBuilder/CodeTokenUtils/CodeTokenizer.cs @@ -10,7 +10,7 @@ namespace CodeIndex.IndexBuilder /// /// Reference the SmartCn Tokenizer /// - internal sealed class CodeTokenizer : SegmentingTokenizerBase + internal class CodeTokenizer : SegmentingTokenizerBase { static readonly BreakIterator sentenceProto = BreakIterator.GetSentenceInstance(CultureInfo.InvariantCulture); readonly WordSegmenter wordSegmenter = new WordSegmenter(); diff --git a/src/CodeIndex.MaintainIndex/IndexMaintainer.cs b/src/CodeIndex.MaintainIndex/IndexMaintainer.cs index f375526..e12e332 100644 --- a/src/CodeIndex.MaintainIndex/IndexMaintainer.cs +++ b/src/CodeIndex.MaintainIndex/IndexMaintainer.cs @@ -27,9 +27,10 @@ public IndexMaintainer(IndexConfig indexConfig, CodeIndexConfiguration codeIndex ExcludedExtensions = indexConfig.ExcludedExtensionsArray.Select(u => u.ToUpperInvariant()).ToArray(); ExcludedPaths = FilePathHelper.GetPaths(indexConfig.ExcludedPathsArray, codeIndexConfiguration.IsInLinux); IncludedExtensions = indexConfig.IncludedExtensionsArray?.Select(u => u.ToUpperInvariant()).ToArray() ?? Array.Empty(); + TokenSource = new CancellationTokenSource(); } - public async Task InitializeIndex(bool forceRebuild, CancellationToken cancellationToken) + public async Task InitializeIndex(bool forceRebuild) { if (Status != IndexStatus.Idle) { @@ -45,8 +46,8 @@ public async Task InitializeIndex(bool forceRebuild, CancellationToken cancellat { await Task.Run(() => { - InitializeIndexCore(forceRebuild, cancellationToken); - }, cancellationToken); + InitializeIndexCore(forceRebuild); + }, TokenSource.Token); Status = IndexStatus.Initialized; } @@ -65,21 +66,21 @@ await Task.Run(() => } } - public async Task MaintainIndexes(CancellationToken cancellationToken) + public async Task MaintainIndexes() { if (Status != IndexStatus.Initialized) { return; } - await MaintainIndexesCore(cancellationToken); + await MaintainIndexesCore(); } - async Task MaintainIndexesCore(CancellationToken cancellationToken) + async Task MaintainIndexesCore() { while (true) { - cancellationToken.ThrowIfCancellationRequested(); + TokenSource.Token.ThrowIfCancellationRequested(); var fetchEndDate = DateTime.Now.AddSeconds(-6); var notChangedDuring = fetchEndDate.AddSeconds(3); @@ -95,16 +96,16 @@ async Task MaintainIndexesCore(CancellationToken cancellationToken) ProcessingChanges(orderedNeedProcessingChanges); - // TODO: Commit Changes If Needed + IndexBuilderLight.Commit(); } } - if (cancellationToken.IsCancellationRequested) + if (TokenSource.Token.IsCancellationRequested) { throw new OperationCanceledException(); } - await Task.Delay(3000, cancellationToken); + await Task.Delay(3000, TokenSource.Token); } } @@ -114,6 +115,8 @@ void ProcessingChanges(List orderedNeedProcessingChanges) foreach (var changes in orderedNeedProcessingChanges) { + TokenSource.Token.ThrowIfCancellationRequested(); + switch (changes.ChangesType) { case WatcherChangeTypes.Changed: @@ -139,24 +142,30 @@ void ProcessingChanges(List orderedNeedProcessingChanges) void CreateIndex(ChangedSource changes) { - throw new NotImplementedException(); + if (IsFile(changes.FilePath)) + { + IndexBuilderLight.CreateIndex(new FileInfo(changes.FilePath)); + } } void RenameIndex(ChangedSource changes) { if (IsExcludedFromIndex(changes.FilePath)) { - IndexBuilderLight.DeleteIndex(changes.OldPath); + if (!IsExcludedFromIndex(changes.OldPath)) + { + IndexBuilderLight.DeleteIndex(changes.OldPath); + } } else { - if(IsFile(changes.FilePath)) + if (IsFile(changes.FilePath)) { IndexBuilderLight.RenameFileIndex(changes.OldPath, changes.FilePath); } else if (IsDirectory(changes.FilePath)) { - IndexBuilderLight.RenameFolderIndexes(changes.OldPath, changes.FilePath); + IndexBuilderLight.RenameFolderIndexes(changes.OldPath, changes.FilePath, TokenSource.Token); } } } @@ -178,10 +187,13 @@ void DeleteIndex(ChangedSource changes) void UpdateIndex(ChangedSource changes) { - throw new NotImplementedException(); + if (IsFile(changes.FilePath)) + { + IndexBuilderLight.UpdateIndex(new FileInfo(changes.FilePath), TokenSource.Token); + } } - void InitializeIndexCore(bool forceRebuild, CancellationToken cancellationToken) + void InitializeIndexCore(bool forceRebuild) { var folders = IndexConfig.GetFolders(CodeIndexConfiguration.LuceneIndex); @@ -220,14 +232,14 @@ void InitializeIndexCore(bool forceRebuild, CancellationToken cancellationToken) foreach (var codeSource in allCodeSource) { - cancellationToken.ThrowIfCancellationRequested(); + TokenSource.Token.ThrowIfCancellationRequested(); if (allFilesDictionary.TryGetValue(codeSource.FilePath, out var fileInfo)) { if (fileInfo.LastWriteTimeUtc != codeSource.LastWriteTimeUtc) { Log.Info($"{IndexConfig.IndexName}: File {fileInfo.FullName} modified"); - if (!IndexBuilderLight.UpdateIndex(fileInfo, cancellationToken)) + if (!IndexBuilderLight.UpdateIndex(fileInfo, TokenSource.Token)) { failedUpdateOrDeleteFiles.Add(codeSource.FilePath); } @@ -253,7 +265,7 @@ void InitializeIndexCore(bool forceRebuild, CancellationToken cancellationToken) } } - AddNewIndexFiles(needToBuildIndex ?? allFiles, out var failedIndexFiles, cancellationToken); + AddNewIndexFiles(needToBuildIndex ?? allFiles, out var failedIndexFiles); IndexBuilderLight.Commit(); @@ -267,34 +279,41 @@ void InitializeIndexCore(bool forceRebuild, CancellationToken cancellationToken) } } - void AddNewIndexFiles(IEnumerable needToBuildIndex, out ConcurrentBag failedIndexFiles, CancellationToken cancellationToken) + void AddNewIndexFiles(IEnumerable needToBuildIndex, out ConcurrentBag failedIndexFiles) { - failedIndexFiles = IndexBuilderLight.BuildIndexByBatch(needToBuildIndex, true, false, false, cancellationToken); + failedIndexFiles = IndexBuilderLight.BuildIndexByBatch(needToBuildIndex, true, false, false, TokenSource.Token); if (failedIndexFiles.Count > 0) { Log.Info($"{IndexConfig.IndexName}: Retry failed build indexes files, files count {failedIndexFiles.Count}"); - failedIndexFiles = IndexBuilderLight.BuildIndexByBatch(failedIndexFiles, true, false, false, cancellationToken); + failedIndexFiles = IndexBuilderLight.BuildIndexByBatch(failedIndexFiles, true, false, false, TokenSource.Token); } } void OnChange(object sender, FileSystemEventArgs e) { - ChangedSources.Enqueue(new ChangedSource + var changeSource = new ChangedSource { ChangesType = e.ChangeType, FilePath = e.FullPath - }); + }; + + if (!IsExcludedFromIndex(changeSource.FilePath)) + { + ChangedSources.Enqueue(changeSource); + } } void OnRename(object sender, RenamedEventArgs e) { - ChangedSources.Enqueue(new ChangedSource + var changeSource = new ChangedSource { ChangesType = e.ChangeType, FilePath = e.FullPath, OldPath = e.OldFullPath - }); + }; + + ChangedSources.Enqueue(changeSource); } public IndexConfig IndexConfig { get; } @@ -305,6 +324,7 @@ void OnRename(object sender, RenamedEventArgs e) public string Description { get; set; } public bool IsDisposing { get; private set; } FileSystemWatcher FilesWatcher { get; set; } + public CancellationTokenSource TokenSource { get; } ConcurrentQueue ChangedSources { get; set; } ConcurrentQueue PendingRetryCodeSources { get; set; } string[] ExcludedExtensions { get; } @@ -316,6 +336,8 @@ public void Dispose() if (!IsDisposing) { IsDisposing = true; + TokenSource.Cancel(); + TokenSource.Dispose(); FilesWatcher?.Dispose(); IndexBuilderLight?.Dispose(); } diff --git a/src/CodeIndex.Test/Search/CodeIndexSearcherTest.cs b/src/CodeIndex.Test/Search/CodeIndexSearcherTest.cs index 66df500..2c6f261 100644 --- a/src/CodeIndex.Test/Search/CodeIndexSearcherTest.cs +++ b/src/CodeIndex.Test/Search/CodeIndexSearcherTest.cs @@ -91,7 +91,8 @@ public void TestGenerateHtmlPreviewText() { result = CodeIndexSearcher.GenerateHtmlPreviewText(generator.GetQueryFromStr("ABC"), content, 10, LucenePool.GetAnalyzer()); Assert.AreEqual(@"My -Is A ...s in", result); +Is A ... +It's ", result); } }