diff --git a/src/CodeIndex.Common/CodeIndexConfiguration.cs b/src/CodeIndex.Common/CodeIndexConfiguration.cs index d9c783a..cc0e60c 100644 --- a/src/CodeIndex.Common/CodeIndexConfiguration.cs +++ b/src/CodeIndex.Common/CodeIndexConfiguration.cs @@ -6,6 +6,10 @@ namespace CodeIndex.Common public class CodeIndexConfiguration { public const char SplitChar = '|'; + public const string CodeIndexesFolder = "CodeIndexes"; + public const string ConfigurationIndexFolder = "Configuration"; + public const string CodeIndexFolder = "CodeIndex"; + public const string HintIndexFolder = "HintIndex"; public string LuceneIndex { get; set; } = string.Empty; public string MonitorFolder { get; set; } = string.Empty; @@ -15,8 +19,8 @@ public class CodeIndexConfiguration public int SaveIntervalSeconds { get; set; } = 300; public string LocalUrl { get; set; } = string.Empty; public string MonitorFolderRealPath { get; set; } = string.Empty; - public string LuceneIndexForCode => luceneIndexForCode ??= GetIndexPath("CodeIndex"); - public string LuceneIndexForHint => luceneIndexForHint ??= GetIndexPath("HintIndex"); + public string LuceneIndexForCode => luceneIndexForCode ??= GetIndexPath(CodeIndexFolder); + public string LuceneIndexForHint => luceneIndexForHint ??= GetIndexPath(HintIndexFolder); public string ExcludedExtensions { get; set; } = string.Empty; public string ExcludedPaths { get; set; } = string.Empty; public string IncludedExtensions { get; set; } = string.Empty; @@ -56,5 +60,8 @@ string[] GetSplitStringArray(string excludedExtensions) return excludedExtensions.Split(SplitChar, StringSplitOptions.RemoveEmptyEntries); } + + string luceneConfigurationIndex; + public string LuceneConfigurationIndex => luceneConfigurationIndex ??= GetIndexPath(ConfigurationIndexFolder); } } diff --git a/src/CodeIndex.Common/IndexConfig.cs b/src/CodeIndex.Common/IndexConfig.cs index ea8e5e9..ce1ad2d 100644 --- a/src/CodeIndex.Common/IndexConfig.cs +++ b/src/CodeIndex.Common/IndexConfig.cs @@ -1,21 +1,78 @@ using System; -using System.Collections.Generic; +using System.IO; namespace CodeIndex.Common { public class IndexConfig { + public const char SplitChar = '|'; + public Guid Pk { get; set; } public string IndexName { get; set; } public string MonitorFolder { get; set; } - public IEnumerable IncludedExtensions { get; set; } - public IEnumerable ExcludedExtensions { get; set; } public int MaxContentHighlightLength { get; set; } - public IEnumerable ExcludedPaths { get; set; } public int SaveIntervalSeconds { get; set; } public string OpenIDEUriFormat { get; set; } public string MonitorFolderRealPath { get; set; } public DateTime IndexCreatedDate { get; set; } public DateTime IndexLastUpdatedDate { get; set; } + + public string ExcludedPaths + { + get => excludedPaths; + set + { + excludedPaths = value; + excludedPathsArray = null; + } + } + + public string IncludedExtensions + { + get => includedExtensions; + set + { + includedExtensions = value; + includedExtensionsArray = null; + } + } + + public string ExcludedExtensions + { + get => excludedExtensions; + set + { + excludedExtensions = value; + excludedExtensionsArray = null; + } + } + + public string[] ExcludedPathsArray => excludedPathsArray ??= GetSplitStringArray(ExcludedPaths); + + public string[] IncludedExtensionsArray => includedExtensionsArray ??= GetSplitStringArray(IncludedExtensions); + + public string[] ExcludedExtensionsArray => excludedExtensionsArray ??= GetSplitStringArray(ExcludedExtensions); + + public (string CodeIndexFolder,string HintIndexFolder) GetFolders(string parentFolder) + { + return (Path.Combine(parentFolder, IndexName, CodeIndexConfiguration.CodeIndexFolder), Path.Combine(parentFolder, IndexName, CodeIndexConfiguration.HintIndexFolder)); + } + + string[] GetSplitStringArray(string value) + { + if (string.IsNullOrEmpty(value)) + { + return Array.Empty(); + } + + return value.Split(SplitChar, StringSplitOptions.RemoveEmptyEntries); + } + + string[] excludedPathsArray; + string[] includedExtensionsArray; + string[] excludedExtensionsArray; + string excludedPaths; + string includedExtensions; + string excludedExtensions; } } diff --git a/src/CodeIndex.Common/IndexStatus.cs b/src/CodeIndex.Common/IndexStatus.cs index 746d268..ac152d3 100644 --- a/src/CodeIndex.Common/IndexStatus.cs +++ b/src/CodeIndex.Common/IndexStatus.cs @@ -2,9 +2,11 @@ { public enum IndexStatus { - Created, + Idle, Initializing, Monitoring, - Deleting + Error, + Deleting, + Initialized } } diff --git a/src/CodeIndex.ConsoleApp/Program.cs b/src/CodeIndex.ConsoleApp/Program.cs index b9c8fc5..4001d2d 100644 --- a/src/CodeIndex.ConsoleApp/Program.cs +++ b/src/CodeIndex.ConsoleApp/Program.cs @@ -23,7 +23,7 @@ static void Main(string[] args) var maintainer = new CodeFilesIndexMaintainer(config, logger); maintainer.StartWatch(); initializer.InitializeIndex(config, out var failedIndexFiles); - maintainer.SetInitalizeFinishedToTrue(failedIndexFiles); + maintainer.SetInitializeFinishedToTrue(failedIndexFiles); Console.WriteLine("Initialize complete"); diff --git a/src/CodeIndex.IndexBuilder/CodeIndexBuilderLight.cs b/src/CodeIndex.IndexBuilder/CodeIndexBuilderLight.cs new file mode 100644 index 0000000..f63d01d --- /dev/null +++ b/src/CodeIndex.IndexBuilder/CodeIndexBuilderLight.cs @@ -0,0 +1,216 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using CodeIndex.Common; +using CodeIndex.Files; +using Lucene.Net.Documents; +using Lucene.Net.Index; +using Lucene.Net.Search; + +namespace CodeIndex.IndexBuilder +{ + public class CodeIndexBuilderLight : IDisposable + { + public CodeIndexBuilderLight(string name, LucenePoolLight codeIndexPool, LucenePoolLight hintIndexPool, ILog log) + { + name.RequireNotNullOrEmpty(nameof(name)); + codeIndexPool.RequireNotNull(nameof(codeIndexPool)); + hintIndexPool.RequireNotNull(nameof(hintIndexPool)); + log.RequireNotNull(nameof(log)); + + Name = name; + CodeIndexPool = codeIndexPool; + HintIndexPool = hintIndexPool; + Log = log; + } + + public string Name { get; } + public LucenePoolLight CodeIndexPool { get; } + public LucenePoolLight HintIndexPool { get; } + public ILog Log { get; } + + public void InitIndexFolderIfNeeded() + { + if (!Directory.Exists(CodeIndexPool.LuceneIndex)) + { + Log.Info($"Create {Name} index folder {CodeIndexPool.LuceneIndex}"); + Directory.CreateDirectory(CodeIndexPool.LuceneIndex); + } + + if (!Directory.Exists(HintIndexPool.LuceneIndex)) + { + Log.Info($"Create {Name} index folder {HintIndexPool.LuceneIndex}"); + Directory.CreateDirectory(HintIndexPool.LuceneIndex); + } + } + + public void BuildIndexByBatch(IEnumerable fileInfos, out List failedIndexFiles, bool needCommit, bool triggerMerge, bool applyAllDeletes, CancellationToken cancellationToken, int batchSize = 10000) + { + fileInfos.RequireNotNull(nameof(fileInfos)); + batchSize.RequireRange(nameof(batchSize), int.MaxValue, 50); + + var codeDocuments = new List(); + var hintWords = new List(); + failedIndexFiles = new List(); + + foreach (var fileInfo in fileInfos) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + if (fileInfo.Exists) + { + var source = CodeSource.GetCodeSource(fileInfo, FilesContentHelper.ReadAllText(fileInfo.FullName)); + + var words = WordSegmenter.GetWords(source.Content).Where(word => word.Length > 3 && word.Length < 200); + hintWords.AddRange(words); + + var doc = CodeIndexBuilder.GetDocumentFromSource(source); + codeDocuments.Add(doc); + + Log.Info($"{Name}: Add index For {source.FilePath}"); + } + } + catch (Exception ex) + { + failedIndexFiles.Add(fileInfo); + Log.Error($"{Name}: Add index for {fileInfo.FullName} failed, exception: " + ex); + } + + if (codeDocuments.Count >= batchSize) + { + BuildIndex(needCommit, triggerMerge, applyAllDeletes, codeDocuments, hintWords, cancellationToken); + codeDocuments.Clear(); + hintWords.Clear(); + } + } + + if (codeDocuments.Count > 0) + { + BuildIndex(needCommit, triggerMerge, applyAllDeletes, codeDocuments, hintWords, cancellationToken); + } + } + + public void DeleteAllIndex() + { + Log.Info($"{Name}: Delete All Index start"); + CodeIndexPool.DeleteAllIndex(); + HintIndexPool.DeleteAllIndex(); + Log.Info($"{Name}: Delete All Index finished"); + } + + public IEnumerable<(string FilePath, DateTime LastWriteTimeUtc)> GetAllIndexedCodeSource() + { + return CodeIndexPool.Search(new MatchAllDocsQuery(), int.MaxValue).Select(u => (u.Get(nameof(CodeSource.FilePath)), new DateTime(long.Parse(u.Get(nameof(CodeSource.LastWriteTimeUtc)))))).ToList(); + } + + void BuildIndex(bool needCommit, bool triggerMerge, bool applyAllDeletes, List codeDocuments, List words, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + Log.Info($"{Name}: Build code index start, documents count {codeDocuments.Count}"); + CodeIndexPool.BuildIndex(codeDocuments, needCommit, triggerMerge, applyAllDeletes); + Log.Info($"{Name}: Build code index finished"); + + Log.Info($"{Name}: Build hint index start, documents count {words.Count}"); + words.ForEach(word => + { + cancellationToken.ThrowIfCancellationRequested(); + + HintIndexPool.UpdateIndex(new Term(nameof(CodeWord.Word), word), new Document + { + new StringField(nameof(CodeWord.Word), word, Field.Store.YES), + new StringField(nameof(CodeWord.WordLower), word.ToLowerInvariant(), Field.Store.YES) + }); + }); + + if (needCommit || triggerMerge || applyAllDeletes) + { + HintIndexPool.Commit(); + } + + Log.Info($"{Name}: Build hint index finished"); + } + + public bool IsDisposing { get; private set; } + + public void Dispose() + { + if (!IsDisposing) + { + IsDisposing = true; + CodeIndexPool.Dispose(); + HintIndexPool.Dispose(); + } + } + + public bool UpdateIndex(FileInfo fileInfo, CancellationToken cancellationToken) + { + try + { + if (fileInfo.Exists) + { + var source = CodeSource.GetCodeSource(fileInfo, FilesContentHelper.ReadAllText(fileInfo.FullName)); + var words = WordSegmenter.GetWords(source.Content).Where(word => word.Length > 3 && word.Length < 200).ToList(); + var doc = CodeIndexBuilder.GetDocumentFromSource(source); + CodeIndexPool.UpdateIndex(GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), source.FilePath), doc); + words.ForEach(word => + { + cancellationToken.ThrowIfCancellationRequested(); + + HintIndexPool.UpdateIndex(new Term(nameof(CodeWord.Word), word), new Document + { + new StringField(nameof(CodeWord.Word), word, Field.Store.YES), + new StringField(nameof(CodeWord.WordLower), word.ToLowerInvariant(), Field.Store.YES) + }); + }); + + Log.Info($"{Name}: Update index For {source.FilePath} finished"); + } + + return true; + } + catch (Exception ex) + { + Log.Error($"{Name}: Update index for {fileInfo.FullName} failed, exception: " + ex); + + if (ex is OperationCanceledException) + { + throw; + } + + return false; + } + } + + public bool DeleteIndex(string filePath) + { + try + { + CodeIndexPool.DeleteIndex(GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), filePath)); + Log.Info($"{Name}: Delete index For {filePath} finished"); + + return true; + } + catch (Exception ex) + { + Log.Error($"{Name}: Delete index for {filePath} failed, exception: " + ex); + return false; + } + } + + public void Commit() + { + CodeIndexPool.Commit(); + HintIndexPool.Commit(); + } + + public Term GetNoneTokenizeFieldTerm(string fieldName, string termValue) + { + return new Term($"{fieldName}{Constants.NoneTokenizeFieldSuffix}", termValue); + } + } +} diff --git a/src/CodeIndex.IndexBuilder/ILucenePool.cs b/src/CodeIndex.IndexBuilder/ILucenePool.cs new file mode 100644 index 0000000..3b2705a --- /dev/null +++ b/src/CodeIndex.IndexBuilder/ILucenePool.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using Lucene.Net.Analysis; +using Lucene.Net.Documents; +using Lucene.Net.Index; +using Lucene.Net.Search; + +namespace CodeIndex.IndexBuilder +{ + public interface ILucenePool : IDisposable + { + void BuildIndex(IEnumerable documents, bool needCommit, bool triggerMerge = false, bool applyAllDeletes = false); + + Document[] Search(Query query, int maxResults, Filter filter = null); + + void DeleteIndex(params Query[] searchQueries); + + void DeleteIndex(params Term[] terms); + + void UpdateIndex(Term term, Document document); + + void DeleteAllIndex(); + + string LuceneIndex { get; } + + Analyzer Analyzer { get; } + } +} diff --git a/src/CodeIndex.IndexBuilder/IndexTypeAttribute.cs b/src/CodeIndex.IndexBuilder/IndexTypeAttribute.cs new file mode 100644 index 0000000..dd03022 --- /dev/null +++ b/src/CodeIndex.IndexBuilder/IndexTypeAttribute.cs @@ -0,0 +1,24 @@ +using System; + +namespace CodeIndex.IndexBuilder +{ + [AttributeUsage(AttributeTargets.Property)] + public class IndexTypeAttribute : Attribute + { + public IndexTypeAttribute(IndexTypes indexTypes) + { + IndexType = indexTypes; + } + + public IndexTypes IndexType { get; } + } + + public enum IndexTypes + { + Default, + StringType, + TextType, + StoredType, + Number + } +} diff --git a/src/CodeIndex.IndexBuilder/LucenePool.cs b/src/CodeIndex.IndexBuilder/LucenePool.cs index 1aa69d3..c699561 100644 --- a/src/CodeIndex.IndexBuilder/LucenePool.cs +++ b/src/CodeIndex.IndexBuilder/LucenePool.cs @@ -159,9 +159,7 @@ public static void SaveResultsAndClearLucenePool(string luceneIndex) static IndexWriter CreateOrGetIndexWriter(string luceneIndex) { - IndexWriter indexWriter; - - if (!IndexWritesPool.TryGetValue(luceneIndex, out indexWriter)) + if (!IndexWritesPool.TryGetValue(luceneIndex, out var indexWriter)) { lock (syncLockForWriter) { @@ -185,9 +183,7 @@ static IndexWriter CreateOrGetIndexWriter(string luceneIndex) static IndexSearcher CreateOrGetIndexSearcher(string luceneIndex) { - IndexSearcher indexSearcher; - - if (!IndexSearcherPool.TryGetValue(luceneIndex, out indexSearcher) || IndexGotChanged.TryGetValue(luceneIndex, out var indexChangedTimes) && indexChangedTimes > 0) + if (!IndexSearcherPool.TryGetValue(luceneIndex, out IndexSearcher indexSearcher) || IndexGotChanged.TryGetValue(luceneIndex, out var indexChangedTimes) && indexChangedTimes > 0) { lock (syncLockForSearcher) { @@ -213,7 +209,7 @@ static IndexSearcher CreateOrGetIndexSearcher(string luceneIndex) return indexSearcher; } - static Document[] SearchDocuments(string luceneIndex, Query query, int maxResult, FieldValueFilter fieldValueFilter = null) + static Document[] SearchDocuments(string luceneIndex, Query query, int maxResult, Filter filter = null) { Document[] documents = null; IndexSearcher indexSearcher = null; @@ -222,9 +218,9 @@ static Document[] SearchDocuments(string luceneIndex, Query query, int maxResult { indexSearcher = CreateOrGetIndexSearcher(luceneIndex); - if (fieldValueFilter != null) + if (filter != null) { - documents = indexSearcher.Search(query, fieldValueFilter, maxResult).ScoreDocs.Select(hit => indexSearcher.Doc(hit.Doc)).ToArray(); + documents = indexSearcher.Search(query, filter, maxResult).ScoreDocs.Select(hit => indexSearcher.Doc(hit.Doc)).ToArray(); } else { @@ -243,9 +239,7 @@ static Document[] SearchDocuments(string luceneIndex, Query query, int maxResult static IndexReader CreateOrGetIndexReader(string luceneIndex, bool forceRefresh) { - IndexReader indexReader; - - if (!IndexReaderPool.TryGetValue(luceneIndex, out indexReader) || forceRefresh) + if (!IndexReaderPool.TryGetValue(luceneIndex, out var indexReader) || forceRefresh) { lock (syncLockForReader) { @@ -266,13 +260,13 @@ static IndexReader CreateOrGetIndexReader(string luceneIndex, bool forceRefresh) return indexReader; } - public static Document[] Search(string luceneIndex, Query query, int maxResults) + public static Document[] Search(string luceneIndex, Query query, int maxResults, Filter filter = null) { readWriteLock.TryEnterReadLock(Constants.ReadWriteLockTimeOutMilliseconds); try { - return SearchDocuments(luceneIndex, query, maxResults); + return SearchDocuments(luceneIndex, query, maxResults, filter); } finally { diff --git a/src/CodeIndex.IndexBuilder/LucenePoolLight.cs b/src/CodeIndex.IndexBuilder/LucenePoolLight.cs new file mode 100644 index 0000000..fa09556 --- /dev/null +++ b/src/CodeIndex.IndexBuilder/LucenePoolLight.cs @@ -0,0 +1,251 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using CodeIndex.Common; +using Lucene.Net.Analysis; +using Lucene.Net.Documents; +using Lucene.Net.Index; +using Lucene.Net.Search; +using Lucene.Net.Store; + +namespace CodeIndex.IndexBuilder +{ + public class LucenePoolLight : ILucenePool + { + public LucenePoolLight(string luceneIndex) + { + luceneIndex.RequireNotNullOrEmpty(nameof(luceneIndex)); + LuceneIndex = luceneIndex; + } + + #region ILucenePool + + public string LuceneIndex { get; } + + public void BuildIndex(IEnumerable documents, bool needCommit, bool triggerMerge = false, bool applyAllDeletes = false) + { + using var readLock = new EnterReaderWriterLock(readerWriteLock); + IndexWriter.AddDocuments(documents); + + if (triggerMerge || applyAllDeletes) + { + IndexWriter.Flush(triggerMerge, applyAllDeletes); + } + + if (needCommit) + { + IndexWriter.Commit(); + } + + indexChangeCount++; + } + + public void DeleteIndex(params Query[] searchQueries) + { + using var readLock = new EnterReaderWriterLock(readerWriteLock); + IndexWriter.DeleteDocuments(searchQueries); + + indexChangeCount++; + } + + public void DeleteIndex(params Term[] terms) + { + using var readLock = new EnterReaderWriterLock(readerWriteLock); + IndexWriter.DeleteDocuments(terms); + + indexChangeCount++; + } + + public void Dispose() + { + using var readLock = new EnterReaderWriterLock(readerWriteLock, false); + + if (!isDisposing) + { + isDisposing = true; + indexChangeCount = 0; + indexReader?.Dispose(); + indexWriter?.Dispose(); + } + } + + public Document[] Search(Query query, int maxResults, Filter filter = null) + { + using var readLock = new EnterReaderWriterLock(readerWriteLock); + Document[] documents; + var searcher = GetIndexSearcher(); + + try + { + if (filter != null) + { + documents = searcher.Search(query, filter, maxResults).ScoreDocs.Select(hit => searcher.Doc(hit.Doc)).ToArray(); + } + else + { + documents = searcher.Search(query, maxResults).ScoreDocs.Select(hit => searcher.Doc(hit.Doc)).ToArray(); + } + } + finally + { + searcher.IndexReader.DecRef(); // Dispose Safe + } + + return documents; + } + + public Analyzer Analyzer => analyzer ??= new CodeAnalyzer(Constants.AppLuceneVersion, true); + + #endregion + + #region Fields + + readonly ReaderWriterLockSlim readerWriteLock = new ReaderWriterLockSlim(); + int indexChangeCount; + bool isDisposing; + CodeAnalyzer analyzer; + + #endregion + + #region IndexWriter + + readonly object syncLockForWriter = new object(); + IndexWriter indexWriter; + IndexWriter IndexWriter + { + get + { + if (indexWriter == null) + { + lock (syncLockForWriter) + { + var dir = FSDirectory.Open(LuceneIndex); + //create an analyzer to process the text + //create an index writer + var indexConfig = new IndexWriterConfig(Constants.AppLuceneVersion, Analyzer); + indexWriter = new IndexWriter(dir, indexConfig); + } + } + + return indexWriter; + } + } + + #endregion + + #region IndexSearcher + + readonly object syncLockForSearcher = new object(); + IndexSearcher indexSearcher; + IndexSearcher GetIndexSearcher() + { + if (indexSearcher == null || indexChangeCount > 0) + { + lock (syncLockForSearcher) + { + indexSearcher = new IndexSearcher(IndexReader); + } + } + + if (!indexSearcher.IndexReader.TryIncRef()) + { + return GetIndexSearcher(); // try get the IndexSearcher again + } + + return indexSearcher; + } + + public void DeleteAllIndex() + { + using var readLock = new EnterReaderWriterLock(readerWriteLock); + + IndexWriter.DeleteAll(); + indexWriter.Commit(); + + indexChangeCount++; + } + + public void Commit() + { + using var readLock = new EnterReaderWriterLock(readerWriteLock); + indexWriter.Commit(); + } + + public void UpdateIndex(Term term, Document document) + { + using var readLock = new EnterReaderWriterLock(readerWriteLock); + + IndexWriter.UpdateDocument(term, document); + indexChangeCount++; + } + + #endregion + + #region IndexReader + + readonly object syncLockForReader = new object(); + IndexReader indexReader; + IndexReader IndexReader + { + get + { + if (indexReader == null || indexChangeCount > 0) + { + lock (syncLockForReader) + { + if (indexReader == null) + { + indexReader = IndexWriter.GetReader(true); + } + else + { + indexReader.DecRef(); // Dispose safely + indexReader = IndexWriter.GetReader(true); + } + + indexChangeCount = 0; + } + } + + return indexReader; + } + } + + #endregion + } + + class EnterReaderWriterLock : IDisposable + { + public EnterReaderWriterLock(ReaderWriterLockSlim readerWriterLock, bool enterReadLock = true) + { + ReaderWriterLock = readerWriterLock; + EnterReadLock = enterReadLock; + + if (EnterReadLock) + { + ReaderWriterLock.TryEnterReadLock(Constants.ReadWriteLockTimeOutMilliseconds); + } + else + { + ReaderWriterLock.TryEnterWriteLock(Constants.ReadWriteLockTimeOutMilliseconds); + } + } + + bool EnterReadLock { get; } + + ReaderWriterLockSlim ReaderWriterLock { get; } + + public void Dispose() + { + if (EnterReadLock) + { + ReaderWriterLock.ExitReadLock(); + } + else + { + ReaderWriterLock.ExitWriteLock(); + } + } + } +} diff --git a/src/CodeIndex.MaintainIndex/ChangedSource.cs b/src/CodeIndex.MaintainIndex/ChangedSource.cs new file mode 100644 index 0000000..f963594 --- /dev/null +++ b/src/CodeIndex.MaintainIndex/ChangedSource.cs @@ -0,0 +1,13 @@ +using System; +using System.IO; + +namespace CodeIndex.MaintainIndex +{ + public class ChangedSource + { + public string FilePath { get; set; } + public string OldPath { get; set; } + public WatcherChangeTypes ChangesType { get; set; } + public DateTime ChangedUTCDate { get; } = DateTime.UtcNow; + } +} diff --git a/src/CodeIndex.MaintainIndex/CodeFilesIndexMaintainer.cs b/src/CodeIndex.MaintainIndex/CodeFilesIndexMaintainer.cs index 96167be..784fcef 100644 --- a/src/CodeIndex.MaintainIndex/CodeFilesIndexMaintainer.cs +++ b/src/CodeIndex.MaintainIndex/CodeFilesIndexMaintainer.cs @@ -31,13 +31,13 @@ public CodeFilesIndexMaintainer(CodeIndexConfiguration config, ILog log) tokenSource = new CancellationTokenSource(); } - public void SetInitalizeFinishedToTrue(List initalizeFailedFiles = null) + public void SetInitializeFinishedToTrue(List initializeFailedFiles = null) { - if (initalizeFailedFiles?.Count > 0) + if (initializeFailedFiles?.Count > 0) { var retryDate = DateTime.UtcNow.AddDays(-1); - foreach (var failedFiles in initalizeFailedFiles) + foreach (var failedFiles in initializeFailedFiles) { pendingRetryCodeSources.Enqueue(new PendingRetrySource { @@ -81,13 +81,13 @@ public void Dispose() FileSystemWatcher FileSystemWatcher { get; set; } const int Wait100Milliseconds = 100; - CodeIndexConfiguration config; - string[] excludedExtensions; - string[] excludedPaths; - int saveIntervalSeconds; - string[] includedExtensions; - ILog log; - CancellationTokenSource tokenSource; + readonly CodeIndexConfiguration config; + readonly string[] excludedExtensions; + readonly string[] excludedPaths; + readonly int saveIntervalSeconds; + readonly string[] includedExtensions; + readonly ILog log; + readonly CancellationTokenSource tokenSource; void OnFileChange(object sender, FileSystemEventArgs e) { @@ -355,7 +355,7 @@ void RetryAllFailed(CancellationToken cancellationToken) { if (pendingRetrySource.RetryTimes <= 10) // Always Failed, Stop Retry { - log?.Info($"Retry failed - ChangesType: {pendingRetrySource.ChangesType} FilePath:{pendingRetrySource.FilePath} LastRetryUTCDate: {pendingRetrySource.LastRetryUTCDate.ToString("yyyyMMddHHmmssfff")} OldPath: {pendingRetrySource.OldPath} RetryTimes: {pendingRetrySource.RetryTimes}"); + log?.Info($"Retry failed - ChangesType: {pendingRetrySource.ChangesType} FilePath:{pendingRetrySource.FilePath} LastRetryUTCDate: {pendingRetrySource.LastRetryUTCDate:yyyyMMddHHmmssfff} OldPath: {pendingRetrySource.OldPath} RetryTimes: {pendingRetrySource.RetryTimes}"); Task.Run(() => { @@ -382,7 +382,7 @@ void RetryAllFailed(CancellationToken cancellationToken) } else { - log?.Warn($"Stop retry failed - ChangesType: {pendingRetrySource.ChangesType} FilePath:{pendingRetrySource.FilePath} LastRetryUTCDate: {pendingRetrySource.LastRetryUTCDate.ToString("yyyyMMddHHmmssfff")} OldPath: {pendingRetrySource.OldPath} RetryTimes: {pendingRetrySource.RetryTimes}"); + log?.Warn($"Stop retry failed - ChangesType: {pendingRetrySource.ChangesType} FilePath:{pendingRetrySource.FilePath} LastRetryUTCDate: {pendingRetrySource.LastRetryUTCDate:yyyyMMddHHmmssfff} OldPath: {pendingRetrySource.OldPath} RetryTimes: {pendingRetrySource.RetryTimes}"); } } else diff --git a/src/CodeIndex.MaintainIndex/DocumentConverter.cs b/src/CodeIndex.MaintainIndex/DocumentConverter.cs new file mode 100644 index 0000000..066834e --- /dev/null +++ b/src/CodeIndex.MaintainIndex/DocumentConverter.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using CodeIndex.Common; +using Lucene.Net.Documents; + +namespace CodeIndex.MaintainIndex +{ + public static class DocumentConverter + { + static readonly ConcurrentDictionary propertiesDictionary = new ConcurrentDictionary(); + + public static T GetObject(this Document document) where T : new() + { + var type = typeof(T); + var result = new T(); + + if (!propertiesDictionary.TryGetValue(type.FullName ?? type.Name, out var propertyInfos)) + { + propertyInfos = type.GetProperties(BindingFlags.Public | BindingFlags.Instance).Where(p => p.CanRead && p.CanWrite).ToArray(); + + propertiesDictionary.TryAdd(nameof(T), propertyInfos); + } + + foreach (var property in propertyInfos) + { + property.SetValue(result, GetValue(property, document)); + } + + return result; + } + + static object GetValue(PropertyInfo property, Document document) + { + var propertyType = property.PropertyType; + + var value = GetValue(propertyType, document.Get(property.Name)); + + if (value != null) + { + return value; + } + + if (propertyType.IsGenericType && propertyType.GetGenericTypeDefinition() == typeof(IEnumerable<>)) + { + var genericType = propertyType.GetGenericArguments().First(); + var instance = Activator.CreateInstance(typeof(List<>).MakeGenericType(genericType)); + var collectionValues = document.Get(property.Name).Split(CodeIndexConfiguration.SplitChar).Where(u => !string.IsNullOrEmpty(u)); + var method = instance.GetType().GetMethod("Add"); + + foreach (var sub in collectionValues) + { + var subValue = GetValue(genericType, sub); + + if (subValue == null) + { + throw new NotImplementedException($"Not able to set value for {property.Name}, type: {property.PropertyType}"); + } + + method?.Invoke(instance, new[] { subValue }); + } + + return instance; + } + + throw new NotImplementedException($"Not able to set value for {property.Name}, type: {property.PropertyType}"); + } + + static object GetValue(Type type, string value) + { + if (type == typeof(string)) + { + return value; + } + + if (type == typeof(int)) + { + return Convert.ToInt32(value); + } + + if (type == typeof(DateTime)) + { + return new DateTime(long.Parse(value)); + } + + if (type == typeof(Guid)) + { + return new Guid(value); + } + + if (type == typeof(double)) + { + return Convert.ToDouble(value); + } + + if (type == typeof(float)) + { + return Convert.ToSingle(value); + } + + return null; + } + } +} diff --git a/src/CodeIndex.MaintainIndex/IndexMaintainer.cs b/src/CodeIndex.MaintainIndex/IndexMaintainer.cs new file mode 100644 index 0000000..54730a4 --- /dev/null +++ b/src/CodeIndex.MaintainIndex/IndexMaintainer.cs @@ -0,0 +1,240 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using CodeIndex.Common; +using CodeIndex.Files; +using CodeIndex.IndexBuilder; + +namespace CodeIndex.MaintainIndex +{ + public class IndexMaintainer : IDisposable + { + public IndexMaintainer(IndexConfig indexConfig, CodeIndexConfiguration codeIndexConfiguration, ILog log) + { + indexConfig.RequireNotNull(nameof(indexConfig)); + codeIndexConfiguration.RequireNotNull(nameof(codeIndexConfiguration)); + log.RequireNotNull(nameof(log)); + + IndexConfig = indexConfig; + CodeIndexConfiguration = codeIndexConfiguration; + Log = log; + Status = IndexStatus.Idle; + + ExcludedExtensions = indexConfig.ExcludedExtensionsArray.Select(u => u.ToUpperInvariant()).ToArray(); + ExcludedPaths = FilePathHelper.GetPaths(indexConfig.ExcludedPathsArray, codeIndexConfiguration.IsInLinux); + IncludedExtensions = indexConfig.IncludedExtensionsArray?.Select(u => u.ToUpperInvariant()).ToArray() ?? Array.Empty(); + } + + public async Task InitializeIndex(bool forceRebuild, CancellationToken cancellationToken) + { + if (Status != IndexStatus.Idle) + { + return; + } + + try + { + Log.Info($"{IndexConfig.IndexName}: Start Initializing"); + Status = IndexStatus.Initializing; + + if (Directory.Exists(IndexConfig.MonitorFolder)) + { + await Task.Run(() => + { + InitializeIndexCore(forceRebuild, cancellationToken); + }, cancellationToken); + + Status = IndexStatus.Initialized; + } + else + { + Status = IndexStatus.Error; + Description = "Monitor Folder Not Exist"; + Log.Warn($"{IndexConfig.IndexName}: Initializing failed: {Description}"); + } + } + catch (Exception ex) + { + Status = IndexStatus.Error; + Description = ex.Message; + Log.Error($"{IndexConfig.IndexName}: Initializing failed: {ex}"); + } + } + + public async Task MaintainIndexes(CancellationToken cancellationToken) + { + if (Status != IndexStatus.Initialized) + { + return; + } + + await Task.Run(() => + { + MaintainIndexesCore(cancellationToken); + }, cancellationToken); + } + + void MaintainIndexesCore(CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + void InitializeIndexCore(bool forceRebuild, CancellationToken cancellationToken) + { + var folders = IndexConfig.GetFolders(CodeIndexConfiguration.LuceneIndex); + + IndexBuilderLight = new CodeIndexBuilderLight( + IndexConfig.IndexName, + new LucenePoolLight(folders.CodeIndexFolder), + new LucenePoolLight(folders.HintIndexFolder), + Log); + + IndexBuilderLight.InitIndexFolderIfNeeded(); + + ChangedSources = new ConcurrentBag(); + PendingRetryCodeSources = new ConcurrentQueue(); + FilesWatcher = FilesWatcherHelper.StartWatch(IndexConfig.MonitorFolder, OnChange, OnRename); + + var allFiles = FilesFetcher.FetchAllFiles(IndexConfig.MonitorFolder, IndexConfig.ExcludedExtensionsArray, IndexConfig.ExcludedPathsArray, includedExtensions: IndexConfig.IncludedExtensionsArray, isInLinux: CodeIndexConfiguration.IsInLinux).ToList(); + Log.Info($"{IndexConfig.IndexName}: Fetching {allFiles.Count} files need to indexing"); + + List needToBuildIndex = null; + List failedUpdateOrDeleteFiles = new List(); + + if (CodeIndexBuilder.IndexExists(IndexConfig.MonitorFolder)) + { + if (forceRebuild) + { + Log.Info($"{IndexConfig.IndexName}: Force rebuild all indexes"); + IndexBuilderLight.DeleteAllIndex(); + } + else + { + Log.Info($"{IndexConfig.IndexName}: Compare index difference"); + + var allCodeSource = IndexBuilderLight.GetAllIndexedCodeSource(); + needToBuildIndex = new List(); + var allFilesDictionary = allFiles.ToDictionary(u => u.FullName); + + foreach (var codeSource in allCodeSource) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (allFilesDictionary.TryGetValue(codeSource.FilePath, out var fileInfo)) + { + if (fileInfo.LastWriteTimeUtc != codeSource.LastWriteTimeUtc) + { + Log.Info($"{IndexConfig.IndexName}: File {fileInfo.FullName} modified"); + if (!IndexBuilderLight.UpdateIndex(fileInfo, cancellationToken)) + { + failedUpdateOrDeleteFiles.Add(codeSource.FilePath); + } + } + + allFilesDictionary.Remove(codeSource.FilePath); + } + else + { + Log.Info($"{IndexConfig.IndexName}: File {codeSource.FilePath} deleted"); + if (!IndexBuilderLight.DeleteIndex(codeSource.FilePath)) + { + failedUpdateOrDeleteFiles.Add(codeSource.FilePath); + } + } + } + + foreach (var needToCreateFiles in allFilesDictionary) + { + Log.Info($"{IndexConfig.IndexName}: Found new file {needToCreateFiles.Value.FullName}"); + needToBuildIndex.Add(needToCreateFiles.Value); + } + } + } + + AddNewIndexFiles(needToBuildIndex ?? allFiles, out var failedIndexFiles, cancellationToken); + + IndexBuilderLight.Commit(); + + if (failedIndexFiles.Count > 0) + { + Log.Warn($"{IndexConfig.IndexName}: Initialize finished for {IndexConfig.MonitorFolder}, failed with these file(s): {string.Join(", ", failedIndexFiles.Select(u => u.FullName).Concat(failedUpdateOrDeleteFiles))}"); + } + else + { + Log.Info($"{IndexConfig.IndexName}: Initialize finished for {IndexConfig.MonitorFolder}"); + } + } + + void AddNewIndexFiles(List needToBuildIndex, out List failedIndexFiles, CancellationToken cancellationToken) + { + IndexBuilderLight.BuildIndexByBatch(needToBuildIndex, out failedIndexFiles, true, false, false, cancellationToken); + + if (failedIndexFiles.Count > 0) + { + Log.Info($"{IndexConfig.IndexName}: Retry failed build indexes files, files count {failedIndexFiles.Count}"); + IndexBuilderLight.BuildIndexByBatch(failedIndexFiles, out failedIndexFiles, true, false, false, cancellationToken); + } + } + + void OnChange(object sender, FileSystemEventArgs e) + { + ChangedSources.Add(new ChangedSource + { + ChangesType = e.ChangeType, + FilePath = e.FullPath + }); + } + + void OnRename(object sender, RenamedEventArgs e) + { + ChangedSources.Add(new ChangedSource + { + ChangesType = e.ChangeType, + FilePath = e.FullPath, + OldPath = e.OldFullPath + }); + } + + public IndexConfig IndexConfig { get; } + public CodeIndexConfiguration CodeIndexConfiguration { get; } + public ILog Log { get; } + public IndexStatus Status { get; private set; } + public CodeIndexBuilderLight IndexBuilderLight { get; private set; } + public string Description { get; set; } + public bool IsDisposing { get; private set; } + FileSystemWatcher FilesWatcher { get; set; } + ConcurrentBag ChangedSources { get; set; } + ConcurrentQueue PendingRetryCodeSources { get; set; } + string[] ExcludedExtensions { get; } + string[] ExcludedPaths { get; } + string[] IncludedExtensions { get; } + + public void Dispose() + { + if (!IsDisposing) + { + IsDisposing = true; + FilesWatcher?.Dispose(); + IndexBuilderLight?.Dispose(); + } + } + + bool IsExcludedFromIndex(string fullPath) + { + var excluded = ExcludedPaths.Any(u => fullPath.ToUpperInvariant().Contains(u)) + || ExcludedExtensions.Any(u => fullPath.EndsWith(u, StringComparison.InvariantCultureIgnoreCase)) + || IncludedExtensions.Length > 0 && !IncludedExtensions.Any(u => fullPath.EndsWith(u, StringComparison.InvariantCultureIgnoreCase)); + + if (excluded) + { + Log.Debug($"{IndexConfig.IndexName}: {fullPath} is excluded from index"); + } + + return excluded; + } + } +} diff --git a/src/CodeIndex.MaintainIndex/PendingRetrySource.cs b/src/CodeIndex.MaintainIndex/PendingRetrySource.cs index b313ea4..a941ee7 100644 --- a/src/CodeIndex.MaintainIndex/PendingRetrySource.cs +++ b/src/CodeIndex.MaintainIndex/PendingRetrySource.cs @@ -3,12 +3,9 @@ namespace CodeIndex.MaintainIndex { - public class PendingRetrySource + public class PendingRetrySource : ChangedSource { - public string FilePath { get; set; } - public string OldPath { get; set; } public int RetryTimes { get; set; } public DateTime LastRetryUTCDate { get; set; } - public WatcherChangeTypes ChangesType { get; set; } } } \ No newline at end of file diff --git a/src/CodeIndex.Search/CodeIndexSearcher.cs b/src/CodeIndex.Search/CodeIndexSearcher.cs index 809d28f..9879561 100644 --- a/src/CodeIndex.Search/CodeIndexSearcher.cs +++ b/src/CodeIndex.Search/CodeIndexSearcher.cs @@ -33,9 +33,11 @@ public static string GenerateHtmlPreviewText(Query query, string text, int lengt var scorer = new QueryScorer(query); var formatter = new SimpleHTMLFormatter(HighLightPrefix, HighLightSuffix); - var highlighter = new Highlighter(formatter, scorer); - highlighter.TextFragmenter = new SimpleFragmenter(length); - highlighter.MaxDocCharsToAnalyze = maxContentHighlightLength; + var highlighter = new Highlighter(formatter, scorer) + { + TextFragmenter = new SimpleFragmenter(length), + MaxDocCharsToAnalyze = maxContentHighlightLength + }; var stream = analyzer.GetTokenStream(nameof(CodeSource.Content), new StringReader(text)); @@ -80,9 +82,11 @@ public static (string MatchedLineContent, int LineNumber)[] GeneratePreviewTextW var scorer = new QueryScorer(query); var formatter = new SimpleHTMLFormatter(HighLightPrefix, HighLightSuffix); - var highlighter = new Highlighter(formatter, scorer); - highlighter.TextFragmenter = new SimpleFragmenter(length); - highlighter.MaxDocCharsToAnalyze = maxContentHighlightLength; + var highlighter = new Highlighter(formatter, scorer) + { + TextFragmenter = new SimpleFragmenter(length), + MaxDocCharsToAnalyze = maxContentHighlightLength + }; var stream = analyzer.GetTokenStream(nameof(CodeSource.Content), new StringReader(text)); diff --git a/src/CodeIndex.Server/Pages/IndexManagement.razor b/src/CodeIndex.Server/Pages/IndexManagement.razor index 6953c10..211d44b 100644 --- a/src/CodeIndex.Server/Pages/IndexManagement.razor +++ b/src/CodeIndex.Server/Pages/IndexManagement.razor @@ -53,15 +53,15 @@
- +
- +
- +
@@ -77,18 +77,14 @@ public List IndexStatusInfos { get; set; } - public IndexStatusInfo IndexStatusInfoForEditOrAdd { get; set; } = new IndexStatusInfo(IndexStatus.Created, new IndexConfig()); - - public string IncludedExtensions { get; set; } = string.Empty; - public string ExcludedExtensions { get; set; } = string.Empty; - public string ExcludedPaths { get; set; } = string.Empty; + public IndexStatusInfo IndexStatusInfoForEditOrAdd { get; set; } = new IndexStatusInfo(IndexStatus.Idle, new IndexConfig()); protected override void OnInitialized() { base.OnInitialized(); IndexStatusInfos = new List{ - new IndexStatusInfo(IndexStatus.Created, new IndexConfig + new IndexStatusInfo(IndexStatus.Idle, new IndexConfig { IndexName = "Dummy1", MonitorFolder = "c:/dummy1" @@ -123,19 +119,11 @@ void SaveIndexInfo() { - IndexStatusInfoForEditOrAdd.IndexConfig.IncludedExtensions = IncludedExtensions.Split(CodeIndexConfiguration.SplitChar); - IndexStatusInfoForEditOrAdd.IndexConfig.ExcludedExtensions = ExcludedExtensions.Split(CodeIndexConfiguration.SplitChar); - IndexStatusInfoForEditOrAdd.IndexConfig.ExcludedPaths = ExcludedPaths.Split(CodeIndexConfiguration.SplitChar); - if (!IndexStatusInfos.Contains(IndexStatusInfoForEditOrAdd)) { IndexStatusInfos.Add(IndexStatusInfoForEditOrAdd); } - IndexStatusInfoForEditOrAdd = new IndexStatusInfo(IndexStatus.Created, new IndexConfig()); - - IncludedExtensions = string.Empty; - ExcludedExtensions = string.Empty; - ExcludedPaths = string.Empty; + IndexStatusInfoForEditOrAdd = new IndexStatusInfo(IndexStatus.Initializing, new IndexConfig()); } } diff --git a/src/CodeIndex.Server/Startup.cs b/src/CodeIndex.Server/Startup.cs index b7a6634..e6a041b 100644 --- a/src/CodeIndex.Server/Startup.cs +++ b/src/CodeIndex.Server/Startup.cs @@ -93,7 +93,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env, IHostApp maintainer.StartWatch(); initializer.InitializeIndex(config, out var failedIndexFiles); - maintainer.SetInitalizeFinishedToTrue(failedIndexFiles); + maintainer.SetInitializeFinishedToTrue(failedIndexFiles); } catch (Exception ex) { diff --git a/src/CodeIndex.Test/BaseTest.cs b/src/CodeIndex.Test/BaseTest.cs index 963b0f7..3d29e4c 100644 --- a/src/CodeIndex.Test/BaseTest.cs +++ b/src/CodeIndex.Test/BaseTest.cs @@ -1,18 +1,12 @@ -using System; -using System.IO; -using CodeIndex.Common; +using CodeIndex.Common; using CodeIndex.IndexBuilder; using CodeIndex.Search; using NUnit.Framework; namespace CodeIndex.Test { - public class BaseTest + public class BaseTest : BaseTestLight { - protected string TempDir { get; set; } - string TempIndexDir => Path.Combine(TempDir, "IndexFolder"); - public string MonitorFolder => Path.Combine(TempDir, "CodeFolder"); - public CodeIndexConfiguration Config => new CodeIndexConfiguration { LuceneIndex = TempIndexDir, @@ -23,33 +17,19 @@ public class BaseTest protected QueryGenerator Generator => generator ??= new QueryGenerator(); [SetUp] - protected virtual void SetUp() + protected override void SetUp() { - TempDir = Path.Combine(Path.GetTempPath(), "CodeIndex.Test_" + Guid.NewGuid()); - + base.SetUp(); WordsHintBuilder.Words.Clear(); - - var dir = new DirectoryInfo(TempDir); - if (!dir.Exists) - { - dir.Create(); - } } [TearDown] - protected virtual void TearDown() + protected override void TearDown() { LucenePool.SaveResultsAndClearLucenePool(Config); - WordsHintBuilder.Words.Clear(); - DeleteAllFilesInTempDir(TempDir); - } - - void DeleteAllFilesInTempDir(string srcPath) - { - var dir = new DirectoryInfo(srcPath); - dir.Delete(true); + base.TearDown(); } } } diff --git a/src/CodeIndex.Test/BaseTestLight.cs b/src/CodeIndex.Test/BaseTestLight.cs new file mode 100644 index 0000000..11d91ab --- /dev/null +++ b/src/CodeIndex.Test/BaseTestLight.cs @@ -0,0 +1,37 @@ +using System; +using System.IO; +using NUnit.Framework; + +namespace CodeIndex.Test +{ + public class BaseTestLight + { + protected string TempDir { get; set; } + protected string TempIndexDir => Path.Combine(TempDir, "IndexFolder"); + public string MonitorFolder => Path.Combine(TempDir, "CodeFolder"); + + [SetUp] + protected virtual void SetUp() + { + TempDir = Path.Combine(Path.GetTempPath(), "CodeIndex.Test_" + Guid.NewGuid()); + + var dir = new DirectoryInfo(TempDir); + if (!dir.Exists) + { + dir.Create(); + } + } + + [TearDown] + protected virtual void TearDown() + { + DeleteAllFilesInTempDir(TempDir); + } + + void DeleteAllFilesInTempDir(string srcPath) + { + var dir = new DirectoryInfo(srcPath); + dir.Delete(true); + } + } +} diff --git a/src/CodeIndex.Test/Common/IndexConfigTest.cs b/src/CodeIndex.Test/Common/IndexConfigTest.cs index 7d812d8..e4271b2 100644 --- a/src/CodeIndex.Test/Common/IndexConfigTest.cs +++ b/src/CodeIndex.Test/Common/IndexConfigTest.cs @@ -1,6 +1,7 @@ using CodeIndex.Common; using NUnit.Framework; using System; +using System.Linq; namespace CodeIndex.Test { @@ -12,9 +13,9 @@ public void TestConstructor() var pk = Guid.NewGuid(); var config = new IndexConfig { - ExcludedExtensions = new[] { "A" }, - ExcludedPaths = new[] { "B" }, - IncludedExtensions = new[] { "C" }, + ExcludedExtensions = "A|B|C", + ExcludedPaths = "B|C|D", + IncludedExtensions = "E|F", IndexCreatedDate = new DateTime(2020, 1, 1), IndexLastUpdatedDate = new DateTime(2020, 1, 2), IndexName = "ABC", @@ -26,9 +27,13 @@ public void TestConstructor() SaveIntervalSeconds = 10 }; - CollectionAssert.AreEquivalent(config.ExcludedExtensions, new[] { "A" }); - CollectionAssert.AreEquivalent(config.ExcludedPaths, new[] { "B" }); - CollectionAssert.AreEquivalent(config.IncludedExtensions, new[] { "C" }); + Assert.AreEqual("A|B|C", config.ExcludedExtensions); + Assert.AreEqual("B|C|D", config.ExcludedPaths); + Assert.AreEqual("E|F", config.IncludedExtensions); + Assert.AreEqual(new DateTime(2020, 1, 1), config.IndexCreatedDate); + CollectionAssert.AreEquivalent(config.ExcludedExtensionsArray, new[] { "A", "B", "C" }); + CollectionAssert.AreEquivalent(config.ExcludedPathsArray, new[] { "B", "C", "D" }); + CollectionAssert.AreEquivalent(config.IncludedExtensionsArray, new[] { "E", "F" }); Assert.AreEqual(new DateTime(2020, 1, 1), config.IndexCreatedDate); Assert.AreEqual(new DateTime(2020, 1, 2), config.IndexLastUpdatedDate); Assert.AreEqual("ABC", config.IndexName); @@ -38,6 +43,9 @@ public void TestConstructor() Assert.AreEqual("BBB", config.OpenIDEUriFormat); Assert.AreEqual(pk, config.Pk); Assert.AreEqual(10, config.SaveIntervalSeconds); + + config.IncludedExtensions = null; + Assert.AreEqual(0, config.IncludedExtensionsArray.Count()); } } } \ No newline at end of file diff --git a/src/CodeIndex.Test/Files/FilesWatcherHelperTest.cs b/src/CodeIndex.Test/Files/FilesWatcherHelperTest.cs index 67f8b42..a88eb6f 100644 --- a/src/CodeIndex.Test/Files/FilesWatcherHelperTest.cs +++ b/src/CodeIndex.Test/Files/FilesWatcherHelperTest.cs @@ -16,61 +16,60 @@ public void TestStartWatch() { var renameHit = 0; var changeHit = 0; - var waitMS = 10; + var waitMS = 100; Directory.CreateDirectory(Path.Combine(TempDir, "SubDir")); - using (var watcher = FilesWatcherHelper.StartWatch(TempDir, OnChangedHandler, OnRenameHandler)) - { - File.Create(Path.Combine(TempDir, "AAA.cs")).Close(); - Thread.Sleep(waitMS); - Assert.AreEqual(1, changeHit); - Assert.AreEqual(0, renameHit); + using var watcher = FilesWatcherHelper.StartWatch(TempDir, OnChangedHandler, OnRenameHandler); - File.AppendAllText(Path.Combine(TempDir, "AAA.cs"), "12345"); - Thread.Sleep(waitMS); - Assert.AreEqual(2, changeHit); - Assert.AreEqual(0, renameHit); + File.Create(Path.Combine(TempDir, "AAA.cs")).Close(); + Thread.Sleep(waitMS); + Assert.AreEqual(1, changeHit); + Assert.AreEqual(0, renameHit); - File.Move(Path.Combine(TempDir, "AAA.cs"), Path.Combine(TempDir, "BBB.cs")); - Thread.Sleep(waitMS); - Assert.AreEqual(2, changeHit); - Assert.AreEqual(1, renameHit); + File.AppendAllText(Path.Combine(TempDir, "AAA.cs"), "12345"); + Thread.Sleep(waitMS); + Assert.AreEqual(2, changeHit); + Assert.AreEqual(0, renameHit); - File.Delete(Path.Combine(TempDir, "BBB.cs")); - Thread.Sleep(waitMS); - Assert.AreEqual(3, changeHit); - Assert.AreEqual(1, renameHit); + File.Move(Path.Combine(TempDir, "AAA.cs"), Path.Combine(TempDir, "BBB.cs")); + Thread.Sleep(waitMS); + Assert.AreEqual(2, changeHit); + Assert.AreEqual(1, renameHit); - File.Create(Path.Combine(TempDir, "SubDir", "AAA.cs")).Close(); - Thread.Sleep(waitMS); - Assert.AreEqual(4, changeHit); - Assert.AreEqual(1, renameHit); + File.Delete(Path.Combine(TempDir, "BBB.cs")); + Thread.Sleep(waitMS); + Assert.AreEqual(3, changeHit); + Assert.AreEqual(1, renameHit); - File.AppendAllText(Path.Combine(TempDir, "SubDir", "AAA.cs"), "AA BB"); - Thread.Sleep(waitMS); - Assert.AreEqual(6, changeHit, "One for folder, one for file"); - Assert.AreEqual(1, renameHit); + File.Create(Path.Combine(TempDir, "SubDir", "AAA.cs")).Close(); + Thread.Sleep(waitMS); + Assert.AreEqual(5, changeHit); + Assert.AreEqual(1, renameHit); - Directory.Move(Path.Combine(TempDir, "SubDir"), Path.Combine(TempDir, "SubDir2")); - Thread.Sleep(waitMS); - Assert.AreEqual(6, changeHit); - Assert.AreEqual(2, renameHit); + File.AppendAllText(Path.Combine(TempDir, "SubDir", "AAA.cs"), "AA BB"); + Thread.Sleep(waitMS); + Assert.AreEqual(6, changeHit, "One for folder, one for file"); + Assert.AreEqual(1, renameHit); - Directory.CreateDirectory(Path.Combine(TempDir, "SubDir3")); - Thread.Sleep(waitMS); - Assert.AreEqual(7, changeHit); - Assert.AreEqual(2, renameHit); + Directory.Move(Path.Combine(TempDir, "SubDir"), Path.Combine(TempDir, "SubDir2")); + Thread.Sleep(waitMS); + Assert.AreEqual(6, changeHit); + Assert.AreEqual(2, renameHit); - File.Create(Path.Combine(TempDir, "CCCC")).Close(); - Thread.Sleep(waitMS); - Assert.AreEqual(8, changeHit); - Assert.AreEqual(2, renameHit); + Directory.CreateDirectory(Path.Combine(TempDir, "SubDir3")); + Thread.Sleep(waitMS); + Assert.AreEqual(7, changeHit); + Assert.AreEqual(2, renameHit); - File.SetLastAccessTime(Path.Combine(TempDir, "CCCC"), DateTime.Now.AddDays(1)); - Thread.Sleep(waitMS); - Assert.AreEqual(8, changeHit, "Do not watch last access time for file"); - Assert.AreEqual(2, renameHit); - } + File.Create(Path.Combine(TempDir, "CCCC")).Close(); + Thread.Sleep(waitMS); + Assert.AreEqual(8, changeHit); + Assert.AreEqual(2, renameHit); + + File.SetLastAccessTime(Path.Combine(TempDir, "CCCC"), DateTime.Now.AddDays(1)); + Thread.Sleep(waitMS); + Assert.AreEqual(8, changeHit, "Do not watch last access time for file"); + Assert.AreEqual(2, renameHit); void OnRenameHandler(object sender, RenamedEventArgs e) { diff --git a/src/CodeIndex.Test/IndexBuilder/LucenePoolLightTest.cs b/src/CodeIndex.Test/IndexBuilder/LucenePoolLightTest.cs new file mode 100644 index 0000000..b06c4cd --- /dev/null +++ b/src/CodeIndex.Test/IndexBuilder/LucenePoolLightTest.cs @@ -0,0 +1,179 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using CodeIndex.Common; +using CodeIndex.IndexBuilder; +using Lucene.Net.Documents; +using Lucene.Net.Index; +using Lucene.Net.Search; +using NUnit.Framework; + +namespace CodeIndex.Test +{ + public class LucenePoolLightTest : BaseTestLight + { + [Test] + public void TestSearch() + { + using var light = new LucenePoolLight(TempIndexDir); + + light.BuildIndex(new[] { + GetDocument(new CodeSource + { + FileName = "Dummy File 1", + FileExtension = "cs", + FilePath = @"C:\Dummy File 1.cs", + Content = "Test Content" + Environment.NewLine + "A New Line For Test" + }), + GetDocument(new CodeSource + { + FileName = "Dummy File 2", + FileExtension = "cs", + FilePath = @"C:\Dummy File 2.cs", + Content = "Test Content" + Environment.NewLine + "A New Line For Test" + })}, true, true, true); + + var documents = light.Search(new MatchAllDocsQuery(), int.MaxValue); + Assert.AreEqual(2, documents.Length); + + documents = light.Search(new TermQuery(new Term(nameof(CodeSource.FileName), "2")), int.MaxValue); + Assert.AreEqual(1, documents.Length); + } + + [Test] + public void TestDeleteIndex() + { + using var light = new LucenePoolLight(TempIndexDir); + + light.BuildIndex(new[] { + GetDocument(new CodeSource + { + FileName = "Dummy File 1", + FileExtension = "cs", + FilePath = @"C:\Dummy File 1.cs", + Content = "Test Content" + Environment.NewLine + "A New Line For Test" + }), + GetDocument(new CodeSource + { + FileName = "Dummy File 2", + FileExtension = "cs", + FilePath = @"C:\Dummy File 2.cs", + Content = "Test Content" + Environment.NewLine + "A New Line For Test" + })}, true, true, true); + + var documents = light.Search(new MatchAllDocsQuery(), int.MaxValue); + Assert.AreEqual(2, documents.Length); + + light.DeleteIndex(new TermQuery(new Term(nameof(CodeSource.FileName), "2"))); + documents = light.Search(new MatchAllDocsQuery(), int.MaxValue); + Assert.AreEqual(1, documents.Length); + + light.DeleteIndex(new Term(nameof(CodeSource.FileName), "1")); + documents = light.Search(new MatchAllDocsQuery(), int.MaxValue); + Assert.AreEqual(0, documents.Length); + } + + [Test] + public void TestAnalyzer() + { + using var light = new LucenePoolLight("Dummy"); + Assert.NotNull(light.Analyzer); + } + + [Test] + public void TestDeleteAllIndex() + { + using var light = new LucenePoolLight(TempIndexDir); + + light.BuildIndex(new[] { + GetDocument(new CodeSource + { + FileName = "Dummy File 1", + FileExtension = "cs", + FilePath = @"C:\Dummy File 1.cs", + Content = "Test Content" + Environment.NewLine + "A New Line For Test" + }), + GetDocument(new CodeSource + { + FileName = "Dummy File 2", + FileExtension = "cs", + FilePath = @"C:\Dummy File 2.cs", + Content = "Test Content" + Environment.NewLine + "A New Line For Test" + })}, true, true, true); + + var documents = light.Search(new MatchAllDocsQuery(), int.MaxValue); + Assert.AreEqual(2, documents.Length); + + light.DeleteAllIndex(); + documents = light.Search(new MatchAllDocsQuery(), int.MaxValue); + Assert.AreEqual(0, documents.Length); + } + + [Test] + [Timeout(60000)] + public void TestThreadSafeForIndexReader() + { + using var light = new LucenePoolLight(TempIndexDir); + + light.BuildIndex(new[] { + GetDocument(new CodeSource + { + FileName = "Dummy File 1", + FileExtension = "cs", + FilePath = @"C:\Dummy File 1.cs", + Content = "Test Content" + Environment.NewLine + "A New Line For Test" + }), + GetDocument(new CodeSource + { + FileName = "Dummy File 2", + FileExtension = "cs", + FilePath = @"C:\Dummy File 2.cs", + Content = "Test Content" + Environment.NewLine + "A New Line For Test" + }), + GetDocument(new CodeSource + { + FileName = "Dummy File 2", + FileExtension = "xml", + FilePath = @"C:\Dummy File.xml", + Content = "Test Content" + Environment.NewLine + "A New Line For Test" + }) + }, true, true, true); + + var taskList = new List(); + var taskNumber = 3; // set to larger one for real test + var addDocumentCount = 10; // set to larger one for real test + + for (var i = 0; i < taskNumber; i++) + { + taskList.Add(Task.Run(() => + { + for (var j = 0; j < addDocumentCount; j++) + { + if (j % 4 == 0) + { + light.BuildIndex(new[] { + GetDocument(new CodeSource + { + FileName = $"Dummy File 1 {i} {j}", + FileExtension = "cs", + FilePath = $@"C:\Dummy File 1 {i} {j}.cs", + Content = "Test Content" + Environment.NewLine + "A New Line For Test" + }) + }, true, true, true); + } + + light.Search(new MatchAllDocsQuery(), int.MaxValue); + light.Search(new MatchAllDocsQuery(), int.MaxValue); + } + })); + } + + Assert.DoesNotThrowAsync(async () => await Task.WhenAll(taskList)); + } + + Document GetDocument(CodeSource codeSource) + { + return CodeIndexBuilder.GetDocumentFromSource(codeSource); + } + } +} diff --git a/src/CodeIndex.Test/MaintainIndex/CodeFilesIndexMaintainerTest.cs b/src/CodeIndex.Test/MaintainIndex/CodeFilesIndexMaintainerTest.cs index ecacea0..e1d87e8 100644 --- a/src/CodeIndex.Test/MaintainIndex/CodeFilesIndexMaintainerTest.cs +++ b/src/CodeIndex.Test/MaintainIndex/CodeFilesIndexMaintainerTest.cs @@ -57,7 +57,7 @@ public void TestMaintainerIndex() using var maintainer = new CodeFilesIndexMaintainer(Config, new DummyLog()); maintainer.StartWatch(); - maintainer.SetInitalizeFinishedToTrue(); + maintainer.SetInitializeFinishedToTrue(); File.AppendAllText(fileAPath, "56789"); // Changed File.Delete(fileBPath); // Deleted @@ -98,7 +98,7 @@ public void TestMaintainerIndex_RetryFailed() Config.ExcludedExtensions = ".dll"; using var maintainer = new CodeFilesIndexMaintainerForTest(Config, new DummyLog()); maintainer.StartWatch(); - maintainer.SetInitalizeFinishedToTrue(); + maintainer.SetInitializeFinishedToTrue(); maintainer.PendingRetryCodeSources.Enqueue(new PendingRetrySource() { diff --git a/src/CodeIndex.Test/MaintainIndex/DocumentConverterTest.cs b/src/CodeIndex.Test/MaintainIndex/DocumentConverterTest.cs new file mode 100644 index 0000000..e457c41 --- /dev/null +++ b/src/CodeIndex.Test/MaintainIndex/DocumentConverterTest.cs @@ -0,0 +1,72 @@ +using System; +using System.Collections.Generic; +using CodeIndex.MaintainIndex; +using Lucene.Net.Documents; +using NUnit.Framework; + +namespace CodeIndex.Test +{ + public class DocumentConverterTest + { + [Test] + public void TestConvert() + { + var document = new Document + { + new StringField(nameof(DummyForTest.Pk), Guid.NewGuid().ToString(), Field.Store.YES), + new StringField(nameof(DummyForTest.AAA), "AAA", Field.Store.YES), + new StringField(nameof(DummyForTest.BBB), "32", Field.Store.YES), + new StringField(nameof(DummyForTest.CCC), "32.3", Field.Store.YES), + new StringField(nameof(DummyForTest.DDD), "120.0", Field.Store.YES), + new Int64Field(nameof(DummyForTest.EEE), DateTime.Now.Ticks, Field.Store.YES), + new StringField(nameof(DummyForTest.FFF), "A|B|C|D|E", Field.Store.YES), + new StringField(nameof(DummyForTest.ReadonlyProperty), "ReadonlyProperty", Field.Store.YES), + }; + + var dummyForTest = document.GetObject(); + Assert.AreNotEqual(Guid.Empty, dummyForTest.Pk); + Assert.AreEqual("AAA", dummyForTest.AAA); + Assert.AreEqual(32, dummyForTest.BBB); + Assert.AreEqual(32.3, dummyForTest.CCC); + Assert.AreEqual(120.0f, dummyForTest.DDD); + Assert.AreNotEqual(new DateTime(), dummyForTest.EEE); + CollectionAssert.AreEquivalent(new[] { "A", "B", "C", "D", "E" }, dummyForTest.FFF); + Assert.IsNull(dummyForTest.ReadonlyProperty, "Don't set readonly property"); + } + + [Test] + public void TestThrowException() + { + var document = new Document + { + new StringField(nameof(DummyForTest2.BlaBla), "10", Field.Store.YES), + new StringField(nameof(DummyForTest3.BlaBlaEnum), "32|12", Field.Store.YES), + }; + + Assert.Throws(() => document.GetObject()); + Assert.Throws(() => document.GetObject()); + } + + class DummyForTest + { + public Guid Pk { get; set; } + public string AAA { get; set; } + public int BBB { get; set; } + public double CCC { get; set; } + public float DDD { get; set; } + public DateTime EEE { get; set; } + public IEnumerable FFF { get; set; } + public string ReadonlyProperty { get; } + } + + class DummyForTest2 + { + public decimal BlaBla { get; set; } + } + + class DummyForTest3 + { + public IEnumerable BlaBlaEnum { get; set; } + } + } +}