From 0cf7218e784715243d559526d7390681e3c63cbf Mon Sep 17 00:00:00 2001 From: Paul Irwin Date: Mon, 12 Aug 2024 11:08:20 -0600 Subject: [PATCH] Restore fsync behavior in FSDirectory via P/Invoke, #933 (#938) * Restore fsync behavior in FSDirectory via P/Invoke This restores the commented-out fsync behavior in FSDirectory to help mitigate a performance regression in .NET 8. * Use System.IO.Directory.Exists to avoid caching exists status * Add unit test for ConcurrentHashSet.ExceptWith * Improve errors thrown by CreateFileW * Change FileSystemInfo use to string in IOUtils.Fsync * Change Debug.Assert to Debugging use * Lucene.Net.Index.TestIndexWriterOnJRECrash::TestNRTThreads_Mem(): Removed AwaitsFix attribute. The FSync implementation should fix this test. * Make ExceptWith atomic * Improve error handling if directory not found on Linux/macOS * Refactor interop methods into separate partial class files * Lucene.Net.Index.TestIndexWriterOnJRECrash::TestNRTThreads_Mem(): Added [Repeat(25)] attribute. * Lucene.Net.Index.TestIndexWriterOnJRECrash: Instead of using a temp file to pass the process ID to kill back to the original test process, open a socket and listen for the process ID to be written. * Synchronize access to stale files collection This is necessary to prevent race conditions, even though this code is not in the upstream Java code. A thread could try to add an item to the collection after it has been synced in `Sync` but before it is removed from the collection, then the file is removed from the collection, resulting in a missed sync. * Rename syncLock to m_syncLock * Lucene.Net.Index.TestIndexWriterOnJRECrash: Added try/finally block and refactored to ensure the TcpListener and Process are cleaned up at the end of each test iteration. This makes it run ~20% faster. * Refactor rename namespace to Lucene.Net.Native * Mark JRE crash test as [AwaitsFix] --------- Co-authored-by: Shad Storhaug --- .../Index/TestIndexWriterOnJRECrash.cs | 150 ++++++++++-------- .../Support/TestConcurrentHashSet.cs | 51 ++++++ src/Lucene.Net/Store/FSDirectory.cs | 146 ++++++++++------- src/Lucene.Net/Support/ConcurrentHashSet.cs | 87 ++++++++-- .../Support/IO/PosixFsyncSupport.cs | 85 ++++++++++ .../Support/IO/WindowsFsyncSupport.cs | 85 ++++++++++ .../Support/Native/Interop.MacOS.Constants.cs | 28 ++++ .../Support/Native/Interop.Posix.Close.cs | 31 ++++ .../Support/Native/Interop.Posix.Constants.cs | 31 ++++ .../Support/Native/Interop.Posix.Fcntl.cs | 32 ++++ .../Support/Native/Interop.Posix.Fsync.cs | 31 ++++ .../Support/Native/Interop.Posix.Open.cs | 31 ++++ .../Native/Interop.Win32.CloseHandle.cs | 32 ++++ .../Support/Native/Interop.Win32.Constants.cs | 43 +++++ .../Native/Interop.Win32.CreateFileW.cs | 40 +++++ .../Native/Interop.Win32.FlushFileBuffers.cs | 32 ++++ src/Lucene.Net/Util/IOUtils.cs | 57 ++++++- 17 files changed, 855 insertions(+), 137 deletions(-) create mode 100644 src/Lucene.Net.Tests/Support/TestConcurrentHashSet.cs create mode 100644 src/Lucene.Net/Support/IO/PosixFsyncSupport.cs create mode 100644 src/Lucene.Net/Support/IO/WindowsFsyncSupport.cs create mode 100644 src/Lucene.Net/Support/Native/Interop.MacOS.Constants.cs create mode 100644 src/Lucene.Net/Support/Native/Interop.Posix.Close.cs create mode 100644 src/Lucene.Net/Support/Native/Interop.Posix.Constants.cs create mode 100644 src/Lucene.Net/Support/Native/Interop.Posix.Fcntl.cs create mode 100644 src/Lucene.Net/Support/Native/Interop.Posix.Fsync.cs create mode 100644 src/Lucene.Net/Support/Native/Interop.Posix.Open.cs create mode 100644 src/Lucene.Net/Support/Native/Interop.Win32.CloseHandle.cs create mode 100644 src/Lucene.Net/Support/Native/Interop.Win32.Constants.cs create mode 100644 src/Lucene.Net/Support/Native/Interop.Win32.CreateFileW.cs create mode 100644 src/Lucene.Net/Support/Native/Interop.Win32.FlushFileBuffers.cs diff --git a/src/Lucene.Net.Tests/Index/TestIndexWriterOnJRECrash.cs b/src/Lucene.Net.Tests/Index/TestIndexWriterOnJRECrash.cs index 32e1f4c765..9056852124 100644 --- a/src/Lucene.Net.Tests/Index/TestIndexWriterOnJRECrash.cs +++ b/src/Lucene.Net.Tests/Index/TestIndexWriterOnJRECrash.cs @@ -4,15 +4,16 @@ using NUnit.Framework; using RandomizedTesting.Generators; using System; -using System.Data; using System.Diagnostics; using System.Globalization; using System.IO; using System.Linq; +using System.Net; +using System.Net.Sockets; using System.Reflection; -using System.Text; using System.Threading; using BaseDirectoryWrapper = Lucene.Net.Store.BaseDirectoryWrapper; +using Assert = Lucene.Net.TestFramework.Assert; using Console = Lucene.Net.Util.SystemConsole; namespace Lucene.Net.Index @@ -42,7 +43,7 @@ namespace Lucene.Net.Index [TestFixture] public class TestIndexWriterOnJRECrash : TestNRTThreads { - // LUCENENET: Setup unnecessary because we create a new temp directory + // LUCENENET: Setup of directory unnecessary because we create a new temp directory // in each iteration of the test. [Test] @@ -51,7 +52,7 @@ public class TestIndexWriterOnJRECrash : TestNRTThreads public override void TestNRTThreads_Mem() { //if we are not the fork - if (SystemProperties.GetProperty("tests:crashmode") is null) + if (!SystemProperties.GetPropertyAsBoolean("tests:crashmode", false)) { // try up to 10 times to create an index for (int i = 0; i < 10; i++) @@ -63,34 +64,45 @@ public override void TestNRTThreads_Mem() // lexicographical order rather than checking the one we create in the current iteration. DirectoryInfo tempDir = CreateTempDir("netcrash"); - FileInfo tempProcessToKillFile = CreateTempFile(prefix: "netcrash-processToKill", suffix: ".txt"); - tempProcessToKillFile.Delete(); // We use the creation of this file as a signal to parse it. + // Set up a TCP listener to receive the process ID + TcpListener listener = SetupSocketListener(); + Process p = null; + try + { + // Get the port that we picked at random. + int port = ((IPEndPoint)listener.LocalEndpoint).Port; - // Note this is the vstest.console process we are tracking here. - Process p = ForkTest(tempDir.FullName, tempProcessToKillFile.FullName); + // Note this is the vstest.console process we are tracking here. + p = ForkTest(tempDir.FullName, port); - TextWriter childOut = BeginOutput(p, out ThreadJob stdOutPumper, out ThreadJob stdErrPumper); + TextWriter childOut = BeginOutput(p, out ThreadJob stdOutPumper, out ThreadJob stdErrPumper); - // LUCENENET: Note that ForkTest() creates the vstest.console.exe process. - // This spawns testhost.exe, which runs our test. We wait until - // the process starts and logs its own Id so we know who to kill later. - int processIdToKill = WaitForProcessToKillLogFile(tempProcessToKillFile.FullName); + // LUCENENET: Note that ForkTest() creates the vstest.console.exe process. + // This spawns testhost.exe, which runs our test. We wait until + // the process starts and transmits its own PID so we know who to kill later. + int processIdToKill = WaitForProcessId(listener); - // Setup a time to crash the forked thread - int crashTime = TestUtil.NextInt32(Random, 4000, 5000); // LUCENENET: Adjusted these up by 1 second to give our tests some more time to spin up - ThreadJob t = new ThreadAnonymousClass(this, crashTime, processIdToKill); + // Setup a time to crash the forked thread + int crashTime = TestUtil.NextInt32(Random, 4000, 5000); // LUCENENET: Adjusted these up by 1 second to give our tests some more time to spin up + ThreadJob t = new ThreadAnonymousClass(this, crashTime, processIdToKill); - t.Priority = ThreadPriority.Highest; - t.Start(); - t.Join(); // Wait for our thread to kill the other process + t.Priority = ThreadPriority.Highest; + t.Start(); + t.Join(); // Wait for our thread to kill the other process - // if we succeeded in finding an index, we are done. - if (CheckIndexes(tempDir)) - { + // if we succeeded in finding an index, we are done. + if (CheckIndexes(tempDir)) + { + EndOutput(p, childOut, stdOutPumper, stdErrPumper); + return; + } EndOutput(p, childOut, stdOutPumper, stdErrPumper); - return; } - EndOutput(p, childOut, stdOutPumper, stdErrPumper); + finally + { + listener.Stop(); + p?.Dispose(); + } } } else @@ -100,13 +112,12 @@ public override void TestNRTThreads_Mem() // we are the fork, log our processId so the original test can kill us. int processIdToKill = Process.GetCurrentProcess().Id; - string processIdToKillFile = SystemProperties.GetProperty("tests:tempProcessToKillFile"); + int port = SystemProperties.GetPropertyAsInt32("tests:crashtestport"); - assertNotNull("No tests:tempProcessToKillFile value was passed to the fork. This is a required system property.", processIdToKillFile); + assertTrue("No tests:crashtestport value was passed to the fork. This is a required system property.", port > 0); - // Writing this file will kick off the thread that crashes us. - using (var writer = new StreamWriter(processIdToKillFile, append: false, Encoding.UTF8, bufferSize: 32)) - writer.WriteLine(processIdToKill.ToString(CultureInfo.InvariantCulture)); + // Sending the process id will kick off the thread that crashes us. + SendProcessId(processIdToKill, port); // run the test until we crash. for (int i = 0; i < 100; i++) @@ -145,7 +156,7 @@ public override void Run() } } - public Process ForkTest(string tempDir, string tempProcessToKillFile) + public Process ForkTest(string tempDir, int port) { //get the full location of the assembly with DaoTests in it string testAssemblyPath = Assembly.GetAssembly(typeof(TestIndexWriterOnJRECrash)).Location; @@ -174,8 +185,8 @@ public Process ForkTest(string tempDir, string tempProcessToKillFile) // passing NIGHTLY to this test makes it run for much longer, easier to catch it in the act... TestRunParameter("tests:nightly", "true"), TestRunParameter("tempDir", tempDir), - // This file is for passing the process ID of the fork back to the original test so it can kill it. - TestRunParameter("tests:tempProcessToKillFile", tempProcessToKillFile), + // This port is for passing the process ID of the fork back to the original test so it can kill it. + TestRunParameter("tests:crashtestport", port.ToString(CultureInfo.InvariantCulture)), }), WorkingDirectory = theDirectory, RedirectStandardOutput = true, @@ -335,25 +346,31 @@ public virtual bool CheckIndexes(FileSystemInfo file) return false; } - // LUCENENET: Wait for our test to spin up and log its PID so we can kill it. - private static int WaitForProcessToKillLogFile(string processToKillFile) + private TcpListener SetupSocketListener() { - bool exists = false; - Thread.Sleep(500); - for (int i = 0; i < 150; i++) - { - if (File.Exists(processToKillFile)) - { - exists = true; - break; - } - Thread.Sleep(200); - } - // If the fork didn't log its process id, it is a failure. - assertTrue("The test fork didn't log its process id, so we cannot kill it", exists); - using var reader = new StreamReader(processToKillFile, Encoding.UTF8); - // LUCENENET: Our file only has one line with the process Id in it - return int.Parse(reader.ReadLine().Trim(), CultureInfo.InvariantCulture); + // Pick a random port that is available on the local machine. + TcpListener listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + return listener; + } + + // LUCENENET: Wait for our test to spin up and send its process ID so we can kill it. + private int WaitForProcessId(TcpListener listener) + { + using var client = listener.AcceptTcpClient(); + using var stream = client.GetStream(); + // Directly read the process ID as a 32-bit integer + using var reader = new BinaryReader(stream); + return reader.ReadInt32(); + } + + private void SendProcessId(int processId, int port) + { + using var client = new TcpClient("127.0.0.1", port); + using var stream = client.GetStream(); + // Directly write the process ID as a 32-bit integer + using var writer = new BinaryWriter(stream); + writer.Write(processId); } public virtual void CrashDotNet(int processIdToKill) @@ -361,22 +378,29 @@ public virtual void CrashDotNet(int processIdToKill) Process process = null; try { - process = Process.GetProcessById(processIdToKill); - } - catch (ArgumentException) - { - // We get here if the process wasn't running for some reason. - // We should fix the forked test to make it run longer if we get here. - fail("The test completed before we could kill it."); - } + try + { + process = Process.GetProcessById(processIdToKill); + } + catch (ArgumentException) + { + // We get here if the process wasn't running for some reason. + // We should fix the forked test to make it run longer if we get here. + fail("The test completed before we could kill it."); + } #if FEATURE_PROCESS_KILL_ENTIREPROCESSTREE - process.Kill(entireProcessTree: true); + process.Kill(entireProcessTree: true); #else - process.Kill(); + process.Kill(); #endif - process.WaitForExit(10000); - // We couldn't get .NET to crash for some reason. - assertTrue(process.HasExited); + process.WaitForExit(10000); + // We couldn't get .NET to crash for some reason. + assertTrue(process.HasExited); + } + finally + { + process?.Dispose(); + } } } } diff --git a/src/Lucene.Net.Tests/Support/TestConcurrentHashSet.cs b/src/Lucene.Net.Tests/Support/TestConcurrentHashSet.cs new file mode 100644 index 0000000000..39728c3428 --- /dev/null +++ b/src/Lucene.Net.Tests/Support/TestConcurrentHashSet.cs @@ -0,0 +1,51 @@ +using Lucene.Net.Attributes; +using Lucene.Net.Support; +using NUnit.Framework; +using System.Linq; +using System.Threading.Tasks; + +namespace Lucene.Net +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + public class TestConcurrentHashSet + { + [Test, LuceneNetSpecific] + public void TestExceptWith() + { + // Numbers 0-8, 10-80, 99 + var initialSet = Enumerable.Range(1, 8) + .Concat(Enumerable.Range(1, 8).Select(i => i * 10)) + .Append(99) + .Append(0); + + var hashSet = new ConcurrentHashSet(initialSet); + + Parallel.ForEach(Enumerable.Range(1, 8), i => + { + // Remove i and i * 10, i.e. 1 and 10, 2 and 20, etc. + var except = new[] { i, i * 10 }; + hashSet.ExceptWith(except); + }); + + Assert.AreEqual(2, hashSet.Count); + Assert.IsTrue(hashSet.Contains(0)); + Assert.IsTrue(hashSet.Contains(99)); + } + } +} diff --git a/src/Lucene.Net/Store/FSDirectory.cs b/src/Lucene.Net/Store/FSDirectory.cs index 2439b8a92b..66950592c6 100644 --- a/src/Lucene.Net/Store/FSDirectory.cs +++ b/src/Lucene.Net/Store/FSDirectory.cs @@ -1,5 +1,7 @@ using Lucene.Net.Support; using Lucene.Net.Support.IO; +using Lucene.Net.Support.Threading; +using Lucene.Net.Util; using System; using System.Collections.Generic; using System.Globalization; @@ -29,9 +31,6 @@ namespace Lucene.Net.Store * limitations under the License. */ - using Constants = Lucene.Net.Util.Constants; - using IOUtils = Lucene.Net.Util.IOUtils; - /// /// Base class for implementations that store index /// files in the file system. @@ -67,11 +66,11 @@ namespace Lucene.Net.Store /// the best implementation given your /// environment, and the known limitations of each /// implementation. For users who have no reason to prefer a - /// specific implementation, it's best to simply use + /// specific implementation, it's best to simply use /// (or one of its overloads). For all others, you should instantiate the /// desired implementation directly. /// - /// The locking implementation is by default + /// The locking implementation is by default /// , but can be changed by /// passing in a custom instance. /// @@ -80,7 +79,7 @@ namespace Lucene.Net.Store /// in .NET /// in conjunction with an open because it is not guaranteed to exit atomically. /// Any lock statement or call can throw a - /// , which makes shutting down unpredictable. + /// , which makes shutting down unpredictable. /// To exit parallel tasks safely, we recommend using s /// and "interrupt" them with s. /// @@ -95,9 +94,24 @@ public abstract class FSDirectory : BaseDirectory protected readonly DirectoryInfo m_directory; // The underlying filesystem directory - // LUCENENET specific: No such thing as "stale files" in .NET, since Flush(true) writes everything to disk before - // our FileStream is disposed. - //protected readonly ISet m_staleFiles = new ConcurrentHashSet(); // Files written, but not yet sync'ed + /// + /// The collection of stale files that need to be 'ed + /// + /// + /// LUCENENET NOTE: This is a non-thread-safe collection so that we can synchronize access to it + /// using the field. This is to prevent race conditions, i.e. one thread + /// adding a file to the collection while another thread is trying to sync the files, which could + /// cause a missed sync. If you need to access this collection from a derived type, you should + /// synchronize access to it using the protected field. + /// + protected readonly ISet m_staleFiles = new HashSet(); // Files written, but not yet sync'ed + + /// + /// A object to synchronize access to the collection. + /// You should synchronize access to using this object from derived types. + /// + protected readonly object m_syncLock = new object(); + #pragma warning disable 612, 618 private int chunkSize = DEFAULT_READ_CHUNK_SIZE; #pragma warning restore 612, 618 @@ -301,29 +315,39 @@ public override long FileLength(string name) public override void DeleteFile(string name) { EnsureOpen(); - FileInfo file = new FileInfo(Path.Combine(m_directory.FullName, name)); - // LUCENENET specific: We need to explicitly throw when the file has already been deleted, - // since FileInfo doesn't do that for us. - // (An enhancement carried over from Lucene 8.2.0) - if (!File.Exists(file.FullName)) - { - throw new FileNotFoundException("Cannot delete " + file + " because it doesn't exist."); - } + string file = Path.Combine(m_directory.FullName, name); + + // LUCENENET Specific: See remarks for m_staleFiles field. + UninterruptableMonitor.Enter(m_syncLock); try { - file.Delete(); - if (File.Exists(file.FullName)) + // LUCENENET specific: We need to explicitly throw when the file has already been deleted, + // since FileInfo doesn't do that for us. + // (An enhancement carried over from Lucene 8.2.0) + if (!File.Exists(file)) { - throw new IOException("Cannot delete " + file); + throw new FileNotFoundException("Cannot delete " + file + " because it doesn't exist."); } + + try + { + File.Delete(file); + if (File.Exists(file)) + { + throw new IOException("Cannot delete " + file); + } + } + catch (Exception e) + { + throw new IOException("Cannot delete " + file, e); + } + + m_staleFiles.Remove(name); } - catch (Exception e) + finally { - throw new IOException("Cannot delete " + file, e); + UninterruptableMonitor.Exit(m_syncLock); } - // LUCENENET specific: No such thing as "stale files" in .NET, since Flush(true) writes everything to disk before - // our FileStream is disposed. - //m_staleFiles.Remove(name); } /// @@ -366,35 +390,48 @@ protected virtual void EnsureCanWrite(string name) protected virtual void OnIndexOutputClosed(FSIndexOutput io) { - // LUCENENET specific: No such thing as "stale files" in .NET, since Flush(true) writes everything to disk before - // our FileStream is disposed. - //m_staleFiles.Add(io.name); + // LUCENENET Specific: See remarks for m_staleFiles field. + UninterruptableMonitor.Enter(m_syncLock); + try + { + m_staleFiles.Add(io.name); + } + finally + { + UninterruptableMonitor.Exit(m_syncLock); + } } public override void Sync(ICollection names) { EnsureOpen(); - // LUCENENET specific: No such thing as "stale files" in .NET, since Flush(true) writes everything to disk before - // our FileStream is disposed. Therefore, there is nothing else to do in this method. - //ISet toSync = new HashSet(names); - //toSync.IntersectWith(m_staleFiles); - - //// LUCENENET specific: Fsync breaks concurrency here. - //// Part of a solution suggested by Vincent Van Den Berghe: http://apache.markmail.org/message/hafnuhq2ydhfjmi2 - ////foreach (var name in toSync) - ////{ - //// Fsync(name); - ////} - - //// fsync the directory itsself, but only if there was any file fsynced before - //// (otherwise it can happen that the directory does not yet exist)! - //if (toSync.Count > 0) - //{ - // IOUtils.Fsync(m_directory.FullName, true); - //} + ISet toSync = new HashSet(names); + + // LUCENENET Specific: See remarks for m_staleFiles field. + UninterruptableMonitor.Enter(m_syncLock); + try + { + toSync.IntersectWith(m_staleFiles); + + foreach (var name in toSync) + { + Fsync(name); + } + + // fsync the directory itself, but only if there was any file fsynced before + // (otherwise it can happen that the directory does not yet exist)! + if (toSync.Count > 0) + { + IOUtils.Fsync(m_directory.FullName, true); + } - //m_staleFiles.ExceptWith(toSync); + m_staleFiles.ExceptWith(toSync); + } + finally + { + UninterruptableMonitor.Exit(m_syncLock); + } } public override string GetLockID() @@ -546,7 +583,7 @@ protected override void Dispose(bool disposing) Exception priorE = null; // LUCENENET: No need to cast to IOExcpetion try { - file.Flush(flushToDisk: true); + file.Flush(flushToDisk: false); } catch (Exception ioe) when (ioe.IsIOException()) { @@ -586,12 +623,9 @@ public override void Seek(long pos) public override long Position => file.Position; // LUCENENET specific - need to override, since we are buffering locally, renamed from getFilePointer() to match FileStream } - // LUCENENET specific: Fsync is pointless in .NET, since we are - // calling FileStream.Flush(true) before the stream is disposed - // which means we never need it at the point in Java where it is called. - //protected virtual void Fsync(string name) - //{ - // IOUtils.Fsync(Path.Combine(m_directory.FullName, name), false); - //} + protected virtual void Fsync(string name) + { + IOUtils.Fsync(Path.Combine(m_directory.FullName, name), false); + } } -} \ No newline at end of file +} diff --git a/src/Lucene.Net/Support/ConcurrentHashSet.cs b/src/Lucene.Net/Support/ConcurrentHashSet.cs index dd79ead7f0..877e4685e0 100644 --- a/src/Lucene.Net/Support/ConcurrentHashSet.cs +++ b/src/Lucene.Net/Support/ConcurrentHashSet.cs @@ -74,9 +74,9 @@ public int Count { AcquireAllLocks(ref acquiredLocks); - for (var i = 0; i < _tables.CountPerLock.Length; i++) + foreach (var t in _tables.CountPerLock) { - count += _tables.CountPerLock[i]; + count += t; } } finally @@ -88,6 +88,21 @@ public int Count } } + private int CountInternal + { + get + { + int count = 0; + + foreach (var t in _tables.CountPerLock) + { + count += t; + } + + return count; + } + } + /// /// Gets a value that indicates whether the is empty. /// @@ -204,16 +219,16 @@ public ConcurrentHashSet(IEnumerable collection, IEqualityComparer compare /// - /// Initializes a new instance of the - /// class that contains elements copied from the specified , - /// has the specified concurrency level, has the specified initial capacity, and uses the specified + /// Initializes a new instance of the + /// class that contains elements copied from the specified , + /// has the specified concurrency level, has the specified initial capacity, and uses the specified /// . /// - /// The estimated number of threads that will update the + /// The estimated number of threads that will update the /// concurrently. - /// The whose elements are copied to the new + /// The whose elements are copied to the new /// . - /// The implementation to use + /// The implementation to use /// when comparing items. /// /// is a null reference. @@ -298,9 +313,7 @@ public void Clear() { AcquireAllLocks(ref locksAcquired); - var newTables = new Tables(new Node[DefaultCapacity], _tables.Locks, new int[_tables.CountPerLock.Length]); - _tables = newTables; - _budget = Math.Max(1, newTables.Buckets.Length / newTables.Locks.Length); + ClearInternal(); } finally { @@ -308,6 +321,13 @@ public void Clear() } } + private void ClearInternal() + { + var newTables = new Tables(new Node[DefaultCapacity], _tables.Locks, new int[_tables.CountPerLock.Length]); + _tables = newTables; + _budget = Math.Max(1, newTables.Buckets.Length / newTables.Locks.Length); + } + /// /// Determines whether the contains the specified /// item. @@ -347,6 +367,11 @@ public bool Contains(T item) public bool TryRemove(T item) { var hashcode = _comparer.GetHashCode(item); + return TryRemoveInternal(item, hashcode, acquireLock: true); + } + + private bool TryRemoveInternal(T item, int hashcode, bool acquireLock) + { while (true) { var tables = _tables; @@ -354,9 +379,13 @@ public bool TryRemove(T item) GetBucketAndLockNo(hashcode, out int bucketNo, out int lockNo, tables.Buckets.Length, tables.Locks.Length); object syncRoot = tables.Locks[lockNo]; - UninterruptableMonitor.Enter(syncRoot); + var lockTaken = false; + try { + if (acquireLock) + UninterruptableMonitor.Enter(syncRoot, ref lockTaken); + // If the table just got resized, we may not be holding the right lock, and must retry. // This should be a rare occurrence. if (tables != _tables) @@ -388,7 +417,8 @@ public bool TryRemove(T item) } finally { - UninterruptableMonitor.Exit(syncRoot); + if (lockTaken) + UninterruptableMonitor.Exit(syncRoot); } return false; @@ -638,7 +668,7 @@ private void GrowTable(Tables tables) // We want to make sure that GrowTable will not be called again, since table is at the maximum size. // To achieve that, we set the budget to int.MaxValue. // - // (There is one special case that would allow GrowTable() to be called in the future: + // (There is one special case that would allow GrowTable() to be called in the future: // calling Clear() on the ConcurrentHashSet will shrink the table and lower the budget.) _budget = int.MaxValue; } @@ -753,7 +783,34 @@ private void CopyToItems(T[] array, int index) public void ExceptWith(IEnumerable other) { - throw new NotImplementedException(); + if (other is null) + throw new ArgumentNullException(nameof(other)); + + var locksAcquired = 0; + try + { + AcquireAllLocks(ref locksAcquired); + + if (CountInternal == 0) + { + return; + } + + if (ReferenceEquals(this, other)) + { + ClearInternal(); + return; + } + + foreach (var item in other) + { + TryRemoveInternal(item, _comparer.GetHashCode(item), acquireLock: false); + } + } + finally + { + ReleaseLocks(0, locksAcquired); + } } public void IntersectWith(IEnumerable other) diff --git a/src/Lucene.Net/Support/IO/PosixFsyncSupport.cs b/src/Lucene.Net/Support/IO/PosixFsyncSupport.cs new file mode 100644 index 0000000000..d5f4098404 --- /dev/null +++ b/src/Lucene.Net/Support/IO/PosixFsyncSupport.cs @@ -0,0 +1,85 @@ +using Lucene.Net.Util; +using System; +using System.IO; +using System.Runtime.InteropServices; +using static Lucene.Net.Native.Interop.Posix; +using static Lucene.Net.Native.Interop.MacOS; + +namespace Lucene.Net.Support.IO +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + internal static class PosixFsyncSupport + { + public static void Fsync(string path, bool isDir) + { + using DescriptorWrapper handle = new DescriptorWrapper(path, isDir); + handle.Flush(); + } + + private readonly ref struct DescriptorWrapper + { + private readonly int fd; + + public DescriptorWrapper(string path, bool isDir) + { + fd = open(path, isDir ? O_RDONLY : O_WRONLY); + + if (fd == -1) + { + int error = Marshal.GetLastWin32Error(); + + throw error switch + { + ENOENT when isDir => new DirectoryNotFoundException($"Directory/path not found: {path}"), + ENOENT => new FileNotFoundException($"File not found: {path}"), + EACCES => new UnauthorizedAccessException($"Access denied to {(isDir ? "directory" : "file")}: {path}"), + _ => new IOException($"Unable to open path, error: 0x{error:x8}", error) + }; + } + } + + public void Flush() + { + // if macOS, use F_FULLFSYNC + if (Constants.MAC_OS_X) + { + if (fcntl(fd, F_FULLFSYNC, 0) == -1) + { + int error = Marshal.GetLastWin32Error(); + throw new IOException($"fcntl failed, error: 0x{error:x8}", error); + } + } + else if (fsync(fd) == -1) + { + int error = Marshal.GetLastWin32Error(); + throw new IOException($"fsync failed, error: 0x{error:x8}", error); + } + } + + public void Dispose() + { + if (close(fd) == -1) + { + int error = Marshal.GetLastWin32Error(); + throw new IOException($"close failed, error: 0x{error:x8}", error); + } + } + } + } +} diff --git a/src/Lucene.Net/Support/IO/WindowsFsyncSupport.cs b/src/Lucene.Net/Support/IO/WindowsFsyncSupport.cs new file mode 100644 index 0000000000..ace2ab76af --- /dev/null +++ b/src/Lucene.Net/Support/IO/WindowsFsyncSupport.cs @@ -0,0 +1,85 @@ +using System; +using System.IO; +using System.Runtime.InteropServices; +using static Lucene.Net.Native.Interop.Win32; + +namespace Lucene.Net.Support.IO +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + public static class WindowsFsyncSupport + { + public static void Fsync(string path, bool isDir) + { + using HandleWrapper handle = new HandleWrapper(path, isDir); + handle.Flush(); + } + + private readonly ref struct HandleWrapper + { + private readonly IntPtr handle; + + public HandleWrapper(string path, bool isDir) + { + handle = CreateFileW(path, + GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + IntPtr.Zero, + OPEN_EXISTING, + (uint)(isDir ? FILE_FLAG_BACKUP_SEMANTICS : 0), // FILE_FLAG_BACKUP_SEMANTICS required to open a directory + IntPtr.Zero); + + if (handle == INVALID_HANDLE_VALUE) + { + int error = Marshal.GetLastWin32Error(); + + throw error switch + { + ERROR_FILE_NOT_FOUND => new FileNotFoundException($"File not found: {path}"), + ERROR_PATH_NOT_FOUND => new DirectoryNotFoundException($"Directory/path not found: {path}"), + ERROR_ACCESS_DENIED => new UnauthorizedAccessException($"Access denied to {(isDir ? "directory" : "file")}: {path}"), + _ => new IOException($"Unable to open {(isDir ? "directory" : "file")}, error: 0x{error:x8}", error) + }; + } + } + + public void Flush() + { + if (!FlushFileBuffers(handle)) + { + int error = Marshal.GetLastWin32Error(); + + if (error != ERROR_ACCESS_DENIED) + { + // swallow ERROR_ACCESS_DENIED like in OpenJDK + throw new IOException($"FlushFileBuffers failed, error: 0x{error:x8}", error); + } + } + } + + public void Dispose() + { + if (!CloseHandle(handle)) + { + int error = Marshal.GetLastWin32Error(); + throw new IOException($"CloseHandle failed, error: 0x{error:x8}", error); + } + } + } + } +} diff --git a/src/Lucene.Net/Support/Native/Interop.MacOS.Constants.cs b/src/Lucene.Net/Support/Native/Interop.MacOS.Constants.cs new file mode 100644 index 0000000000..d40b745535 --- /dev/null +++ b/src/Lucene.Net/Support/Native/Interop.MacOS.Constants.cs @@ -0,0 +1,28 @@ +namespace Lucene.Net.Native +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + internal static partial class Interop + { + internal static partial class MacOS + { + // https://opensource.apple.com/source/xnu/xnu-6153.81.5/bsd/sys/fcntl.h.auto.html + internal const int F_FULLFSYNC = 51; + } + } +} diff --git a/src/Lucene.Net/Support/Native/Interop.Posix.Close.cs b/src/Lucene.Net/Support/Native/Interop.Posix.Close.cs new file mode 100644 index 0000000000..d4fb9a7b99 --- /dev/null +++ b/src/Lucene.Net/Support/Native/Interop.Posix.Close.cs @@ -0,0 +1,31 @@ +using System.Runtime.InteropServices; + +namespace Lucene.Net.Native +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + internal static partial class Interop + { + internal static partial class Posix + { + // https://pubs.opengroup.org/onlinepubs/009604499/functions/close.html + [DllImport("libc", SetLastError = true)] + internal static extern int close(int fd); + } + } +} diff --git a/src/Lucene.Net/Support/Native/Interop.Posix.Constants.cs b/src/Lucene.Net/Support/Native/Interop.Posix.Constants.cs new file mode 100644 index 0000000000..ef8c02ac36 --- /dev/null +++ b/src/Lucene.Net/Support/Native/Interop.Posix.Constants.cs @@ -0,0 +1,31 @@ +namespace Lucene.Net.Native +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + internal static partial class Interop + { + internal static partial class Posix + { + internal const int O_RDONLY = 0; + internal const int O_WRONLY = 1; + + internal const int EACCES = 13; + internal const int ENOENT = 2; + } + } +} diff --git a/src/Lucene.Net/Support/Native/Interop.Posix.Fcntl.cs b/src/Lucene.Net/Support/Native/Interop.Posix.Fcntl.cs new file mode 100644 index 0000000000..a9e20d5c10 --- /dev/null +++ b/src/Lucene.Net/Support/Native/Interop.Posix.Fcntl.cs @@ -0,0 +1,32 @@ +using System.Runtime.InteropServices; + +namespace Lucene.Net.Native +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + internal static partial class Interop + { + internal static partial class Posix + { + // https://pubs.opengroup.org/onlinepubs/007904975/functions/fcntl.html + // and https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/fcntl.2.html + [DllImport("libc", SetLastError = true)] + internal static extern int fcntl(int fd, int cmd, int arg); + } + } +} diff --git a/src/Lucene.Net/Support/Native/Interop.Posix.Fsync.cs b/src/Lucene.Net/Support/Native/Interop.Posix.Fsync.cs new file mode 100644 index 0000000000..eafb1715f0 --- /dev/null +++ b/src/Lucene.Net/Support/Native/Interop.Posix.Fsync.cs @@ -0,0 +1,31 @@ +using System.Runtime.InteropServices; + +namespace Lucene.Net.Native +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + internal static partial class Interop + { + internal static partial class Posix + { + // https://pubs.opengroup.org/onlinepubs/009695399/functions/fsync.html + [DllImport("libc", SetLastError = true)] + internal static extern int fsync(int fd); + } + } +} diff --git a/src/Lucene.Net/Support/Native/Interop.Posix.Open.cs b/src/Lucene.Net/Support/Native/Interop.Posix.Open.cs new file mode 100644 index 0000000000..b66409ee7c --- /dev/null +++ b/src/Lucene.Net/Support/Native/Interop.Posix.Open.cs @@ -0,0 +1,31 @@ +using System.Runtime.InteropServices; + +namespace Lucene.Net.Native +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + internal static partial class Interop + { + internal static partial class Posix + { + // https://pubs.opengroup.org/onlinepubs/007904875/functions/open.html + [DllImport("libc", SetLastError = true)] + internal static extern int open([MarshalAs(UnmanagedType.LPStr)] string pathname, int flags); + } + } +} diff --git a/src/Lucene.Net/Support/Native/Interop.Win32.CloseHandle.cs b/src/Lucene.Net/Support/Native/Interop.Win32.CloseHandle.cs new file mode 100644 index 0000000000..fa4dafb919 --- /dev/null +++ b/src/Lucene.Net/Support/Native/Interop.Win32.CloseHandle.cs @@ -0,0 +1,32 @@ +using System; +using System.Runtime.InteropServices; + +namespace Lucene.Net.Native +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + internal static partial class Interop + { + internal static partial class Win32 + { + // https://learn.microsoft.com/en-us/windows/win32/api/handleapi/nf-handleapi-closehandle + [DllImport("kernel32.dll", SetLastError = true)] + internal static extern bool CloseHandle(IntPtr hObject); + } + } +} diff --git a/src/Lucene.Net/Support/Native/Interop.Win32.Constants.cs b/src/Lucene.Net/Support/Native/Interop.Win32.Constants.cs new file mode 100644 index 0000000000..2279c02c9a --- /dev/null +++ b/src/Lucene.Net/Support/Native/Interop.Win32.Constants.cs @@ -0,0 +1,43 @@ +using System; + +namespace Lucene.Net.Native +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + internal static partial class Interop + { + internal static partial class Win32 + { + internal static readonly IntPtr INVALID_HANDLE_VALUE = new IntPtr(-1); + + // https://learn.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499- + internal const int ERROR_FILE_NOT_FOUND = 2; + internal const int ERROR_PATH_NOT_FOUND = 3; + internal const int ERROR_ACCESS_DENIED = 5; + + // https://learn.microsoft.com/en-us/windows/win32/secauthz/generic-access-rights + internal const int GENERIC_WRITE = 0x40000000; + + internal const int FILE_SHARE_READ = 0x00000001; + internal const int FILE_SHARE_WRITE = 0x00000002; + internal const int FILE_SHARE_DELETE = 0x00000004; + internal const int FILE_FLAG_BACKUP_SEMANTICS = 0x02000000; + internal const int OPEN_EXISTING = 3; + } + } +} diff --git a/src/Lucene.Net/Support/Native/Interop.Win32.CreateFileW.cs b/src/Lucene.Net/Support/Native/Interop.Win32.CreateFileW.cs new file mode 100644 index 0000000000..e6732c0b2c --- /dev/null +++ b/src/Lucene.Net/Support/Native/Interop.Win32.CreateFileW.cs @@ -0,0 +1,40 @@ +using System; +using System.Runtime.InteropServices; + +namespace Lucene.Net.Native +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + internal static partial class Interop + { + internal static partial class Win32 + { + // https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + internal static extern IntPtr CreateFileW( + string lpFileName, + uint dwDesiredAccess, + uint dwShareMode, + IntPtr lpSecurityAttributes, + uint dwCreationDisposition, + uint dwFlagsAndAttributes, + IntPtr hTemplateFile + ); + } + } +} diff --git a/src/Lucene.Net/Support/Native/Interop.Win32.FlushFileBuffers.cs b/src/Lucene.Net/Support/Native/Interop.Win32.FlushFileBuffers.cs new file mode 100644 index 0000000000..c47ffa95d3 --- /dev/null +++ b/src/Lucene.Net/Support/Native/Interop.Win32.FlushFileBuffers.cs @@ -0,0 +1,32 @@ +using System; +using System.Runtime.InteropServices; + +namespace Lucene.Net.Native +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + internal static partial class Interop + { + internal static partial class Win32 + { + // https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-flushfilebuffers + [DllImport("kernel32.dll", SetLastError = true)] + internal static extern bool FlushFileBuffers(IntPtr hFile); + } + } +} diff --git a/src/Lucene.Net/Util/IOUtils.cs b/src/Lucene.Net/Util/IOUtils.cs index bbfdb1ccfa..dc17cdbdd9 100644 --- a/src/Lucene.Net/Util/IOUtils.cs +++ b/src/Lucene.Net/Util/IOUtils.cs @@ -1,10 +1,14 @@ using J2N; +using Lucene.Net.Diagnostics; using Lucene.Net.Support; +using Lucene.Net.Support.IO; using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Runtime.CompilerServices; using System.Runtime.ExceptionServices; +using System.Runtime.InteropServices; using System.Text; namespace Lucene.Net.Util @@ -516,8 +520,55 @@ public static void ReThrowUnchecked(Exception th) } } - // LUCENENET specific: Fsync is pointless in .NET, since we are - // calling FileStream.Flush(true) before the stream is disposed - // which means we never need it at the point in Java where it is called. + // LUCENENET specific: using string instead of FileSystemInfo to avoid extra allocation + public static void Fsync(string fileToSync, bool isDir) + { + // LUCENENET NOTE: there is a bug in 4.8 where it tries to fsync a directory on Windows, + // which is not supported in OpenJDK. This change adopts the latest Lucene code as of 9.10 + // and only fsyncs directories on Linux and macOS. + + // If the file is a directory we have to open read-only, for regular files we must open r/w for + // the fsync to have an effect. + // See http://blog.httrack.com/blog/2013/11/15/everything-you-always-wanted-to-know-about-fsync/ + if (isDir && Constants.WINDOWS) + { + // opening a directory on Windows fails, directories can not be fsynced there + if (System.IO.Directory.Exists(fileToSync) == false) + { + // yet do not suppress trying to fsync directories that do not exist + throw new DirectoryNotFoundException($"Directory does not exist: {fileToSync}"); + } + return; + } + + try + { + // LUCENENET specific: was: file.force(true); + // We must call fsync on the parent directory, requiring some custom P/Invoking + if (Constants.WINDOWS) + { + WindowsFsyncSupport.Fsync(fileToSync, isDir); + } + else + { + PosixFsyncSupport.Fsync(fileToSync, isDir); + } + } + catch (Exception e) when (e.IsIOException() && isDir && e is not DirectoryNotFoundException) + { + // LUCENENET specific - make catch specific to IOExceptions when it's a directory, + // but allow DirectoryNotFoundException to pass through as an equivalent would normally be + // thrown by the FileChannel.open call in Java which is outside the try block. + + if (Debugging.AssertsEnabled) + { + Debugging.Assert((Constants.LINUX || Constants.MAC_OS_X) == false, + "On Linux and MacOSX fsyncing a directory should not throw IOException, we just don't want to rely on that in production (undocumented). Got: {0}", + e); + } + + // Ignore exception if it is a directory + } + } } }