Mysterious intermittently corrupted gzipped tar files

[WenningQiu]

My application uses SharpZipLib 1.4.2 to process data files into gzipped tar archives. It had been working flawlessly for years on Windows Server 2016 VMs. However, we started to notice intermittent corruptions in the generated archive files after our QA server was migrated to Microsoft Windows Server 2022 Standard (Version 10.0.20348 Build 20348). That being said, I am hardly convinced that Windows Server 2022 is the culprit, as the corruption appears to happen just intermittently - rerunning the application on the same machine and dataset fails to reproduce the corruption as of this writing.

Below is what a corrupted file looks like:

D:\Billing\QAInt\Compressed\20240413>tar ztvf NA_SlbosNet_actdet.20240413-2300.tar.gz
----------  0 user   None   563527 Dec 31  1969 NA_accdevislba0059_SlbNet_actdet.20240413-2300.1.15096.log
----------  0 user   None   558562 Dec 31  1969 NA_accdevislba0059_SlbNet_actdet.20240413-2305.1.15096.log
----------  0 user   None   562643 Dec 31  1969 NA_accdevislba0059_SlbNet_actdet.20240413-2310.1.15096.log
----------  0 user   None   564807 Dec 31  1969 NA_accdevislba0059_SlbNet_actdet.20240413-2315.1.15096.log
----------  0 user   None   560546 Dec 31  1969 NA_accdevislba0059_SlbNet_actdet.20240413-2320.1.15096.log
----------  0 user   None   569440 Dec 31  1969 NA_accdevislba0059_SlbNet_actdet.20240413-2325.1.15096.log
----------  0 user   None   563344 Dec 31  1969 NA_accdevislba0059_SlbNet_actdet.20240413-2330.1.15096.log
----------  0 user   None   566060 Dec 31  1969 NA_accdevislba0059_SlbNet_actdet.20240413-2335.1.15096.log
----------  0 user   None   688378 Dec 31  1969 NA_accdevislba0059_SlbNet_actdet.20240413-2340.1.15096.log
----------  0 user   None   579953 Dec 31  1969 NA_accdevislba0059_SlbNet_actdet.20240413-2345.1.15096.log
----------  0 user   None   560667 Dec 31  1969 NA_accdevislba0059_SlbNet_actdet.20240413-2350.1.15096.log
----------  0 user   None   564881 Dec 31  1969 NA_accdevislba0059_SlbNet_actdet.20240413-2355.1.15096.log
----------  0 user   None   547921 Dec 31  1969 NA_accdevislba0060_SlbNet_actdet.20240413-2300.1.13160.log
----------  0 user   None   551916 Dec 31  1969 NA_accdevislba0060_SlbNet_actdet.20240413-2305.1.13160.log
----------  0 user   None   550054 Dec 31  1969 NA_accdevislba0060_SlbNet_actdet.20240413-2310.1.13160.log
----------  0 user   None   545675 Dec 31  1969 NA_accdevislba0060_SlbNet_actdet.20240413-2315.1.13160.log
tar: Damaged tar archive
tar: Retrying...
tar: Damaged tar archive
tar: Retrying...
tar: Damaged tar archive
tar: Retrying...
tar: Damaged tar archive
tar: Retrying...
tar: Damaged tar archive
tar: Retrying...
tar: Damaged tar archive
tar: Retrying...
tar: Damaged tar archive
tar: Retrying...
tar: Damaged tar archive
tar: Retrying...
tar: Damaged tar archive
tar: Retrying...
tar: Damaged tar archive
tar: Retrying...
tar: Damaged tar archive
tar: Retrying...
tar: Damaged tar archive
tar: Retrying...
tar: Damaged tar archive
tar: Retrying...
tar: Damaged tar archive
tar: Retrying...
tar: Damaged tar archive

The following is the core function of the application that takes a set of gzip files and packs them into a gzipped tar file. Not sure if it has any weakness that can corrupt the file.

    public static class FileCompressor
    {
        public static void CompressFiles(string tarGzipFileName, IEnumerable<string> files, bool onlyGzip)
        {
            string renamedExistingTarGzipFile = null;
            if (!Directory.Exists(Path.GetDirectoryName(tarGzipFileName)))
            {
                Directory.CreateDirectory(Path.GetDirectoryName(tarGzipFileName));
            }
            else if (File.Exists(tarGzipFileName))
            {
                Logger.Info(TraceCategory.Common, "FBE5A1F5-F118-4310-94FD-6FE14F07786E", null, null,
                    "Compressed file {0} already exists.", tarGzipFileName);

                renamedExistingTarGzipFile = tarGzipFileName + "_existing";
                if (File.Exists(renamedExistingTarGzipFile))
                    File.Delete(renamedExistingTarGzipFile);

                File.Move(tarGzipFileName, renamedExistingTarGzipFile);
            }

            var content = new HashSet<string>();
            var badFiles = new List<string>();

            using (var gzipStream = new GZipOutputStream(File.Create(tarGzipFileName)))
            using (var tarStream = new TarOutputStream(gzipStream, Encoding.ASCII))
            {
                if (renamedExistingTarGzipFile != null)
                {
                    try
                    {
                        // Copy the content of the existing compressed file.
                        using (var fsIn = new FileStream(renamedExistingTarGzipFile, FileMode.Open, FileAccess.Read))
                        using (var gzipIn = new GZipInputStream(fsIn))
                        using (var tarIn = new TarInputStream(gzipIn, Encoding.ASCII))
                        {
                            const int bufferSize64K = 65536;
                            using (var stream = new BufferedStream(tarIn, bufferSize64K))
                            {
                                TarEntry tarEntry;
                                while ((tarEntry = tarIn.GetNextEntry()) != null)
                                {
                                    tarStream.PutNextEntry(tarEntry);
                                    CopyStream(stream, tarStream);
                                    tarStream.CloseEntry();

                                    content.Add(tarEntry.Name);

                                    Logger.Info(TraceCategory.Common, "CABDE64A-12F0-4B98-B726-A736FBE71D76", null, null,
                                        "File {0} in the existing compressed file {1} is merged into the new compressed file.",
                                        tarEntry.Name, tarGzipFileName);
                                }

                                tarIn.Close();
                            }
                        }

                        File.Delete(renamedExistingTarGzipFile);
                    }
                    catch (Exception e)
                    {
                        // Log and carry on to keep the data files that can be recovered from the corrupted compressed file.
                        Logger.Major(TraceCategory.Compressor, "44BD23D8-380E-461E-9454-12E0DAE1A86A", null, e, 
                            $"Error copying {renamedExistingTarGzipFile} to {tarGzipFileName}: {e.Message}");
                    }
                }
                
                if (files is null)
                    return;

                // ReSharper disable once PossibleMultipleEnumeration
                foreach (var file in files)
                {
                    var fileName = Path.GetFileName(file).Replace(".gz", string.Empty);
                    if (content.Contains(fileName))
                    {
                        Logger.Info(TraceCategory.Compressor, "65AB4766-A59B-447A-90C1-F69F89363EF9", null, null, $"Ignore {file}, it already exists in {tarGzipFileName}.");
                        continue;
                    }

                    content.Add(fileName);
                    
                    try
                    {
                        using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read))
                        {
                            var gzipStreamLength = GetGzipStreamLength(fileStream);
                            if (onlyGzip && gzipStreamLength < 0)
                                continue;

                            var tarHeader = new TarHeader { Size = gzipStreamLength > 0 ? gzipStreamLength : fileStream.Length };
                            var entry = new TarEntry(tarHeader) {Name = fileName};
                            tarStream.PutNextEntry(entry);

                            if (gzipStreamLength > 0)
                            {
                                using (var decompGzipStream = new GZipInputStream(fileStream))
                                {
                                    CopyStream(decompGzipStream, tarStream);
                                }
                            }
                            else
                            {
                                // I suspect this path might have caused the corruption, but this log message is not observed.
                                Logger.Warning(TraceCategory.Common, "A6872DDD-D332-4AA8-8318-412F9E5B7FDB", null, null, $"File has zero length: {file}");

                                CopyStream(fileStream, tarStream);
                            }

                            tarStream.CloseEntry();
                        }
                    }
                    catch (Exception e)
                    {
                        badFiles.Add(file);

                        // Log and carry on. Do not let a bad file block the processing.
                        var message = $"Failed to compress file {file}.";
                        Logger.Warning(TraceCategory.Common, "467E6E85-EE45-42B7-9CEE-970B88A10D8B", null, e, message);
                    }
                }
            }

            //Delete the compressed files
            foreach (var file in files)
            {
                if (!badFiles.Contains(file))
                    File.Delete(file);
            }
        }

        private static int GetGzipStreamLength(Stream stream)
        {
            var position = stream.Position;
            var buffer = new byte[4];
            try
            {
                // Refer: http://www.gzip.org/zlib/rfc-gzip.html#header-trailer
                if (stream.Read(buffer, 0, 2) != 2 || buffer[0] != 0x1f || buffer[1] != 0x8b)
                    return -1;

                stream.Position = stream.Length - 4;
                if (stream.Read(buffer, 0, 4) != 4)
                    return -1;

                return BitConverter.ToInt32(buffer, 0);
            }
            finally
            {
                stream.Position = position;
            }
        }

        private static void CopyStream(Stream decompGzipStream, Stream tarStream)
        {
            var dataBuffer = new byte[4096];
            StreamUtils.Copy(decompGzipStream, tarStream, dataBuffer);
        }
    }