15.0.56

LICENSE.txt

@@ -158,5 +158,5 @@ ABOVE STATED REMEDY FAILS OF ITS ESSENTIAL PURPOSE.
 
 END OF TERMS AND CONDITIONS
 
-Genozip license version: 15.0.55
+Genozip license version: 15.0.56
 

RELEASE_NOTES.txt

@@ -3,6 +3,9 @@ Note on versioning:
 - Minor version changes with bug fixes and minor feature updates
 - Some minor versions are skipped due to failed deployment pipelines
 
+15.0.56 9/4/2024
+- Bug fixes.
+
 15.0.55 5/4/2024
 - FASTQ: Support for compressing .fastq.ora files (files produced by Illumina Ora).
 - VCF: Incrementally better compression of VCF 

installers/LICENSE.html

@@ -34,4 +34,4 @@
 10. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides Genozip on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Genozip and assume any risks associated with Your exercise of permissions under this License.<br><br>
 11. LIMITATION OF LIABILITY. TO THE FULLEST EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL THEORY, WHETHER IN TORT (INCLUDING NEGLIGENCE), CONTRACT, STRICT LIABILITY OR OTHER LEGAL OR EQUITABLE THEORY, SHALL LICENSOR OR DEVELOPER BE LIABLE FOR DAMAGES, INCLUDING ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER ARISING AS A RESULT OF THIS LICENSE OR OUT OF THE USE OR INABILITY TO USE GENOZIP (INCLUDING BUT NOT LIMITED TO DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, FILE CORRUPTION, DATA LOSS, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES), EVEN IF LICENSOR OR DEVELOPER HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.  IN NO EVENT WILL LICENSOR'S OR DEVELOPER'S TOTAL LIABILITY TO LICENSEE FOR ALL DAMAGES (OTHER THAN AS MAY BE REQUIRED BY APPLICABLE LAW IN CASES INVOLVING PERSONAL INJURY) EXCEED THE AMOUNT OF $500 USD. THE FOREGOING LIMITATIONS WILL APPLY EVEN IF THE ABOVE STATED REMEDY FAILS OF ITS ESSENTIAL PURPOSE.<br><br>
 END OF TERMS AND CONDITIONS<br><br>
-Genozip license version: 15.0.55<br><br>
+Genozip license version: 15.0.56<br><br>

src/Makefile

@@ -19,6 +19,7 @@ architecture := $(shell uname -sm | sed -e 's/amd/x86_/')
 ifeq ($(OS),Windows_NT)
 	Windows := 1
 	OBJDIR=$(OBJDIR_WINDOWS)
+	architecture := Windows $(shell uname -m)
 	WSL=wsl
 endif
 
@@ -84,7 +85,7 @@ ifeq ($(CPU),x86_64) # or amd64
 
 	ifdef linux_x86
 		ASFLAGS += -f elf64 -DINTEL_CET_ENABLED
-		# DEBUGFLAGS += -fsanitize=undefined
+		# DEBUGFLAGS += -fsanitize=undefined 
 		DEBUGFLAGS += -fsanitize=address -Dsanitize_address
 	endif
 
@@ -364,39 +365,39 @@ $(OBJDIR)/%.d: %.cpp | $(OBJDIR) # directory is an "order only prerequesite": ht
 	@$(CXX) $(CXXFLAGS) -MM -MT $@ $< -MF $(@:%.o=%.d)
 
 $(OBJDIR)/distribution.%.o : distribution.c 
-	@echo Compiling \($(architecture)\) $< to $@
+	@echo "Compiling ($(architecture)) $< to $@"
 	@$(CC) -c -o $@ $< $(CFLAGS) -DDISTRIBUTION=\"$*\"
 
 # $(OBJDIR)/bsc/qlfc_%.o : bsc/qlfc.cpp 
-# 	@echo Compiling \($(architecture)\) $< to $@
+# 	@echo "Compiling ($(architecture)) $< to $@"
 # 	$(CXX) -c -o $@ $< $(CXXFLAGS) -DLIBBSC_DYNAMIC_CPU_DISPATCH -DLIBBSC_CPU_FEATURE=LIBBSC_CPU_FEATURE_$*
 
 # $(OBJDIR)/bsc/adler32_%.o : bsc/adler32.cpp 
-# 	@echo Compiling \($(architecture)\) $< to $@
+# 	@echo "Compiling ($(architecture)) $< to $@"
 # 	$(CXX) -c -o $@ $< $(CXXFLAGS) -DLIBBSC_DYNAMIC_CPU_DISPATCH -DLIBBSC_CPU_FEATURE=LIBBSC_CPU_FEATURE_$*
 
 $(OBJDIR)/%.o: %.c $(OBJDIR)/%.d
-	@echo Compiling \($(architecture)\) $<
+	@echo "Compiling ($(architecture)) $<"
 	@$(CC) -c -o $@ $< $(CFLAGS) 
 
 $(OBJDIR)/%.debug-o: %.c $(OBJDIR)/%.d
-	@echo "Compiling \($(architecture)\) $< (debug)"
+	@echo "Compiling ($(architecture)) $< (debug)"
 	@$(CC) -c -o $@ $< $(CFLAGS)
 
 $(OBJDIR)/%.opt-o: %.c $(OBJDIR)/%.d
-	@echo "Compiling \($(architecture)\) $< (opt)"
+	@echo "Compiling ($(architecture)) $< (opt)"
 	@$(CC) -c -o $@ $< $(CFLAGS)
 
 $(OBJDIR)/%.o: %.cpp $(OBJDIR)/%.d
-	@echo Compiling \($(architecture)\) $<
+	@echo "Compiling ($(architecture)) $<"
 	@$(CXX) -c -o $@ $< $(CXXFLAGS) 
 
 $(OBJDIR)/%.debug-o: %.cpp $(OBJDIR)/%.d
-	@echo "Compiling \($(architecture)\) $< (debug)"
+	@echo "Compiling ($(architecture)) $< (debug)"
 	@$(CXX) -c -o $@ $< $(CXXFLAGS)
 
 $(OBJDIR)/%.opt-o: %.cpp $(OBJDIR)/%.d
-	@echo "Compiling \($(architecture)\) $< (opt)"
+	@echo "Compiling ($(architecture)) $< (opt)"
 	@$(CXX) -c -o $@ $< $(CXXFLAGS)
 
 $(OBJDIR)/%.d: %.asm
@@ -408,11 +409,11 @@ $(OBJDIR)/%.d: %.S
 	@$(AS) $(ASFLAGS) -M $< > $@
 
 $(OBJDIR)/%.o: %.asm # nasm or yasm
-	@echo Compiling \($(architecture)\) $<
+	@echo "Compiling ($(architecture)) $<"
 	@$(AS) $(ASFLAGS) -o $@ $< 
 
 $(OBJDIR)/%.o: %.S   # GNU Assembler 
-	@echo Compiling \($(architecture)\) $<
+	@echo "Compiling ($(architecture)) $<"
 	@$(AS) $(ASFLAGS) -o $@ $< 
 
 %.S: %.c $(OBJDIR)/%.d

src/arch.c

@@ -90,7 +90,11 @@ static void arch_add_to_windows_path (void)
 
 void arch_set_locale (void)
 {
-#ifndef _WIN32 // TO DO: make this work for Windows
+#ifdef _WIN32 // see bug 679
+    // ASSERTWD (SetThreadLocale (LOCALE_USER_DEFAULT),
+    ASSERTWD (SetThreadLocale (LOCALE_INVARIANT), // same as En_US
+              "Warning: failed SetThreadLocale: %s", str_win_error());
+#else 
     ASSERTWD0 (setlocale (LC_CTYPE,   flag.is_windows ? ".UTF-8" : "en_US.UTF-8"), "Warning: failed to setlocale of LC_CTYPE");   // accept and print UTF-8 text (to do: doesn't work for Windows)
 #endif
     ASSERTWD0 (setlocale (LC_NUMERIC, flag.is_windows ? "english" : "en_US.UTF-8"), "Warning: failed to setlocale of LC_NUMERIC"); // force printf's %f to use '.' as the decimal separator (not ',') (required by the Genozip file format)
@@ -255,7 +259,8 @@ double arch_get_physical_mem_size (void)
 StrText arch_get_filesystem_type (void)
 {
     StrText s = { "unknown" };
-
+    int save_errno = errno; // save errno, as this function is often used in ASSERT.
+
     if (txt_file && txt_file->is_remote) {
         strcpy (s.s, "remote");
         goto done;
@@ -292,6 +297,7 @@ StrText arch_get_filesystem_type (void)
         NAME (0x2fc12fc1, "ZFS");      // Oracle ZFS (originally in Solaris) https://docs.oracle.com/cd/E19253-01/819-5461/zfsover-2/
         NAME (0x19830326, "FhGFS");    // https://www.beegfs.io/docs/SC13_FHGFS_Presentation.pdf
         NAME (0x53464846, "wslfs");    // WSL1: https://github.com/MicrosoftDocs/WSL/issues/465
+        NAME (0x1021994,  "tmpfs");    // Heap Backing Filesystem
         default: snprintf (s.s, sizeof (s.s), "0x%lx", fs.f_type); 
     }
 
@@ -307,10 +313,12 @@ StrText arch_get_filesystem_type (void)
     WCHAR ws[100];
     if (!GetVolumeInformationByHandleW ((HANDLE)_get_osfhandle(fileno (txt_file->file)), 0, 0, 0, 0, 0, ws, ARRAY_LEN(ws))) goto done;
 
-    wcstombs (s.s, ws, sizeof(s.s)-1);
+    if (wcstombs (s.s, ws, sizeof(s.s)-1) == (size_t)-1)
+        strcpy (s.s, "failed-wcstombs"); // can happen if locale is set to non-english
 #endif    
 
 done:
+    errno = save_errno;
     return s;
 } 
 

src/bam_seg.c

@@ -53,7 +53,7 @@ static int32_t bam_unconsumed_scan_forwards (VBlockP vb)
 
     uint32_t aln_size=0, i;
     for (i=0 ; i < txt_len-3; i += aln_size) 
-        aln_size = LTEN32 ((BAMAlignmentFixed *)&txt[i])->block_size + 4;
+        aln_size = GET_UINT32_((BAMAlignmentFixed *)&txt[i], block_size) + 4;
 
     if (aln_size > txt_len) 
         return -1; // this VB doesn't not even contain one single full alignment
@@ -435,11 +435,11 @@ rom bam_seg_txt_line (VBlockP vb_, rom alignment /* BAM terminology for one line
     dl->FLAG.value       = NEXT_UINT16;              // not to be confused with our global var "flag"
     uint32_t l_seq       = NEXT_UINT32;              // note: we stick with the same logic as SAM for consistency - dl->SEQ.len is determined by CIGAR 
     dl->RNEXT            = (int32_t)NEXT_UINT32;     // corresponding to CHROMs in the BAM header
-    PosType32 next_pos  = 1 + (int32_t)NEXT_UINT32; // pos in BAM is 0 based, -1 for unknown
+    PosType32 next_pos  = 1 + (int32_t)NEXT_UINT32;  // pos in BAM is 0 based, -1 for unknown
     SamTlenType tlen     = (SamTlenType)NEXT_UINT32;
     rom read_name        = next_field;
     dl->QNAME            = (TxtWord){ .index = BNUMtxt (read_name), .len = l_read_name-1 }; // -1 don't count \0
-    BamCigarOp *cigar    = (BamCigarOp *)(read_name + l_read_name);
+    BamCigarOp *cigar    = (BamCigarOp *)(read_name + l_read_name); // note: the "cigar" pointer might be mis-aligned, but we don't de-reference it
     bytes seq            = (uint8_t *)(cigar + n_cigar_op);
     dl->SEQ.index        = BNUMtxt (seq);
     rom qual             = (rom)seq + (l_seq+1)/2;
@@ -495,7 +495,8 @@ rom bam_seg_txt_line (VBlockP vb_, rom alignment /* BAM terminology for one line
         (flag.has_biopsy_line && sam_seg_test_biopsy_line (VB, alignment, block_size + 4)) )
         goto done;  
 
-    sam_cigar_binary_to_textual (vb, n_cigar_op, cigar, &vb->textual_cigar); // re-write BAM format CIGAR as SAM textual format in vb->textual_cigar
+    sam_cigar_binary_to_textual (vb, n_cigar_op, B1ST(BamCigarOp, vb->binary_cigar), // binary_cigar and not "cigar", as the latter is mis-aligned 
+                                 &vb->textual_cigar); // re-write BAM format CIGAR as SAM textual format in vb->textual_cigar
 
     // SEQ - calculate diff vs. reference (denovo or loaded)
     ASSERT (dl->SEQ.len == l_seq || (vb->textual_cigar.len == 1 && *B1STc(vb->textual_cigar) == '*') || !l_seq, 

src/bed.c

@@ -146,7 +146,7 @@ static void bed_seg_START (VBlockP vb, STRp(start), WordIndex prev_line_chrom)
 
 static void bed_seg_TSTART_TEND (VBlockP vb, Did did_i, Did base_did_i, STRp(value), STRp(copy_snip))
 {
-    if (str_issame_(STRa(value), STRtxtw(CTX(did_i)->last_txt)))
+    if (str_issame_(STRa(value), STRtxt(CTX(did_i)->last_txt)))
         seg_by_did (vb, STRa(copy_snip), did_i, value_len + 1);
 
     else 

src/bgzf.c

@@ -156,7 +156,7 @@ void bgzf_finalize_discovery (void)
 
         if (flag.show_bgzf || flag.show_gz) 
             iprintf ("%s: %s %s level %u\n", txt_name, 
-                     (n_levels == 1) ? "Identified as generated with" : "Multiple plausible levels, arbitrarily selecting",
+                     (n_levels == 1) ? "Identified as BGZF generated with" : "Multiple plausible levels, arbitrarily selecting",
                      bgzf_library_name (txt_file->bgzf_flags.library, true), txt_file->bgzf_flags.level);
     }    
 
@@ -566,11 +566,13 @@ void bgzf_zip_advance_index (VBlockP vb, uint32_t line_len)
 int64_t bgzf_copy_unconsumed_blocks (VBlockP vb)
 {
     START_TIMER;
+    ASSERTISZERO (txt_file->unconsumed_bgzf_blocks.len32);
 
     if (!vb->bgzf_blocks.len) return 0; // not a BGZF-compressed file
 
-    int32_t consumed = Ltxt +   // amount of data consumed by this VB
-                       vb->bgzf_blocks.consumed_by_prev_vb; 
+    int32_t consumed = // amount of data in vb->bgzf_blocks that does NOT need to be copied to next VB bc it was consumed by this VB or the previous one 
+        Ltxt +         // amount of data consumed by this VB
+        vb->bgzf_blocks.consumed_by_prev_vb; // amount of data in first BGZF block was conusmed by the previous VB
 
     ARRAY (BgzfBlockZip, bb, vb->bgzf_blocks);
 
@@ -601,6 +603,14 @@ int64_t bgzf_copy_unconsumed_blocks (VBlockP vb)
     // sanity check
     ASSERT (-consumed == txt_file->unconsumed_txt.len32, "Expecting (-consumed)=%d == unconsumed_txt.len=%u", -consumed, txt_file->unconsumed_txt.len32);
 
+    // update bb.txt_index for next VB
+    // note: first bb.txt_data of the next VB is possibly negative if some of its data was consumed by the current VB
+    int32_t txt_index = -txt_file->unconsumed_bgzf_blocks.consumed_by_prev_vb;
+    for_buf (BgzfBlockZip, bb, txt_file->unconsumed_bgzf_blocks) {
+        bb->txt_index = txt_index; 
+        txt_index += bb->txt_size;
+    }
+
     COPY_TIMER (bgzf_copy_unconsumed_blocks);
     return consumed_full_bgzf_blocks ? compressed_size : 0;
 } 
@@ -908,7 +918,7 @@ static void bgzf_compress_one_block (VBlockP vb, rom in, uint32_t isize,
 
     BgzfFooter footer = { .crc32 = LTEN32 (crc32 (0, in, isize)),
                           .isize = LTEN32 (isize) };
-    buf_add (compressed, &footer, sizeof (BgzfFooter));
+    buf_add (compressed, (rom)&footer, sizeof (BgzfFooter));
 
     if (flag.show_bgzf)
         #define C(i) (i < isize ? char_to_printable (in[i]).s : "")

src/bgzf.h

@@ -21,7 +21,7 @@
 
 // data type of VBlock.bgzf_blocks
 typedef struct BgzfBlockZip {
-    uint32_t txt_index;                   // index of uncompressed block within vb->txt_data. The first block doesn't necessarily have index=0 bc there could be passed-down data
+    int32_t txt_index;                    // index of uncompressed block within vb->txt_data. If there is passed-down data from previous VB/txt_header, then txt_index of the first block will be negative (see bgzf_copy_unconsumed_blocks)
     uint32_t txt_size        : 17;        // max value is BGZF_MAX_BLOCK_SIZE
     uint32_t is_decompressed : 1;         // has data been BGZF-decompressed by main thread
     uint32_t compressed_index, comp_size; // index within vb->scratch

src/buffer.c

@@ -100,6 +100,16 @@ void buf_remove_do (BufferP buf, unsigned sizeof_item, uint64_t remove_start, ui
     buf->len -= remove_len;
 }
 
+void buf_add (BufferP buf, STRp(data))
+{
+    if (!data_len) return; // don't test for space (and "data" pointer) if length is 0
+
+    ASSERT (buf_has_space (buf, data_len),
+            "buf_add: buffer %s is out of space: len=%u size=%u data_len=%u",
+            buf_desc (buf).s, (uint32_t)buf->len, (uint32_t)buf->size, data_len);
+    buf_add_do (buf, data, data_len);
+}
+
 void buf_insert_do (VBlockP vb, BufferP buf, unsigned width, uint64_t insert_at, const void *new_data, uint64_t new_data_len, rom name, FUNCLINE) 
 { 
     if (!new_data_len) return;

src/buffer.h

@@ -138,18 +138,15 @@ extern void buf_remove_do (BufferP buf, unsigned sizeof_item, uint64_t remove_st
 
 extern void buf_insert_do (VBlockP vb, BufferP buf, unsigned width, uint64_t insert_at, const void *new_data, uint64_t new_data_len, rom name, FUNCLINE);
 
-#define buf_has_space(buf, new_len) ((buf)->len + (new_len) <= (buf)->size)
-
-#define buf_add_do(buf, new_data, new_len)                                              \
-    ({ memcpy (&(buf)->data[(buf)->len], (new_data), (new_len));                        \
-       (buf)->len += (new_len); })
-
-#define buf_add(buf, new_data, new_data_len)                                            \
-    ({ uint32_t new_len = (uint32_t)(new_data_len); /* copy in case caller uses ++ */   \
-       ASSERT (buf_has_space(buf, new_len),                                             \
-            "buf_add: buffer %s is out of space: len=%u size=%u new_data_len=%u",       \
-            buf_desc (buf).s, (uint32_t)(buf)->len, (uint32_t)(buf)->size, new_len);    \
-       buf_add_do ((buf), (new_data), new_len); })
+#define buf_has_space(buf, new_len) ((buf)->data && (buf)->len + (new_len) <= (buf)->size)
+
+static inline void buf_add_do (BufferP buf, STRp(data))
+{
+    memcpy (&buf->data[buf->len], data, data_len);
+    buf->len += data_len;
+}
+
+extern void buf_add (BufferP buf, STRp(data));
 
 #define buf_add_more(vb, buf, new_data, new_data_len, name)                             \
     buf_insert_do ((VBlockP)(vb), (buf), 1, (buf)->len, (new_data), (new_data_len), (name), __FUNCLINE)

src/codec_domq.c

@@ -27,7 +27,7 @@ typedef struct {
     uint8_t *qual;
     uint32_t qual_len; 
     uint8_t dom;     // dom of this line
-    bool is_diverse; // not enough % of of this line is dom
+    bool is_diverse; // too few qual score in this line are dom
     bool is_rev;
 } QualLine;
 
@@ -262,7 +262,7 @@ static uint8_t codec_domq_prepare_normalize (VBlockP vb, ContextP ctx, LocalGetL
 
     // get quality scores 
     buf_free (ql_buf); // we can free local_hash as this function is called from seg_finalize 
-    buf_alloc_exact (vb, ql_buf, get_line_cb ? vb->lines.len : 1, QualLine, "contexts->qual_line");
+    buf_alloc_exact_zero (vb, ql_buf, get_line_cb ? vb->lines.len : 1, QualLine, "contexts->qual_line");
 
     if (get_line_cb)
         for_buf2 (QualLine, ql, line_i, ql_buf) 

src/container.c

@@ -767,7 +767,7 @@ ContainerP container_retrieve (VBlockP vb, ContextP ctx, WordIndex word_index, S
         buf_alloc (vb, &ctx->con_cache, con_size + prefixes_len + CONTAINER_MAX_SELF_TRANS_CHANGE, 0, char, 2, CTX_TAG_CON_CACHE);
 
         char *cached_con = BAFTc (ctx->con_cache);
-        buf_add (&ctx->con_cache, &con, con_size);
+        buf_add (&ctx->con_cache, (rom)&con, con_size);
         if (prefixes_len) buf_add (&ctx->con_cache, prefixes, prefixes_len);
 
         // IMPORTANT! we are returning a pointer into the cache buffer which might have just been realloced ^. 

src/crypt.c

@@ -98,14 +98,14 @@ static void crypt_generate_aes_key (VBlockP vb,
     uint8_t is_header_byte = (uint8_t)is_header;
 
     // add some salt to the password, mixed with vb_i and sec_i for uniqueness
-    buf_add (&vb->spiced_pw, &vb_i, sizeof (uint32_t));
-    buf_add (&vb->spiced_pw, &sec_type_byte, sizeof (uint8_t));
-    buf_add (&vb->spiced_pw, &is_header_byte, sizeof (uint8_t));
-    buf_add (&vb->spiced_pw, salt, salt_len);
+    buf_add (&vb->spiced_pw, (rom)&vb_i, sizeof (uint32_t));
+    buf_add (&vb->spiced_pw, (rom)&sec_type_byte, sizeof (uint8_t));
+    buf_add (&vb->spiced_pw, (rom)&is_header_byte, sizeof (uint8_t));
+    buf_add (&vb->spiced_pw, STRa(salt));
     Digest salty_hash = md5_do (STRb(vb->spiced_pw));
 
     // add some pepper
-    buf_add (&vb->spiced_pw, pepper, pepper_len);
+    buf_add (&vb->spiced_pw, STRa(pepper));
     Digest peppered_hash = md5_do (STRb(vb->spiced_pw));
 
     // get hash

src/fastq.c

@@ -99,7 +99,9 @@ bool fastq_zip_use_pair_identical (DictId dict_id)
 }
 
 // returns the length of the data at the end of vb->txt_data that will not be consumed by this VB is to be passed to the next VB
-int32_t fastq_unconsumed (VBlockP vb, uint32_t first_i, int32_t *i_out /* in/out */)
+int32_t fastq_unconsumed (VBlockP vb, 
+                          uint32_t first_i, // in/out the smallest index in txt_data for which txt_data is populated (the rest might still in uncompressed BGZF blocks) 
+                          int32_t *i_out) 
 {    
     ASSERT (*i_out >= 0 && *i_out < Ltxt, "*i=%d is out of range [0,%u]", *i_out, Ltxt);
 
@@ -127,7 +129,7 @@ int32_t fastq_unconsumed (VBlockP vb, uint32_t first_i, int32_t *i_out /* in/out
 
     // case: the data provided has less than 8 newlines, and within it we didn't find a read. need more data.
     *i_out = (int32_t)first_i - 1; // next index to test - one before first_i
-    return -1;
+    return -1; // more data please
 }
 
 // called by txtfile_read_vblock when reading the 2nd file in a fastq pair - counts the number of fastq "lines" (each being 4 textual lines),