From 1d6ef5f5c01d459d994d6fbd15f7f3acd6d333d4 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Tue, 11 Jun 2024 14:39:46 +0200
Subject: [PATCH 01/90] invalidate CPRs when reaching position reliability zero

---
 track.c | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/track.c b/track.c
index 6f72a4ff..f02f96ce 100644
--- a/track.c
+++ b/track.c
@@ -483,7 +483,7 @@ static int speed_check(struct aircraft *a, datasource_t source, double lat, doub
                )
             ) {
         mm->pos_ignore = 1; // don't decrement pos_reliable
-    } else if (a->pos_reliable_odd < 0.2 || a->pos_reliable_even < 0.2) {
+    } else if (a->pos_reliable_odd < 0.01f || a->pos_reliable_even < 0.01f) {
         override = 1;
     } else if (now - a->position_valid.updated > POS_RELIABLE_TIMEOUT) {
         override = 1; // no reference or older than 60 minutes, assume OK
@@ -1230,7 +1230,7 @@ static void setPosition(struct aircraft *a, struct modesMessage *mm, int64_t now
 }
 
 static int64_t cpr_global_airborne_max_elapsed(int64_t now, struct aircraft *a) {
-    if (trackDataAge(now, &a->gs_valid) > 20 * SECONDS) {
+    if (!trackDataValid(&a->gs_valid) || trackDataAge(now, &a->gs_valid) > 20 * SECONDS) {
         return 10 * SECONDS;
     }
     int speed = imax((int64_t) a->gs, 1);
@@ -3726,9 +3726,18 @@ static void position_bad(struct modesMessage *mm, struct aircraft *a) {
     }
 
     a->pos_reliable_odd -= 0.26f;
-    a->pos_reliable_odd = fmax(0, a->pos_reliable_odd);
     a->pos_reliable_even -= 0.26f;
-    a->pos_reliable_even = fmax(0, a->pos_reliable_even);
+
+    if (a->pos_reliable_odd < 0.1f || a->pos_reliable_even < 0.1f) {
+
+        // when we reach zero, reset reliable state
+        a->pos_reliable_odd = 0;
+        a->pos_reliable_even = 0;
+
+        // invalidate CPRs to start fresh
+        a->cpr_even_valid.source = SOURCE_INVALID;
+        a->cpr_odd_valid.source = SOURCE_INVALID;
+    }
 
 
     if (0 && a->addr == Modes.cpr_focus)

From c9b9020ee3ae4c66231bed0a9acae67393e63f95 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Tue, 11 Jun 2024 14:42:51 +0200
Subject: [PATCH 02/90] get going a bit quicker after reaching zero reliability

---
 track.c | 50 +++++++++++++++++++++++++++++++-------------------
 1 file changed, 31 insertions(+), 19 deletions(-)

diff --git a/track.c b/track.c
index f02f96ce..408fdbf6 100644
--- a/track.c
+++ b/track.c
@@ -1822,6 +1822,31 @@ static void updateAltitude(int64_t now, struct aircraft *a, struct modesMessage
     return;
 }
 
+static int accept_cpr(struct aircraft *a, struct modesMessage *mm) {
+    // CPR, even
+    if (mm->cpr_valid && !mm->cpr_odd && accept_data(&a->cpr_even_valid, mm->source, mm, a, REDUCE_OFTEN)) {
+        a->cpr_even_type = mm->cpr_type;
+        a->cpr_even_lat = mm->cpr_lat;
+        a->cpr_even_lon = mm->cpr_lon;
+        compute_nic_rc_from_message(mm, a, &a->cpr_even_nic, &a->cpr_even_rc);
+        if (0 && a->addr == Modes.cpr_focus)
+            fprintf(stderr, "E \n");
+        return 1;
+    }
+
+    // CPR, odd
+    if (mm->cpr_valid && mm->cpr_odd && accept_data(&a->cpr_odd_valid, mm->source, mm, a, REDUCE_OFTEN)) {
+        a->cpr_odd_type = mm->cpr_type;
+        a->cpr_odd_lat = mm->cpr_lat;
+        a->cpr_odd_lon = mm->cpr_lon;
+        compute_nic_rc_from_message(mm, a, &a->cpr_odd_nic, &a->cpr_odd_rc);
+        if (0 && a->addr == Modes.cpr_focus)
+            fprintf(stderr, "O \n");
+        return 1;
+    }
+    return 0;
+}
+
 //
 //=========================================================================
 //
@@ -2270,26 +2295,8 @@ struct aircraft *trackUpdateFromMessage(struct modesMessage *mm) {
         a->oat_updated = now;
     }
 
-    // CPR, even
-    if (mm->cpr_valid && !mm->cpr_odd && accept_data(&a->cpr_even_valid, mm->source, mm, a, REDUCE_OFTEN)) {
-        a->cpr_even_type = mm->cpr_type;
-        a->cpr_even_lat = mm->cpr_lat;
-        a->cpr_even_lon = mm->cpr_lon;
-        compute_nic_rc_from_message(mm, a, &a->cpr_even_nic, &a->cpr_even_rc);
-        cpr_new = 1;
-        if (0 && a->addr == Modes.cpr_focus)
-            fprintf(stderr, "E \n");
-    }
-
-    // CPR, odd
-    if (mm->cpr_valid && mm->cpr_odd && accept_data(&a->cpr_odd_valid, mm->source, mm, a, REDUCE_OFTEN)) {
-        a->cpr_odd_type = mm->cpr_type;
-        a->cpr_odd_lat = mm->cpr_lat;
-        a->cpr_odd_lon = mm->cpr_lon;
-        compute_nic_rc_from_message(mm, a, &a->cpr_odd_nic, &a->cpr_odd_rc);
+    if (mm->cpr_valid && accept_cpr(a, mm)) {
         cpr_new = 1;
-        if (0 && a->addr == Modes.cpr_focus)
-            fprintf(stderr, "O \n");
     }
 
     if (mm->cpr_valid) {
@@ -3737,6 +3744,11 @@ static void position_bad(struct modesMessage *mm, struct aircraft *a) {
         // invalidate CPRs to start fresh
         a->cpr_even_valid.source = SOURCE_INVALID;
         a->cpr_odd_valid.source = SOURCE_INVALID;
+
+        // accept the CPR we just got to get going again a bit quicker
+        if (mm->cpr_valid) {
+            accept_cpr(a, mm);
+        }
     }
 
 

From dea46ed7b27cf236e8a68e9ac961b191553ecc2c Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Fri, 14 Jun 2024 16:38:18 +0200
Subject: [PATCH 03/90] dockerfile use run mount

---
 Dockerfile | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index ec82072e..01ed2e6a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -13,9 +13,8 @@ RUN JEMALLOC_BDIR=$(mktemp -d) && \
     rm -rf $JEMALLOC_BDIR
 
 # install readsb
-RUN mkdir -p /app/git
-COPY . /app/git
-RUN cd /app/git && \
+RUN --mount=type=bind,source=.,target=/app/git \
+    cd /app/git && \
     READSB_BUILD_DIR=$(mktemp -d) && \
     cp -r /app/git/* $READSB_BUILD_DIR && \
     cd $READSB_BUILD_DIR && \

From 4f7a7f18c5f88ed57145c04038a04a10d48f8638 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Fri, 14 Jun 2024 16:52:35 +0200
Subject: [PATCH 04/90] ifile: don't use synthetic time for stdin

---
 demod_2400.c | 6 ++++--
 sdr_ifile.c  | 8 ++++++--
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/demod_2400.c b/demod_2400.c
index a1081c74..7ed1a299 100644
--- a/demod_2400.c
+++ b/demod_2400.c
@@ -280,8 +280,9 @@ void demodulate2400(struct mag_buf *mag) {
     msg = msg1;
 
     // advance ifile artificial clock even if we don't receive anything
-    if (Modes.sdr_type == SDR_IFILE)
+    if (Modes.sdr_type == SDR_IFILE && Modes.synthetic_now) {
         Modes.synthetic_now = mag->sysTimestamp;
+    }
 
     uint16_t *pa = m;
     uint16_t *stop = m + mlen;
@@ -408,8 +409,9 @@ void demodulate2400(struct mag_buf *mag) {
         mm->sysTimestamp = mag->sysTimestamp + receiveclock_ms_elapsed(mag->sampleTimestamp, mm->timestamp);
 
         // advance ifile artifical clock for every message received
-        if (Modes.sdr_type == SDR_IFILE)
+        if (Modes.sdr_type == SDR_IFILE && Modes.synthetic_now) {
             Modes.synthetic_now = mm->sysTimestamp;
+        }
 
         mm->score = bestscore;
 
diff --git a/sdr_ifile.c b/sdr_ifile.c
index 50608c58..294f74da 100644
--- a/sdr_ifile.c
+++ b/sdr_ifile.c
@@ -77,7 +77,6 @@ void ifileInitConfig(void) {
     ifile.readbuf = NULL;
     ifile.converter = NULL;
     ifile.converter_state = NULL;
-    Modes.synthetic_now = Modes.startup_time;
 }
 
 bool ifileHandleOption(int key, char *arg) {
@@ -121,7 +120,7 @@ bool ifileOpen(void) {
         return false;
     }
 
-    if (!strcmp(ifile.filename, "-")) {
+    if (strcmp(ifile.filename, "-") == 0) {
         ifile.fd = STDIN_FILENO;
     } else if ((ifile.fd = open(ifile.filename, O_RDONLY)) < 0) {
         fprintf(stderr, "ifile: could not open %s: %s\n",
@@ -129,6 +128,11 @@ bool ifileOpen(void) {
         return false;
     }
 
+    if (strcmp(ifile.filename, "-") != 0) {
+        Modes.synthetic_now = mstime();
+    }
+
+
     switch (ifile.input_format) {
         case INPUT_UC8:
             ifile.bytes_per_sample = 2;

From 51f21881dfafc691893e75fdd21fa8c34cc9d421 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Tue, 18 Jun 2024 14:22:26 +0200
Subject: [PATCH 05/90] cpr debug print

---
 track.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/track.c b/track.c
index 408fdbf6..0df72bf8 100644
--- a/track.c
+++ b/track.c
@@ -678,10 +678,11 @@ static int speed_check(struct aircraft *a, datasource_t source, double lat, doub
             char uuid[32]; // needs 18 chars and null byte
             sprint_uuid1(mm->receiverId, uuid);
             fprintTime(stderr, now);
-            fprintf(stderr, " %06x R%3.1f|%3.1f %s %s %s %s %4.0f%%%2ds%2dt %3.0f/%3.0f td %3.0f %8.3fkm in%4.1fs, %4.0fkt %11.6f,%11.6f->%11.6f,%11.6f biT %4.1f s %s rId %s\n",
+            fprintf(stderr, " %06x R%3.1f|%3.1f %s %2.0f %s %s %s %4.0f%%%2ds%2dt %3.0f/%3.0f td %3.0f %8.3fkm in%4.1fs, %4.0fkt %11.6f,%11.6f->%11.6f,%11.6f biT %4.1f s %s rId %s\n",
                     a->addr,
                     a->pos_reliable_odd, a->pos_reliable_even,
                     mm->cpr_odd ? "O" : "E",
+                    mm->cpr_odd ? fmin(99, (now - a->cpr_even_valid.updated) / 1000.0) : fmin(99, (now - a->cpr_odd_valid.updated) / 1000.0),
                     cpr_local == CPR_LOCAL ? "L" : (cpr_local == CPR_GLOBAL ? "G" : "S"),
                     (surface ? "S" : "A"),
                     failMessage,
@@ -1370,9 +1371,8 @@ static void updatePosition(struct aircraft *a, struct modesMessage *mm, int64_t
 
         setPosition(a, mm, now);
     } else if (location_result == -1 && a->addr == Modes.cpr_focus && !mm->duplicate) {
-        fprintf(stderr, "%5.1fs %d: mm->cpr: (%d) (%d) %s %s, %s age: %0.1f sources o: %s %s e: %s %s lpos src: %s \n",
-                (now % (600 * SECONDS)) / 1000.0,
-                location_result,
+        fprintTime(stderr, now);
+        fprintf(stderr, " mm->cpr: (%d) (%d) %s %s, %s age: %0.1f sources o: %s %s e: %s %s lpos src: %s \n",
                 mm->cpr_lat, mm->cpr_lon,
                 mm->cpr_odd ? " odd" : "even", cpr_type_string(mm->cpr_type), mm->cpr_odd ? "even" : " odd",
                 mm->cpr_odd ? fmin(999, ((double) now - a->cpr_even_valid.updated) / 1000.0) : fmin(999, ((double) now - a->cpr_odd_valid.updated) / 1000.0),

From 526c5d7ff9c522f75984abec00a62abc8c968ea5 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Tue, 9 Jul 2024 20:45:06 +0200
Subject: [PATCH 06/90] hackrf oddities

---
 sdr_hackrf.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/sdr_hackrf.c b/sdr_hackrf.c
index 0098d165..1c0ba5bc 100644
--- a/sdr_hackrf.c
+++ b/sdr_hackrf.c
@@ -50,12 +50,12 @@ void hackRFInitConfig() {
 
 bool hackRFHandleOption(int key, char *arg) {
     switch (key) {
-	case OptHackRfGainEnable:
+        case OptHackRfGainEnable:
             hackRF.rf_gain = true;
-	    break;
-	case OptHackRfVgaGain:
-	    hackRF.vga_gain = atoi(arg);
-	    break;
+            break;
+        case OptHackRfVgaGain:
+            hackRF.vga_gain = atoi(arg);
+            break;
         default:
             return false;
     }
@@ -108,8 +108,8 @@ bool hackRFOpen() {
     if (hackRF.rf_gain) {
         if ((status = hackrf_set_amp_enable(hackRF.device, 1)) != HACKRF_SUCCESS) {
             fprintf(stderr, "hackrf_set_amp_enable failed: %s\n", hackrf_error_name(status));
-	    goto error;
-	}
+            goto error;
+        }
     }
 
     if ((status = hackrf_set_lna_gain(hackRF.device, Modes.gain / 10)) != HACKRF_SUCCESS) {
@@ -131,7 +131,7 @@ bool hackRFOpen() {
 
     fprintf (stderr, "HackRF successfully initialized "
                      "(AMP Enable: %i, LNA Gain: %i, VGA Gain: %i).\n",
-                     Modes.biastee, Modes.gain / 10, hackRF.vga_gain);
+                     hackRF.rf_gain, Modes.gain / 10, hackRF.vga_gain);
 
     hackRF.converter = init_converter(INPUT_UC8,
             Modes.sample_rate,

From 874c84ea370feae0727aea8b40873e98ee4a729f Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Wed, 10 Jul 2024 14:50:01 +0200
Subject: [PATCH 07/90] non-MLAT aircraft without valid track data: more trace
 positions

---
 globe_index.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/globe_index.c b/globe_index.c
index f5cd6e91..bd04e8e8 100644
--- a/globe_index.c
+++ b/globe_index.c
@@ -2314,10 +2314,13 @@ int traceAdd(struct aircraft *a, struct modesMessage *mm, int64_t now, int stale
         max_speed_diff *= 2;
     }
 
-    if (Modes.json_trace_interval > 5 * SECONDS) {
+    if (max_elapsed > 5 * SECONDS) {
         if (a->pos_reliable_valid.source == SOURCE_MLAT) {
             min_elapsed = 1500;
-            max_elapsed /= 2;
+            max_elapsed = imax(max_elapsed / 2, 5 * SECONDS);
+        }
+        if (a->pos_reliable_valid.source != SOURCE_MLAT && !trackVState(now, &a->track_valid, &a->pos_reliable_valid)) {
+            max_elapsed = imax(max_elapsed / 4, 5 * SECONDS);
         }
     }
     // some towers on MLAT .... create unnecessary data

From 1a4c123d2d8fa26ffea5b1b4e3b34d5823d92920 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Thu, 11 Jul 2024 17:00:36 +0200
Subject: [PATCH 08/90] fix exit on hang logic not signaling exit properly

---
 readsb.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/readsb.c b/readsb.c
index 9b52f2a9..ef168adf 100644
--- a/readsb.c
+++ b/readsb.c
@@ -2875,12 +2875,12 @@ int main(int argc, char **argv) {
                 fprintf(stderr, "<3>FATAL: priorityTasksRun() interval %.1f seconds! Trying for an orderly shutdown as well as possible!\n", (double) elapsed1 / SECONDS);
                 fprintf(stderr, "<3>lockThreads() probably hung on %s\n", Modes.currentTask);
                 setExit(2);
-                break;
+                // don't break here, otherwise exitNowEventfd isn't signaled and we don't have a proper exit
+                // setExit signals exitSoonEventfd, the main loop will then signal exitNowEventfd
             }
             if (elapsed2 > 60 * SECONDS && !Modes.synthetic_now) {
                 fprintf(stderr, "<3>FATAL: removeStale() interval %.1f seconds! Trying for an orderly shutdown as well as possible!\n", (double) elapsed2 / SECONDS);
                 setExit(2);
-                break;
             }
         }
     }

From 1ed7c88c0a49b7e1a7f2353b0395ca530fbc2709 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Thu, 11 Jul 2024 17:03:16 +0200
Subject: [PATCH 09/90] incrementing version: 3.14.1632

---
 debian/changelog | 4 ++--
 version          | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 887519b4..df040fd6 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-readsb (3.14.1631) UNRELEASED; urgency=medium
+readsb (3.14.1632) UNRELEASED; urgency=medium
 
   * In development
 
- -- Matthias Wirth <matthias.wirth@gmail.com>  Tue, 11 Jun 2024 10:32:26 +0200
+ -- Matthias Wirth <matthias.wirth@gmail.com>  Thu, 11 Jul 2024 17:03:16 +0200
diff --git a/version b/version
index d765721d..6563f3fd 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.14.1631
+3.14.1632

From 00433443d5cf1203a7a3a49249b65ddca2378fb9 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Thu, 11 Jul 2024 22:56:22 +0200
Subject: [PATCH 10/90] --devel=forwardMinMessages,2

require minimum number of messages for a plane needed to forward
messages for it
will not affect non-aircraft messages
---
 net_io.c | 13 ++++++++++---
 readsb.c |  4 ++++
 readsb.h |  1 +
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/net_io.c b/net_io.c
index 688c1ed4..fbab617d 100644
--- a/net_io.c
+++ b/net_io.c
@@ -5765,17 +5765,24 @@ static void outputMessage(struct modesMessage *mm) {
     if (Modes.filterDF && (mm->sbs_in || !(Modes.filterDFbitset & (1 << mm->msgtype)))) {
         return;
     }
+
+    struct aircraft *ac = mm->aircraft;
+
+    if (ac && ac->messages < Modes.net_forward_min_messages) {
+        return;
+    }
+
     int noforward = (mm->timestamp == MAGIC_NOFORWARD_TIMESTAMP) && !Modes.beast_forward_noforward;
     int64_t orig_ts = mm->timestamp;
     if (Modes.beast_set_noforward_timestamp) {
         mm->timestamp = MAGIC_NOFORWARD_TIMESTAMP;
     }
 
-    struct aircraft *ac = mm->aircraft;
-
     // Suppress the first message when using an SDR
     // messages with crc 0 have an explicit checksum and are more reliable, don't suppress them when there was no CRC fix performed
-    if (Modes.net && !mm->sbs_in && (Modes.net_only || Modes.net_verbatim || (mm->crc == 0 && mm->correctedbits == 0) || (ac && ac->messages > 1) || mm->msgtype == DFTYPE_MODEAC)) {
+    if (Modes.net && !mm->sbs_in
+            && (Modes.net_only || Modes.net_verbatim || (mm->crc == 0 && mm->correctedbits == 0) || (ac && ac->messages > 1) || mm->msgtype == DFTYPE_MODEAC)
+       ) {
         int is_mlat = (mm->source == SOURCE_MLAT);
 
         if (mm->jsonPositionOutputEmit && Modes.json_out.connections) {
diff --git a/readsb.c b/readsb.c
index ef168adf..5954c708 100644
--- a/readsb.c
+++ b/readsb.c
@@ -1932,6 +1932,10 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
                 if (strcasecmp(token[0], "messageRateMult") == 0 && token[1]) {
                     Modes.messageRateMult = atof(token[1]);
                 }
+                if (strcasecmp(token[0], "forwardMinMessages") == 0 && token[1]) {
+                    Modes.net_forward_min_messages = atoi(token[1]);
+                    fprintf(stderr, "forwardMinMessages: %u\n", Modes.net_forward_min_messages);
+                }
                 if (strcasecmp(token[0], "incrementId") == 0) {
                     Modes.incrementId = 1;
                 }
diff --git a/readsb.h b/readsb.h
index ae72ff28..5c40ac35 100644
--- a/readsb.h
+++ b/readsb.h
@@ -728,6 +728,7 @@ struct _Modes
     uint64_t receiver_focus;
 
     uint32_t preambleThreshold;
+    uint32_t net_forward_min_messages;
     int net_output_flush_size; // Minimum Size of output data
     int32_t net_output_beast_reduce_interval; // Position update interval for data reduction
     int32_t ping_reduce;

From 8329053d918870f47dc1ac6e2a78acd7cc467b48 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Fri, 12 Jul 2024 13:47:03 +0200
Subject: [PATCH 11/90] gpsd_in: ignore non-TPV messages

---
 net_io.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/net_io.c b/net_io.c
index fbab617d..54b2a1c7 100644
--- a/net_io.c
+++ b/net_io.c
@@ -3605,9 +3605,9 @@ static int handle_gpsd(struct client *c, char *p, int remote, int64_t now, struc
     } while (*d++);
 
     // filter all messages but TPV type
-    if (0 && !strstr(p, "\"class\":\"TPV\"")) {
+    if (!strstr(p, "\"class\":\"TPV\"")) {
         if (Modes.debug_gps) {
-            fprintf(stderr, "gpsdebug: class \"TPV\" : ignoring message.\n");
+            fprintf(stderr, "gpsdebug: class is not \"TPV\" : ignoring message.\n");
         }
         goto exit;
     }

From 94c4b2dec5608dde404353a15ac23996c996bfd4 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sat, 13 Jul 2024 22:58:07 +0200
Subject: [PATCH 12/90] add --devel=tar1090NoGlobe

---
 json_out.c | 2 +-
 readsb.c   | 5 ++++-
 readsb.h   | 1 +
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/json_out.c b/json_out.c
index 91725907..32f91d13 100644
--- a/json_out.c
+++ b/json_out.c
@@ -1892,7 +1892,7 @@ struct char_buffer generateReceiverJson() {
         p = safe_snprintf(p, end, ", \"dbServer\": true");
     }
 
-    if (Modes.json_globe_index) {
+    if (Modes.json_globe_index && !Modes.tar1090_no_globe) {
 
         p = safe_snprintf(p, end, ", \"json_trace_interval\": %.1f", ((double) Modes.json_trace_interval) / (1 * SECONDS));
 
diff --git a/readsb.c b/readsb.c
index 5954c708..aab52810 100644
--- a/readsb.c
+++ b/readsb.c
@@ -1957,6 +1957,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
                 if (strcasecmp(token[0], "enableConnsJson") == 0) {
                     Modes.enableConnsJson = 1;
                 }
+                if (strcasecmp(token[0], "tar1090NoGlobe") == 0) {
+                    Modes.tar1090_no_globe = 1;
+                }
                 if (strcasecmp(token[0], "provokeSegfault") == 0) {
                     Modes.debug_provoke_segfault = 1;
                 }
@@ -2799,7 +2802,7 @@ int main(int argc, char **argv) {
     if (Modes.json_dir) {
         threadCreate(&Threads.json, NULL, jsonEntryPoint, NULL);
 
-        if (Modes.json_globe_index && !Modes.omitGlobeFiles) {
+        if (Modes.json_globe_index && !Modes.omitGlobeFiles && !Modes.tar1090_no_globe) {
             // globe_xxxx.json
             threadCreate(&Threads.globeJson, NULL, globeJsonEntryPoint, NULL);
         }
diff --git a/readsb.h b/readsb.h
index 5c40ac35..33039b4c 100644
--- a/readsb.h
+++ b/readsb.h
@@ -676,6 +676,7 @@ struct _Modes
     int8_t enable_zstd;
     int8_t incrementId;
     int8_t omitGlobeFiles;
+    int8_t tar1090_no_globe;
     int8_t enableAcasCsv;
     int8_t enableAcasJson;
     int8_t dump_accept_synthetic_now;

From 07df2f2532e45222db9b2c40729e8948d71afbf1 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sat, 13 Jul 2024 22:58:32 +0200
Subject: [PATCH 13/90] incrementing version: 3.14.1633

---
 debian/changelog | 4 ++--
 version          | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index df040fd6..24b25efc 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-readsb (3.14.1632) UNRELEASED; urgency=medium
+readsb (3.14.1633) UNRELEASED; urgency=medium
 
   * In development
 
- -- Matthias Wirth <matthias.wirth@gmail.com>  Thu, 11 Jul 2024 17:03:16 +0200
+ -- Matthias Wirth <matthias.wirth@gmail.com>  Sat, 13 Jul 2024 22:58:32 +0200
diff --git a/version b/version
index 6563f3fd..6f70d7f3 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.14.1632
+3.14.1633

From e04f85c7bf8045ff329e0d111df4242a937942df Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sat, 13 Jul 2024 23:09:05 +0200
Subject: [PATCH 14/90] Dockerfile: x86_64: compile with -march=nehalem

that's 2008 processors ... if someone runs on something older they'll
just have to modify the Dockerfile
---
 Dockerfile | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 01ed2e6a..94fc7e9d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -13,17 +13,19 @@ RUN JEMALLOC_BDIR=$(mktemp -d) && \
     rm -rf $JEMALLOC_BDIR
 
 # install readsb
+SHELL ["/bin/bash", "-x", "-o", "pipefail", "-c"]
 RUN --mount=type=bind,source=.,target=/app/git \
     cd /app/git && \
     READSB_BUILD_DIR=$(mktemp -d) && \
     cp -r /app/git/* $READSB_BUILD_DIR && \
     cd $READSB_BUILD_DIR && \
-    make -j$(nproc) RTLSDR=yes OPTIMIZE="-O2" && \
+    [[ $(uname -m) == x86_64 ]] && MARCH=" -march=nehalem" || MARCH="" && \
+    make -j$(nproc) RTLSDR=yes OPTIMIZE="-O2 $MARCH" && \
     mv readsb /usr/local/bin && \
     mv viewadsb /usr/local/bin && \
     chmod +x /usr/local/bin/viewadsb /usr/local/bin/readsb && \
     make clean && \
-    make -j$(nproc) PRINT_UUIDS=yes TRACKS_UUID=yes OPTIMIZE="-O2" && \
+    make -j$(nproc) PRINT_UUIDS=yes TRACKS_UUID=yes OPTIMIZE="-O2 $MARCH" && \
     mv readsb /usr/local/bin/readsb-uuid && \
     mv viewadsb /usr/local/bin/viewadsb-uuid && \
     chmod +x /usr/local/bin/viewadsb-uuid && \

From 6ff799e19ba6c3403b8bd7bdb5ca2aed5bf9ec9d Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Wed, 17 Jul 2024 10:47:58 +0200
Subject: [PATCH 15/90] traces: don't save more ground positions than intended

tones ground position frequency down after
874c84ea370feae0727aea8b40873e98ee4a729f
unintentionally increased it
---
 globe_index.c | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/globe_index.c b/globe_index.c
index bd04e8e8..d85db377 100644
--- a/globe_index.c
+++ b/globe_index.c
@@ -2314,14 +2314,9 @@ int traceAdd(struct aircraft *a, struct modesMessage *mm, int64_t now, int stale
         max_speed_diff *= 2;
     }
 
-    if (max_elapsed > 5 * SECONDS) {
-        if (a->pos_reliable_valid.source == SOURCE_MLAT) {
-            min_elapsed = 1500;
-            max_elapsed = imax(max_elapsed / 2, 5 * SECONDS);
-        }
-        if (a->pos_reliable_valid.source != SOURCE_MLAT && !trackVState(now, &a->track_valid, &a->pos_reliable_valid)) {
-            max_elapsed = imax(max_elapsed / 4, 5 * SECONDS);
-        }
+    if (max_elapsed > 5 * SECONDS && a->pos_reliable_valid.source == SOURCE_MLAT) {
+        min_elapsed = 1500;
+        max_elapsed = imax(max_elapsed / 2, 5 * SECONDS);
     }
     // some towers on MLAT .... create unnecessary data
     // only reduce data produced for configurations with trace interval more than 5 seconds, others migh want EVERY DOT :)
@@ -2350,6 +2345,10 @@ int traceAdd(struct aircraft *a, struct modesMessage *mm, int64_t now, int stale
         }
     }
 
+    if (max_elapsed > 5 * SECONDS && a->pos_reliable_valid.source != SOURCE_MLAT && track == -1) {
+            max_elapsed = imax(max_elapsed / 4, 5 * SECONDS);
+    }
+
     if (a->trace_current_len == 0)
         goto save_state;
 

From 62768aadbc1db3781b6b45d1e16e21653c42efc9 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Fri, 19 Jul 2024 18:05:53 +0200
Subject: [PATCH 16/90] add geomag copyright notice

---
 geomag.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/geomag.c b/geomag.c
index 1be67a5f..3064aaee 100644
--- a/geomag.c
+++ b/geomag.c
@@ -42,6 +42,17 @@
 
 */
 
+// Copyright Notice
+//
+// As required by 17 U.S.C. 403, third parties producing copyrighted works
+// consisting predominantly of the material produced by U.S. government agencies
+// must provide notice with such work(s) identifying the U.S. Government material
+// incorporated and stating that such material is not subject to copyright
+// protection within the United States. The information on government web pages
+// is in the public domain and not subject to copyright protection within the
+// United States unless specifically annotated otherwise (copyright may be held
+// elsewhere). Foreign copyrights may apply.
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>

From e62917e0de14996070a8e8b9cccfc6e1d378b91c Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Fri, 26 Jul 2024 16:55:04 +0200
Subject: [PATCH 17/90] don't use the rpi zero reduction of cpu usage via
 preamble on non-arm platforms

---
 readsb.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/readsb.c b/readsb.c
index aab52810..d5703ba0 100644
--- a/readsb.c
+++ b/readsb.c
@@ -2245,11 +2245,13 @@ static void configAfterParse() {
     cpu_set_t mask;
     if (sched_getaffinity(getpid(), sizeof(mask), &mask) == 0) {
         Modes.num_procs = CPU_COUNT(&mask);
+#if (defined(__arm__))
         if (Modes.num_procs < 2 && !Modes.preambleThreshold && Modes.sdr_type != SDR_NONE) {
             fprintf(stderr, "WARNING: Reducing preamble threshold / decoding performance as this system has only 1 core (explicitely set --preamble-threshold to disable this behaviour)!\n");
             Modes.preambleThreshold = PREAMBLE_THRESHOLD_PIZERO;
             Modes.fixDF = 0;
         }
+#endif
     }
     if (Modes.num_procs < 1) {
         Modes.num_procs = 1; // sanity check

From a3f82ffb13ed2bea2957bbc0dcc2eec85c7c3494 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Fri, 26 Jul 2024 16:59:45 +0200
Subject: [PATCH 18/90] incrementing version: 3.14.1634

---
 debian/changelog | 4 ++--
 version          | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 24b25efc..db931cd9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-readsb (3.14.1633) UNRELEASED; urgency=medium
+readsb (3.14.1634) UNRELEASED; urgency=medium
 
   * In development
 
- -- Matthias Wirth <matthias.wirth@gmail.com>  Sat, 13 Jul 2024 22:58:32 +0200
+ -- Matthias Wirth <matthias.wirth@gmail.com>  Fri, 26 Jul 2024 16:59:45 +0200
diff --git a/version b/version
index 6f70d7f3..65403d4e 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.14.1633
+3.14.1634

From b6715be2e59b835007179ceb5c5b6cf7a5202cc4 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Wed, 7 Aug 2024 10:12:45 +0200
Subject: [PATCH 19/90] supress most errors for repeated connection failures

---
 net_io.c | 21 ++++++++++++++++-----
 net_io.h |  1 +
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/net_io.c b/net_io.c
index 54b2a1c7..98d19744 100644
--- a/net_io.c
+++ b/net_io.c
@@ -447,9 +447,17 @@ static void checkServiceConnected(struct net_connector *con, int64_t now) {
 
     if (optval != 0) {
         // only 0 means "connection ok"
+        con->fail_counter += 1; // increment fail counter
+
         if (!con->silent_fail) {
-            fprintf(stderr, "%s: Connection to %s%s port %s failed: %d (%s)\n",
-                    con->service->descr, con->address, con->resolved_addr, con->port, optval, strerror(optval));
+            if (con->fail_counter < 10 || Modes.debug_net) {
+                fprintf(stderr, "%s: Connection to %s%s port %s failed (%u): %d (%s)\n",
+                        con->service->descr, con->address, con->resolved_addr, con->port, con->fail_counter, optval, strerror(optval));
+            }
+            if ((con->fail_counter == 10 || con->fail_counter % 200 == 0) && !Modes.debug_net) {
+                fprintf(stderr, "%s: Connection to %s port %s failed %u times, suppressing most error messages until connection succeeds\n",
+                        con->service->descr, con->address, con->port, con->fail_counter);
+            }
         }
         con->connecting = 0;
         anetCloseSocket(con->fd);
@@ -504,6 +512,7 @@ static void checkServiceConnected(struct net_connector *con, int64_t now) {
         }
     }
 
+    con->fail_counter = 0; // reset fail counter on successful connection
 }
 
 // Initiate an outgoing connection.
@@ -681,7 +690,9 @@ static void serviceReconnectCallback(int64_t now) {
                 epoll_ctl(Modes.net_epfd, EPOLL_CTL_DEL, con->fd, &con->dummyClient.epollEvent);
                 anetCloseSocket(con->fd);
             }
-            if (!con->connecting && (con->next_reconnect <= now || Modes.synthetic_now)) {
+
+            //fprintf(stderr, "next_reconnect in: %lld\n", (long long) (con->next_reconnect - now));
+            if (!con->connecting && (now >= con->next_reconnect || Modes.synthetic_now)) {
                 serviceConnect(con, now);
             }
         } else {
@@ -859,7 +870,7 @@ void modesInitNet(void) {
         exit(1);
     }
 
-    Modes.net_connector_delay_min = imax(100, Modes.net_connector_delay / 64);
+    Modes.net_connector_delay_min = imax(50, Modes.net_connector_delay / 64);
     Modes.last_connector_fail = Modes.next_reconnect_callback = mstime();
 
     if (!Modes.net)
@@ -1205,7 +1216,7 @@ static void modesCloseClient(struct client *c) {
         // if we were connected for some time, an immediate reconnect is expected
         con->next_reconnect = con->lastConnect + con->backoff;
 
-        Modes.next_reconnect_callback = now;
+        Modes.next_reconnect_callback = imin(Modes.next_reconnect_callback, con->next_reconnect + 1);
         Modes.last_connector_fail = now;
     }
 
diff --git a/net_io.h b/net_io.h
index 38e69613..c4668107 100644
--- a/net_io.h
+++ b/net_io.h
@@ -160,6 +160,7 @@ struct net_connector
     int use_addr;
     int connected;
     int connecting;
+    uint32_t fail_counter;
     int silent_fail;
     int fd;
     int64_t next_reconnect;

From f2a37b0a940021b5f12ccebe903dba20c356ac7f Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Wed, 7 Aug 2024 12:20:12 +0200
Subject: [PATCH 20/90] net: better debug print

---
 net_io.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/net_io.c b/net_io.c
index 98d19744..e3264a4d 100644
--- a/net_io.c
+++ b/net_io.c
@@ -608,7 +608,7 @@ static void serviceConnect(struct net_connector *con, int64_t now) {
 
     if (Modes.debug_net) {
         //fprintf(stderr, "%s: Attempting connection to %s port %s ... (gonna set SNDBUF %d RCVBUF %d)\n", con->service->descr, con->address, con->port, getSNDBUF(con->service), getRCVBUF(con->service));
-        fprintf(stderr, "%s: Attempting connection to %s port %s ...\n", con->service->descr, con->address, con->port);
+        fprintf(stderr, "%s: Attempting connection to %s%s port %s ...\n", con->service->descr, con->address, con->resolved_addr, con->port);
     }
 
     if (!con->try_addr->ai_next) {

From 62718821fde4df9c464dfb7c9f4dee9540c54b6a Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Wed, 7 Aug 2024 12:50:09 +0200
Subject: [PATCH 21/90] incrementing version: 3.14.1635

---
 debian/changelog | 4 ++--
 version          | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index db931cd9..96fc460f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-readsb (3.14.1634) UNRELEASED; urgency=medium
+readsb (3.14.1635) UNRELEASED; urgency=medium
 
   * In development
 
- -- Matthias Wirth <matthias.wirth@gmail.com>  Fri, 26 Jul 2024 16:59:45 +0200
+ -- Matthias Wirth <matthias.wirth@gmail.com>  Wed, 07 Aug 2024 12:50:09 +0200
diff --git a/version b/version
index 65403d4e..c9fb0e74 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.14.1634
+3.14.1635

From d0b4e7193c18128ed3e49654913ed99aa9c96117 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Wed, 7 Aug 2024 13:09:37 +0200
Subject: [PATCH 22/90] net-connector: improve error suppression

---
 net_io.c | 41 ++++++++++++++++++++++++++---------------
 1 file changed, 26 insertions(+), 15 deletions(-)

diff --git a/net_io.c b/net_io.c
index e3264a4d..49817a96 100644
--- a/net_io.c
+++ b/net_io.c
@@ -422,6 +422,20 @@ static int sendUUID(struct client *c, int64_t now) {
     return -1;
 }
 
+static int suppressConnectError(struct net_connector *con) {
+    con->fail_counter += 1; // increment fail counter
+    if (con->silent_fail) {
+        return 1;
+    }
+    if (con->fail_counter < 9 || Modes.debug_net) {
+        return 0;
+    }
+    if (con->fail_counter == 10 || con->fail_counter % 200 == 0) {
+        fprintf(stderr, "%s: Connection to %s port %s failed %u times, suppressing most error messages until connection succeeds\n",
+                con->service->descr, con->address, con->port, con->fail_counter);
+    }
+    return 1;
+}
 
 static void checkServiceConnected(struct net_connector *con, int64_t now) {
 
@@ -447,17 +461,10 @@ static void checkServiceConnected(struct net_connector *con, int64_t now) {
 
     if (optval != 0) {
         // only 0 means "connection ok"
-        con->fail_counter += 1; // increment fail counter
 
-        if (!con->silent_fail) {
-            if (con->fail_counter < 10 || Modes.debug_net) {
-                fprintf(stderr, "%s: Connection to %s%s port %s failed (%u): %d (%s)\n",
-                        con->service->descr, con->address, con->resolved_addr, con->port, con->fail_counter, optval, strerror(optval));
-            }
-            if ((con->fail_counter == 10 || con->fail_counter % 200 == 0) && !Modes.debug_net) {
-                fprintf(stderr, "%s: Connection to %s port %s failed %u times, suppressing most error messages until connection succeeds\n",
-                        con->service->descr, con->address, con->port, con->fail_counter);
-            }
+        if (!suppressConnectError(con)) {
+            fprintf(stderr, "%s: Connection to %s%s port %s failed (%u): %d (%s)\n",
+                    con->service->descr, con->address, con->resolved_addr, con->port, con->fail_counter, optval, strerror(optval));
         }
         con->connecting = 0;
         anetCloseSocket(con->fd);
@@ -626,8 +633,10 @@ static void serviceConnect(struct net_connector *con, int64_t now) {
     fd = anetCreateSocket(Modes.aneterr, ai->ai_family, SOCK_NONBLOCK);
 
     if (fd == ANET_ERR) {
-        fprintf(stderr, "%s: Connection to %s%s port %s failed: %s\n",
-                con->service->descr, con->address, con->resolved_addr, con->port, Modes.aneterr);
+        if (!suppressConnectError(con)) {
+            fprintf(stderr, "%s: Connection to %s%s port %s failed: %s\n",
+                    con->service->descr, con->address, con->resolved_addr, con->port, Modes.aneterr);
+        }
         return;
     }
 
@@ -663,8 +672,10 @@ static void serviceConnect(struct net_connector *con, int64_t now) {
         epoll_ctl(Modes.net_epfd, EPOLL_CTL_DEL, con->fd, &con->dummyClient.epollEvent);
         con->connecting = 0;
         anetCloseSocket(con->fd);
-        fprintf(stderr, "%s: Connection to %s%s port %s failed: %s\n",
-                con->service->descr, con->address, con->resolved_addr, con->port, strerror(errno));
+        if (!suppressConnectError(con)) {
+            fprintf(stderr, "%s: Connection to %s%s port %s failed: %s\n",
+                    con->service->descr, con->address, con->resolved_addr, con->port, strerror(errno));
+        }
     }
 }
 
@@ -681,7 +692,7 @@ static void serviceReconnectCallback(int64_t now) {
         if (!con->connected) {
             // If we've exceeded our connect timeout, close connection.
             if (con->connecting && now >= con->connect_timeout) {
-                if (!con->silent_fail) {
+                if (!suppressConnectError(con)) {
                     fprintf(stderr, "%s: Connection to %s%s port %s timed out.\n",
                             con->service->descr, con->address, con->resolved_addr, con->port);
                 }

From a3a174cb87ce173b3122352f1cee8439e6ad1df0 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Tue, 13 Aug 2024 15:47:56 +0200
Subject: [PATCH 23/90] json output: fix when input is sbs input

json output wasn't emitting anything for positions coming in via SBS
fix that
---
 net_io.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/net_io.c b/net_io.c
index 49817a96..b72ff1ec 100644
--- a/net_io.c
+++ b/net_io.c
@@ -5787,6 +5787,9 @@ static void outputMessage(struct modesMessage *mm) {
     if (Modes.filterDF && (mm->sbs_in || !(Modes.filterDFbitset & (1 << mm->msgtype)))) {
         return;
     }
+    if (!Modes.net) {
+        return;
+    }
 
     struct aircraft *ac = mm->aircraft;
 
@@ -5802,15 +5805,11 @@ static void outputMessage(struct modesMessage *mm) {
 
     // Suppress the first message when using an SDR
     // messages with crc 0 have an explicit checksum and are more reliable, don't suppress them when there was no CRC fix performed
-    if (Modes.net && !mm->sbs_in
+    if (!mm->sbs_in
             && (Modes.net_only || Modes.net_verbatim || (mm->crc == 0 && mm->correctedbits == 0) || (ac && ac->messages > 1) || mm->msgtype == DFTYPE_MODEAC)
        ) {
         int is_mlat = (mm->source == SOURCE_MLAT);
 
-        if (mm->jsonPositionOutputEmit && Modes.json_out.connections) {
-            jsonPositionOutput(mm, ac);
-        }
-
         if (Modes.garbage_ports && (mm->garbage || mm->pos_bad) && !mm->pos_old && Modes.garbage_out.connections) {
             modesSendBeastOutput(mm, &Modes.garbage_out);
         }
@@ -5848,7 +5847,11 @@ static void outputMessage(struct modesMessage *mm) {
         }
     }
 
-    if (mm->sbs_in && Modes.net && ac) {
+    if (mm->jsonPositionOutputEmit && Modes.json_out.connections) {
+        jsonPositionOutput(mm, ac);
+    }
+
+    if (mm->sbs_in && ac) {
         if (mm->reduce_forward || !Modes.sbsReduce) {
             if (Modes.sbs_out.connections) {
                 modesSendSBSOutput(mm, ac, &Modes.sbs_out);

From d2f12d9a22611afffaf5b907848034d5b7e7dc92 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Tue, 13 Aug 2024 15:51:14 +0200
Subject: [PATCH 24/90] incrementing version: 3.14.1636

---
 debian/changelog | 4 ++--
 version          | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 96fc460f..4b826fc1 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-readsb (3.14.1635) UNRELEASED; urgency=medium
+readsb (3.14.1636) UNRELEASED; urgency=medium
 
   * In development
 
- -- Matthias Wirth <matthias.wirth@gmail.com>  Wed, 07 Aug 2024 12:50:09 +0200
+ -- Matthias Wirth <matthias.wirth@gmail.com>  Tue, 13 Aug 2024 15:51:14 +0200
diff --git a/version b/version
index c9fb0e74..9d559030 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.14.1635
+3.14.1636

From 3da4c74de37a249c8208c9751ccc578b89f07cc5 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Wed, 14 Aug 2024 07:16:52 +0200
Subject: [PATCH 25/90] add --devel=legacy_history

---
 readsb.c | 5 ++++-
 readsb.h | 1 +
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/readsb.c b/readsb.c
index d5703ba0..a4aca6c4 100644
--- a/readsb.c
+++ b/readsb.c
@@ -571,7 +571,7 @@ static void *jsonEntryPoint(void *arg) {
             }
             writeJsonToFile(Modes.json_dir, "aircraft.json", cb);
 
-            if ((ALL_JSON) && Modes.onlyBin < 2 && now >= next_history) {
+            if ((Modes.legacy_history || ((ALL_JSON) && Modes.onlyBin < 2)) && now >= next_history) {
                 char filebuf[PATH_MAX];
 
                 snprintf(filebuf, PATH_MAX, "history_%d.json", Modes.json_aircraft_history_next);
@@ -1896,6 +1896,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
                         Modes.apiThreadCount = atoi(token[1]);
                     }
                 }
+                if (strcasecmp(token[0], "legacy_history") == 0) {
+                    Modes.legacy_history = 1;
+                }
                 if (strcasecmp(token[0], "beast_forward_noforward") == 0) {
                     Modes.beast_forward_noforward = 1;
                 }
diff --git a/readsb.h b/readsb.h
index 33039b4c..06064c11 100644
--- a/readsb.h
+++ b/readsb.h
@@ -673,6 +673,7 @@ struct _Modes
     int8_t debug_gps;
     int8_t debug_planefinder;
     int8_t debug_zstd;
+    int8_t legacy_history;
     int8_t enable_zstd;
     int8_t incrementId;
     int8_t omitGlobeFiles;

From cae21dc28134f40b352c4cc5ce7e0cc9df07ce7a Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Wed, 14 Aug 2024 07:19:15 +0200
Subject: [PATCH 26/90] remove legacy history json files from readme

---
 README-json.md | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/README-json.md b/README-json.md
index eecf38d0..d43ca549 100644
--- a/README-json.md
+++ b/README-json.md
@@ -24,7 +24,6 @@ The keys are:
 
  * version: the version of readsb in use
  * refresh: how often aircraft.json is updated (for the file version), in milliseconds. the webmap uses this to control its refresh interval.
- * history: the current number of valid history files (see below)
  * lat: the latitude of the receiver in decimal degrees. Optional, may not be present.
  * lon: the longitude of the receiver in decimal degrees. Optional, may not be present.
 
@@ -202,18 +201,6 @@ The keys are:
   ```
   * Status code 200 during normal operation
 
-## history_0.json, history_1.json, ..., history_119.json
-
-These files are historical copies of aircraft.json at (by default) 30 second intervals. They follow exactly the
-same format as aircraft.json. To know how many are valid, see receiver.json ("history" value). They are written in
-a cycle, with history_0 being overwritten after history_119 is generated, so history_0.json is not necessarily the
-oldest history entry. To load history, you should:
-
- * read "history" from receiver.json.
- * load that many history_N.json files
- * sort the resulting files by their "now" values
- * process the files in order
-
 ## trace jsons
 
  * overall structure

From e174999566520c1d11de8a45375a389f65c729ad Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sun, 18 Aug 2024 16:12:26 +0200
Subject: [PATCH 27/90] update packages

---
 Dockerfile        | 16 ++++++++++++++--
 required_packages |  2 +-
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 94fc7e9d..34b6c1fb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,6 +1,18 @@
-FROM debian:bookworm-20240311 AS builder
+FROM debian:bookworm-slim AS builder
 RUN apt-get update && \
-    apt-get install -y git wget pkg-config autoconf gcc make libusb-1.0-0-dev librtlsdr-dev librtlsdr0 libncurses-dev zlib1g-dev zlib1g libzstd-dev libzstd1
+    apt-get install --no-install-recommends -y \
+    git \
+    wget \
+    pkg-config \
+    autoconf \
+    gcc \
+    make \
+    libusb-1.0-0-dev \
+    librtlsdr-dev \
+    libncurses-dev \
+    zlib1g-dev \
+    libzstd-dev \
+    ca-certificates
 
 # install jemalloc
 RUN JEMALLOC_BDIR=$(mktemp -d) && \
diff --git a/required_packages b/required_packages
index 1b085dae..b9bd6796 100644
--- a/required_packages
+++ b/required_packages
@@ -1 +1 @@
-gcc make libusb-1.0-0-dev librtlsdr-dev librtlsdr0 libncurses-dev zlib1g-dev zlib1g libzstd-dev libzstd1 pkg-config
+gcc make libc6-dev libusb-1.0-0-dev librtlsdr-dev libncurses-dev zlib1g-dev libzstd-dev pkg-config

From c4ba31b4921a005c19b85be34425dbf4a31b67c1 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sun, 18 Aug 2024 17:20:47 +0200
Subject: [PATCH 28/90] docker: use separate repo for builder

this avoids unnecessary caching / rebuilds
---
 .github/workflows/docker.yaml | 10 ++++----
 Dockerfile                    | 44 +++++++++--------------------------
 2 files changed, 15 insertions(+), 39 deletions(-)

diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml
index 2620f896..f8776a1e 100644
--- a/.github/workflows/docker.yaml
+++ b/.github/workflows/docker.yaml
@@ -2,12 +2,10 @@ name: docker build
 
 on:
   schedule:
-  - cron: "0 10 * * *"
+  - cron: "0 5 * * 1"
   push:
     branches:
-    - "**"
-    tags:
-    - "v*.*.*"
+    - "dev"
   pull_request:
     branches:
     - "dev"
@@ -21,8 +19,8 @@ jobs:
     - name: Set variables useful for later
       id: useful_vars
       run: |-
-        echo "::set-output name=timestamp::$(date +%s)"
-        echo "::set-output name=short_sha::${GITHUB_SHA::8}"
+        echo "name=timestamp::$(date +%s)" >> $GITHUB_OUTPUT
+        echo "name=short_sha::${GITHUB_SHA::8}" >> $GITHUB_OUTPUT
     - name: Checkout
       uses: actions/checkout@v3
     - name: Docker meta
diff --git a/Dockerfile b/Dockerfile
index 34b6c1fb..ac2a94cc 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,30 +1,5 @@
-FROM debian:bookworm-slim AS builder
-RUN apt-get update && \
-    apt-get install --no-install-recommends -y \
-    git \
-    wget \
-    pkg-config \
-    autoconf \
-    gcc \
-    make \
-    libusb-1.0-0-dev \
-    librtlsdr-dev \
-    libncurses-dev \
-    zlib1g-dev \
-    libzstd-dev \
-    ca-certificates
+FROM ghcr.io/wiedehopf/readsb-builder:latest AS builder
 
-# install jemalloc
-RUN JEMALLOC_BDIR=$(mktemp -d) && \
-    git clone --depth 1 https://github.com/jemalloc/jemalloc $JEMALLOC_BDIR && \
-    cd $JEMALLOC_BDIR && \
-    ./autogen.sh && \
-    ./configure --with-lg-page=14 && \
-    make -j$(nproc) && \
-    make install && \
-    rm -rf $JEMALLOC_BDIR
-
-# install readsb
 SHELL ["/bin/bash", "-x", "-o", "pipefail", "-c"]
 RUN --mount=type=bind,source=.,target=/app/git \
     cd /app/git && \
@@ -48,18 +23,21 @@ RUN --mount=type=bind,source=.,target=/app/git \
     true
 
 FROM debian:bookworm-slim
-RUN apt-get update && \
+RUN \
+    --mount=type=bind,from=builder,source=/,target=/builder/ \
+    apt-get update && \
     apt-get -y install --no-install-recommends \
-    wget \
     librtlsdr0 libncurses6 zlib1g libzstd1 && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
-    mkdir -p /run/readsb
+    mkdir -p /run/readsb && \
+    cp /builder/usr/local/bin/readsb* /usr/local/bin/ && \
+    cp /builder/usr/local/bin/viewadsb* /usr/local/bin/ && \
+    mkdir -p  /usr/local/share/tar1090 && \
+    cp /builder/usr/local/share/tar1090/aircraft.csv.gz /usr/local/share/tar1090/aircraft.csv.gz && \
+    cp /builder/usr/local/lib/libjemalloc.so.2 /usr/local/lib/libjemalloc.so.2 && \
+    true
 
 ENV LD_PRELOAD=/usr/local/lib/libjemalloc.so.2
-COPY --from=builder /usr/local/bin/readsb* /usr/local/bin/
-COPY --from=builder /usr/local/bin/viewadsb* /usr/local/bin/
-COPY --from=builder /usr/local/share/tar1090/aircraft.csv.gz /usr/local/share/tar1090/aircraft.csv.gz
-COPY --from=builder /usr/local/lib/libjemalloc.so.2 /usr/local/lib/libjemalloc.so.2
 
 ENTRYPOINT ["/usr/local/bin/readsb"]

From 24568a6f2af7d0f9164b9221b74a37168adc9946 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sun, 18 Aug 2024 17:51:18 +0200
Subject: [PATCH 29/90] incrementing version: 3.14.1637

---
 debian/changelog | 4 ++--
 version          | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 4b826fc1..7d168803 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-readsb (3.14.1636) UNRELEASED; urgency=medium
+readsb (3.14.1637) UNRELEASED; urgency=medium
 
   * In development
 
- -- Matthias Wirth <matthias.wirth@gmail.com>  Tue, 13 Aug 2024 15:51:14 +0200
+ -- Matthias Wirth <matthias.wirth@gmail.com>  Sun, 18 Aug 2024 17:51:18 +0200
diff --git a/version b/version
index 9d559030..91cee972 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.14.1636
+3.14.1637

From 7a651de8d6e23e6135e409fa6c2d7e8a5f6fbffb Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sun, 18 Aug 2024 17:56:54 +0200
Subject: [PATCH 30/90] docker: re-add tagged builds and build for sid

---
 .github/workflows/docker.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml
index f8776a1e..61fd8753 100644
--- a/.github/workflows/docker.yaml
+++ b/.github/workflows/docker.yaml
@@ -6,6 +6,9 @@ on:
   push:
     branches:
     - "dev"
+    - "sid"
+  tags:
+    - "v*.*.*"
   pull_request:
     branches:
     - "dev"

From 31aa09e44064995b5653c4d55f479aeea6d3b36e Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sun, 18 Aug 2024 18:00:26 +0200
Subject: [PATCH 31/90] docker: remove cache

this is not really necessary now
---
 .github/workflows/docker.yaml | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml
index 61fd8753..7f4bfa09 100644
--- a/.github/workflows/docker.yaml
+++ b/.github/workflows/docker.yaml
@@ -53,13 +53,6 @@ jobs:
         registry: ghcr.io
         username: ${{ github.repository_owner }}
         password: ${{ secrets.GITHUB_TOKEN }}
-    - name: Cache Docker layers
-      uses: actions/cache@v3
-      with:
-        path: /tmp/.buildx-cache
-        key: ${{ runner.os }}-buildx-${{ github.sha }}
-        restore-keys: |
-          ${{ runner.os }}-buildx-
     - name: Build and push
       uses: docker/build-push-action@v4
       with:
@@ -68,5 +61,3 @@ jobs:
         tags: ${{ steps.docker_meta.outputs.tags }}
         labels: ${{ steps.docker_meta.outputs.labels }}
         platforms: linux/amd64,linux/arm64
-        cache-from: type=local,src=/tmp/.buildx-cache
-        cache-to: type=local,dest=/tmp/.buildx-cache,mode=max

From e7d8c49d378c4e3079f581a6584987d005c3abeb Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sun, 18 Aug 2024 18:07:04 +0200
Subject: [PATCH 32/90] dockerfile yml

---
 .github/workflows/docker.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml
index 7f4bfa09..8a17b8dc 100644
--- a/.github/workflows/docker.yaml
+++ b/.github/workflows/docker.yaml
@@ -2,16 +2,16 @@ name: docker build
 
 on:
   schedule:
-  - cron: "0 5 * * 1"
+    - cron: "0 5 * * 1"
   push:
     branches:
-    - "dev"
-    - "sid"
-  tags:
-    - "v*.*.*"
+      - "dev"
+      - "sid"
+    tags:
+      - "v*.*.*"
   pull_request:
     branches:
-    - "dev"
+      - "dev"
 
 jobs:
   docker:

From bede96fa78adfc7aa76bb982ddaa77ed85971975 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Tue, 20 Aug 2024 23:14:29 +0200
Subject: [PATCH 33/90] readsb api: support accept-encoding: zstd

---
 api.c | 21 +++++++++++++++------
 api.h |  1 +
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/api.c b/api.c
index 957c6c08..4715589d 100644
--- a/api.c
+++ b/api.c
@@ -727,7 +727,7 @@ static struct char_buffer apiReq(struct apiThread *thread, struct apiOptions *op
         sfree(matches);
     }
 
-    if (options->zstd) {
+    if (options->zstd || options->zstd_encode) {
         struct char_buffer new = { 0 };
         size_t new_alloc = API_REQ_PADSTART + ZSTD_compressBound(alloc);
         new.buffer = cmalloc(new_alloc);
@@ -1078,7 +1078,7 @@ static int parseDoubles(char *start, char *end, double *results, int max) {
 }
 
 // expects lower cased input
-static struct char_buffer parseFetch(struct apiCon *con, struct char_buffer *request, struct apiThread *thread) {
+static struct char_buffer parseFetch(struct apiCon *con, struct char_buffer *request, struct apiOptions *options, struct apiThread *thread) {
     struct char_buffer invalid = { 0 };
 
     char *req = request->buffer;
@@ -1100,9 +1100,6 @@ static struct char_buffer parseFetch(struct apiCon *con, struct char_buffer *req
     // we only want the URL
     *eoq = '\0';
 
-    struct apiOptions optionsBack = { 0 };
-    struct apiOptions *options = &optionsBack;
-
     // set some option defaults:
     options->above_alt_baro = INT32_MIN;
     options->below_alt_baro = INT32_MAX;
@@ -1346,6 +1343,8 @@ static struct char_buffer parseFetch(struct apiCon *con, struct char_buffer *req
 
 
     if (options->zstd) {
+        // don't double zstd compress
+        options->zstd_encode = 0;
          con->content_type = "application/zstd";
     } else if (options->binCraft) {
          con->content_type = "application/octet-stream";
@@ -1526,6 +1525,9 @@ static void apiReadRequest(struct apiCon *con, struct apiThread *thread) {
         return;
     }
 
+    struct apiOptions optionsBack = { 0 };
+    struct apiOptions *options = &optionsBack;
+
     // set end padding to zeros for byteMatchStart and byteMatch (memcmp) use without regrets
     memset(req_end, 0, end_pad);
 
@@ -1552,6 +1554,11 @@ static void apiReadRequest(struct apiCon *con, struct apiThread *thread) {
     while (hl < req_end && (eol = memchr(hl, '\n', req_end - hl))) {
         *eol = '\0';
 
+        if (byteMatchStart(hl, "accept-encoding")) {
+            if (strstr(hl, "zstd")) {
+                options->zstd_encode = 1;
+            }
+        }
         if (byteMatchStart(hl, "connection")) {
             if (strstr(hl, "close")) {
                 con->keepalive = 0;
@@ -1589,7 +1596,7 @@ static void apiReadRequest(struct apiCon *con, struct apiThread *thread) {
     }
 
     con->content_type = "multipart/mixed";
-    struct char_buffer reply = parseFetch(con, request, thread);
+    struct char_buffer reply = parseFetch(con, request, options, thread);
     if (reply.len == 0) {
         //fprintf(stderr, "parseFetch returned invalid\n");
         send400(con->fd, con->keepalive);
@@ -1613,10 +1620,12 @@ static void apiReadRequest(struct apiCon *con, struct apiThread *thread) {
             "content-type: %s\r\n"
             "connection: %s\r\n"
             "cache-control: no-store\r\n"
+            "%s"
             "content-length: %d\r\n\r\n",
             con->include_version ? "readsb_version: "MODES_READSB_VERSION"\r\n" : "",
             con->content_type,
             con->keepalive ? "keep-alive" : "close",
+            options->zstd_encode ? "content-encoding: zstd\r\n" : "",
             content_len);
 
     int hlen = p - header;
diff --git a/api.h b/api.h
index 090f270f..c88f668f 100644
--- a/api.h
+++ b/api.h
@@ -50,6 +50,7 @@ struct apiOptions {
     int filter_squawk;
     int binCraft;
     int zstd;
+    int zstd_encode;
     unsigned squawk;
     int filter_dbFlag;
     int filter_mil;

From bcf81a6764b0d718dca76a031d3b0f63b7cf95de Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Tue, 20 Aug 2024 23:26:30 +0200
Subject: [PATCH 34/90] capitalize response headers

---
 api.c | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/api.c b/api.c
index 4715589d..1394aed2 100644
--- a/api.c
+++ b/api.c
@@ -1022,10 +1022,10 @@ static void sendStatus(int fd, int keepalive, const char *http_status) {
 
     p = safe_snprintf(p, end,
     "HTTP/1.1 %s\r\n"
-    "server: readsb/3.1442\r\n"
-    "connection: %s\r\n"
-    "cache-control: no-store\r\n"
-    "content-length: 0\r\n\r\n",
+    "Server: readsb/wiedehopf\r\n"
+    "Connection: %s\r\n"
+    "Cache-control: no-store\r\n"
+    "Content-length: 0\r\n\r\n",
     http_status,
     keepalive ? "keep-alive" : "close");
 
@@ -1345,11 +1345,11 @@ static struct char_buffer parseFetch(struct apiCon *con, struct char_buffer *req
     if (options->zstd) {
         // don't double zstd compress
         options->zstd_encode = 0;
-         con->content_type = "application/zstd";
+        con->content_type = "application/zstd";
     } else if (options->binCraft) {
-         con->content_type = "application/octet-stream";
+        con->content_type = "application/octet-stream";
     } else {
-         con->content_type = "application/json";
+        con->content_type = "application/json";
     }
 
     return apiReq(thread, options);
@@ -1615,17 +1615,17 @@ static void apiReadRequest(struct apiCon *con, struct apiThread *thread) {
 
     p = safe_snprintf(p, end,
             "HTTP/1.1 200 OK\r\n"
-            "server: readsb/3.1442\r\n"
+            "Server: readsb/wiedehopf\r\n"
             "%s"
-            "content-type: %s\r\n"
-            "connection: %s\r\n"
-            "cache-control: no-store\r\n"
+            "Content-Type: %s\r\n"
+            "Connection: %s\r\n"
+            "Cache-Control: no-store\r\n"
             "%s"
-            "content-length: %d\r\n\r\n",
+            "Content-Length: %d\r\n\r\n",
             con->include_version ? "readsb_version: "MODES_READSB_VERSION"\r\n" : "",
             con->content_type,
             con->keepalive ? "keep-alive" : "close",
-            options->zstd_encode ? "content-encoding: zstd\r\n" : "",
+            options->zstd_encode ? "Content-Encoding: zstd\r\n" : "",
             content_len);
 
     int hlen = p - header;

From 4065eb486ef611fd4088d505d7d35612aa162f29 Mon Sep 17 00:00:00 2001
From: Aaron Shaw <shawaj@gmail.com>
Date: Wed, 21 Aug 2024 23:55:25 +0100
Subject: [PATCH 35/90] fix: Ubuntu 24.04 package name change

Adds librtlsdr2 as an alternative package version if librtlsdr0 is not available.

As per https://www.debian.org/doc/debian-policy/ch-relationships.html

Closes: #74
---
 debian/control | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/control b/debian/control
index 412aba71..b045f7c9 100644
--- a/debian/control
+++ b/debian/control
@@ -3,7 +3,7 @@ Section: net
 Priority: optional
 Maintainer: Matthias Wirth <matthias.wirth@gmail.com>
 Build-Depends: debhelper(>=9), libusb-1.0-0-dev, libncurses-dev, zlib1g-dev, zlib1g, pkg-config, libzstd-dev, libzstd1
-Build-Depends-Indep: librtlsdr0, librtlsdr-dev
+Build-Depends-Indep: librtlsdr0 | librtlsdr2, librtlsdr-dev
 Standards-Version: 4.4.0.1
 Homepage: https://github.com/wiedehopf/readsb
 Vcs-Git: https://github.com/wiedehopf/readsb.git

From 66a04fe8278be8097636e35ce735a0bc97cab247 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Thu, 22 Aug 2024 18:58:29 +0200
Subject: [PATCH 36/90] make sure SBS positions out of receiver range are not
 used

---
 track.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/track.c b/track.c
index 0df72bf8..3237ad47 100644
--- a/track.c
+++ b/track.c
@@ -2490,8 +2490,13 @@ struct aircraft *trackUpdateFromMessage(struct modesMessage *mm) {
                     old_jaero = 1;
             }
         }
+        if (Modes.maxRange > 0 && Modes.userLocationValid) {
+            mm->receiver_distance = greatcircle(Modes.fUserLat, Modes.fUserLon, mm->decoded_lat, mm->decoded_lon, 0);
+        }
         if (old_jaero) {
             // avoid using already received positions for JAERO input
+        } else if (mm->receiver_distance > Modes.maxRange) {
+            // ignore positions out of receiver range
         } else if (mm->source == SOURCE_MLAT && mm->mlatEPU > 2 * a->mlatEPU
                 && imin((int)(3000.0f * logf((float)mm->mlatEPU / (float)a->mlatEPU)), TRACE_STALE * 3 / 4) > (int64_t) trackDataAge(mm->sysTimestamp, &a->pos_reliable_valid)
                 ) {

From 5da03b53f3fa7e22aed7f220390e266776af7d1e Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Thu, 22 Aug 2024 19:06:08 +0200
Subject: [PATCH 37/90] incrementing version: 3.14.1638

---
 debian/changelog | 4 ++--
 version          | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 7d168803..bded01c7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-readsb (3.14.1637) UNRELEASED; urgency=medium
+readsb (3.14.1638) UNRELEASED; urgency=medium
 
   * In development
 
- -- Matthias Wirth <matthias.wirth@gmail.com>  Sun, 18 Aug 2024 17:51:18 +0200
+ -- Matthias Wirth <matthias.wirth@gmail.com>  Thu, 22 Aug 2024 19:06:08 +0200
diff --git a/version b/version
index 91cee972..d41a082a 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.14.1637
+3.14.1638

From e8352b6ab22f9fcce988bc128eb3c0a644d5c71f Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Thu, 22 Aug 2024 20:13:51 +0200
Subject: [PATCH 38/90] deb building apparently needs fakeroot

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d9709708..340ae775 100644
--- a/README.md
+++ b/README.md
@@ -20,7 +20,7 @@ Or build the package yourself:
 sudo apt update
 sudo apt install --no-install-recommends --no-install-suggests -y \
     git build-essential debhelper libusb-1.0-0-dev \
-    librtlsdr-dev librtlsdr0 pkg-config \
+    librtlsdr-dev librtlsdr0 pkg-config fakeroot \
     libncurses-dev zlib1g-dev zlib1g libzstd-dev libzstd1
 git clone --depth 20 https://github.com/wiedehopf/readsb.git
 cd readsb

From 18ee0ef4d485cd35a7c1fcf7e2dcb1c6b7fb8562 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Fri, 23 Aug 2024 00:04:27 +0200
Subject: [PATCH 39/90] fixup required packages

---
 required_packages | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/required_packages b/required_packages
index b9bd6796..40de59d6 100644
--- a/required_packages
+++ b/required_packages
@@ -1 +1 @@
-gcc make libc6-dev libusb-1.0-0-dev librtlsdr-dev libncurses-dev zlib1g-dev libzstd-dev pkg-config
+gcc make libc6-dev libusb-1.0-0-dev librtlsdr-dev librtlsdr0 libncurses-dev zlib1g zlib1g-dev libzstd1 libzstd-dev pkg-config

From 61b77e1da9eb21335eb32f0008538717e1408d2a Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sat, 24 Aug 2024 21:15:55 +0200
Subject: [PATCH 40/90] don't call getaddrinfo quite as often

---
 net_io.c | 69 +++++++++++++++++++++++++++++++-------------------------
 net_io.h |  1 +
 2 files changed, 39 insertions(+), 31 deletions(-)

diff --git a/net_io.c b/net_io.c
index b72ff1ec..30e4fd81 100644
--- a/net_io.c
+++ b/net_io.c
@@ -530,22 +530,27 @@ static void serviceConnect(struct net_connector *con, int64_t now) {
     // make sure backoff is never too small
     con->backoff = imax(Modes.net_connector_delay_min, con->backoff);
 
-    if (con->try_addr && con->try_addr->ai_next) {
-        // we have another address to try,
-        // iterate the address info linked list
+    if (con->try_addr) {
+        // iterate the address info linked list if we have one
         con->try_addr = con->try_addr->ai_next;
-    } else {
-        // get the address info
-        if (!con->gai_request_in_progress)  {
+    }
+
+    if (!con->try_addr)  {
+        // don't resolve too often
+        if ((!con->addr_info || now - con->lastResolve > 2 * Modes.net_connector_delay) && !con->gai_request_in_progress)  {
             // launch a pthread for async getaddrinfo
-            con->try_addr = NULL;
             if (con->addr_info) {
                 freeaddrinfo(con->addr_info);
                 con->addr_info = NULL;
             }
 
-            con->gai_request_in_progress = 1;
+            pthread_mutex_lock(&con->mutex);
             con->gai_request_done = 0;
+            pthread_mutex_unlock(&con->mutex);
+
+            if (0 && Modes.debug_net) {
+                fprintf(stderr, "%s: calling getaddrinfo for %s port %s\n", con->service->descr, con->address, con->port);
+            }
 
             if (pthread_create(&con->thread, NULL, pthreadGetaddrinfo, con)) {
                 con->next_reconnect = now + Modes.net_connector_delay;
@@ -558,36 +563,38 @@ static void serviceConnect(struct net_connector *con, int64_t now) {
             return;
         }
 
-        // gai request is in progress, let's check if it's done
+        if (con->gai_request_in_progress) {
+            // gai request is in progress, let's check if it's done
 
-        pthread_mutex_lock(&con->mutex);
-        if (!con->gai_request_done) {
-            con->next_reconnect = now + 20;
+            pthread_mutex_lock(&con->mutex);
+            if (!con->gai_request_done) {
+                con->next_reconnect = now + 20;
+                pthread_mutex_unlock(&con->mutex);
+                return;
+            }
             pthread_mutex_unlock(&con->mutex);
-            return;
-        }
-        pthread_mutex_unlock(&con->mutex);
-
-        // gai request is done, join the thread that performed it
-        con->gai_request_in_progress = 0;
 
-        if (pthread_join(con->thread, NULL)) {
-            fprintf(stderr, "%s: pthread_join ERROR for %s port %s: %s\n", con->service->descr, con->address, con->port, strerror(errno));
-            con->next_reconnect = now + Modes.net_connector_delay;
-            return;
-        }
+            con->gai_request_in_progress = 0;
+            // gai request is done, join the thread that performed it
+            if (pthread_join(con->thread, NULL)) {
+                fprintf(stderr, "%s: pthread_join ERROR for %s port %s: %s\n", con->service->descr, con->address, con->port, strerror(errno));
+                con->next_reconnect = now + Modes.net_connector_delay;
+                return;
+            }
 
-        if (con->gai_error) {
-            if (!con->silent_fail) {
-                fprintf(stderr, "%s: Name resolution for %s failed: %s\n", con->service->descr, con->address, gai_strerror(con->gai_error));
+            if (con->gai_error) {
+                if (!con->silent_fail) {
+                    fprintf(stderr, "%s: Name resolution for %s failed: %s\n", con->service->descr, con->address, gai_strerror(con->gai_error));
+                }
+                // limit name resolution attempts via backoff
+                con->next_reconnect = now + con->backoff;
+                con->backoff = imin(Modes.net_connector_delay, 2 * con->backoff);
+                return;
             }
-            // limit name resolution attempts via backoff
-            con->next_reconnect = now + con->backoff;
-            con->backoff = imin(Modes.net_connector_delay, 2 * con->backoff);
-            return;
+            con->lastResolve = now;
+            // SUCCESS, we got the address info
         }
 
-        // SUCCESS, we got the address info
         // start with the first element of the linked list
         con->try_addr = con->addr_info;
     }
diff --git a/net_io.h b/net_io.h
index c4668107..344f1bad 100644
--- a/net_io.h
+++ b/net_io.h
@@ -167,6 +167,7 @@ struct net_connector
     int64_t connect_timeout;
     int64_t lastConnect; // timestamp for last connection establish
     int64_t backoff;
+    int64_t lastResolve;
     char resolved_addr[NI_MAXHOST+3];
     struct addrinfo *addr_info;
     struct addrinfo *try_addr; // pointer walking addr_info list

From 6f9b0b82f90f8a16e29b9f0f1c84faacdb84f7d4 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sun, 25 Aug 2024 20:16:23 +0200
Subject: [PATCH 41/90] add jaero exception to sbs range limit

---
 track.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/track.c b/track.c
index 3237ad47..a3d2c1e2 100644
--- a/track.c
+++ b/track.c
@@ -2495,8 +2495,8 @@ struct aircraft *trackUpdateFromMessage(struct modesMessage *mm) {
         }
         if (old_jaero) {
             // avoid using already received positions for JAERO input
-        } else if (mm->receiver_distance > Modes.maxRange) {
-            // ignore positions out of receiver range
+        } else if (mm->receiver_distance > Modes.maxRange && mm->source != SOURCE_JAERO) {
+            // ignore positions out of receiver range unless it's jaero
         } else if (mm->source == SOURCE_MLAT && mm->mlatEPU > 2 * a->mlatEPU
                 && imin((int)(3000.0f * logf((float)mm->mlatEPU / (float)a->mlatEPU)), TRACE_STALE * 3 / 4) > (int64_t) trackDataAge(mm->sysTimestamp, &a->pos_reliable_valid)
                 ) {

From 123fbf1012384735e24b1c756b8a0381bec230c7 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sun, 25 Aug 2024 20:16:54 +0200
Subject: [PATCH 42/90] incrementing version: 3.14.1639

---
 debian/changelog | 4 ++--
 version          | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index bded01c7..e56d49b5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-readsb (3.14.1638) UNRELEASED; urgency=medium
+readsb (3.14.1639) UNRELEASED; urgency=medium
 
   * In development
 
- -- Matthias Wirth <matthias.wirth@gmail.com>  Thu, 22 Aug 2024 19:06:08 +0200
+ -- Matthias Wirth <matthias.wirth@gmail.com>  Sun, 25 Aug 2024 20:16:54 +0200
diff --git a/version b/version
index d41a082a..ec8858ea 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.14.1638
+3.14.1639

From d6932bb08f357a22682e8af2086131f92600ad02 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Mon, 2 Sep 2024 23:03:19 +0200
Subject: [PATCH 43/90] make log_ppm work with argument 0 as expected

---
 readsb.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/readsb.c b/readsb.c
index a4aca6c4..8292d919 100644
--- a/readsb.c
+++ b/readsb.c
@@ -1924,9 +1924,12 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
                 if (strcasecmp(token[0], "log_ppm") == 0) {
                     if (token[1]) {
                         Modes.devel_log_ppm = atoi(token[1]);
-                    } else {
+                    }
+                    if (Modes.devel_log_ppm == 0) {
                         Modes.devel_log_ppm = -1;
                     }
+                    // setting to -1 to enable due to the following check
+                    // if (Modes.devel_log_ppm && fabs(ppm) > Modes.devel_log_ppm) {
                 }
 
                 if (strcasecmp(token[0], "sbs_override_squawk") == 0 && token[1]) {

From 603523537e6ce35b2f2704971bbb6d2af61002de Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sun, 15 Sep 2024 14:20:33 +0200
Subject: [PATCH 44/90] copy full DB details to aircraft struct

---
 aircraft.c |  4 ++++
 json_out.c | 31 +++++++++++++------------------
 track.h    |  3 +++
 3 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/aircraft.c b/aircraft.c
index 64a74cdf..07f842ab 100644
--- a/aircraft.c
+++ b/aircraft.c
@@ -692,11 +692,15 @@ void updateTypeReg(struct aircraft *a) {
         memcpy(a->registration, d->registration, sizeof(a->registration));
         memcpy(a->typeCode, d->typeCode, sizeof(a->typeCode));
         memcpy(a->typeLong, d->typeLong, sizeof(a->typeLong));
+        memcpy(a->ownOp, d->ownOp, sizeof(a->ownOp));
+        memcpy(a->year, d->year, sizeof(a->year));
         a->dbFlags = d->dbFlags;
     } else {
         memset(a->registration, 0, sizeof(a->registration));
         memset(a->typeCode, 0, sizeof(a->typeCode));
         memset(a->typeLong, 0, sizeof(a->typeLong));
+        memset(a->ownOp, 0, sizeof(a->ownOp));
+        memset(a->year, 0, sizeof(a->year));
         a->dbFlags = 0;
     }
     uint32_t i = a->addr;
diff --git a/json_out.c b/json_out.c
index 32f91d13..eefbaa75 100644
--- a/json_out.c
+++ b/json_out.c
@@ -658,15 +658,12 @@ char *sprintAircraftObject(char *p, char *end, struct aircraft *a, int64_t now,
             }
 
             if (Modes.jsonLongtype) {
-                dbEntry *e = dbGet(a->addr, Modes.dbIndex);
-                if (e) {
-                    if (e->typeLong[0])
-                        p = safe_snprintf(p, end, ",\"desc\":\"%.*s\"", (int) sizeof(e->typeLong), e->typeLong);
-                    if (e->ownOp[0])
-                        p = safe_snprintf(p, end, ",\n\"ownOp\":\"%.*s\"", (int) sizeof(e->ownOp), e->ownOp);
-                    if (e->year[0])
-                        p = safe_snprintf(p, end, ",\n\"year\":\"%.*s\"", (int) sizeof(e->year), e->year);
-                }
+                if (a->typeLong[0])
+                    p = safe_snprintf(p, end, ",\"desc\":\"%.*s\"", (int) sizeof(a->typeLong), a->typeLong);
+                if (a->ownOp[0])
+                    p = safe_snprintf(p, end, ",\n\"ownOp\":\"%.*s\"", (int) sizeof(a->ownOp), a->ownOp);
+                if (a->year[0])
+                    p = safe_snprintf(p, end, ",\n\"year\":\"%.*s\"", (int) sizeof(a->year), a->year);
             }
         }
 
@@ -1770,15 +1767,13 @@ struct char_buffer generateTraceJson(struct aircraft *a, traceBuffer tb, int sta
             dbFlags &= ~(1 << 7);
             p = safe_snprintf(p, end, ",\n\"dbFlags\":%u", dbFlags);
         }
-        dbEntry *e = dbGet(a->addr, Modes.dbIndex);
-        if (e) {
-            if (e->typeLong[0])
-                p = safe_snprintf(p, end, ",\n\"desc\":\"%.*s\"", (int) sizeof(e->typeLong), e->typeLong);
-            if (e->ownOp[0])
-                p = safe_snprintf(p, end, ",\n\"ownOp\":\"%.*s\"", (int) sizeof(e->ownOp), e->ownOp);
-            if (e->year[0])
-                p = safe_snprintf(p, end, ",\n\"year\":\"%.*s\"", (int) sizeof(e->year), e->year);
-        }
+
+        if (a->typeLong[0])
+            p = safe_snprintf(p, end, ",\"desc\":\"%.*s\"", (int) sizeof(a->typeLong), a->typeLong);
+        if (a->ownOp[0])
+            p = safe_snprintf(p, end, ",\n\"ownOp\":\"%.*s\"", (int) sizeof(a->ownOp), a->ownOp);
+        if (a->year[0])
+            p = safe_snprintf(p, end, ",\n\"year\":\"%.*s\"", (int) sizeof(a->year), a->year);
         if (p == regInfo)
             p = safe_snprintf(p, end, ",\n\"noRegData\":true");
     }
diff --git a/track.h b/track.h
index baba4df7..20574eeb 100644
--- a/track.h
+++ b/track.h
@@ -567,6 +567,9 @@ struct aircraft
 
   char zeroStart;
 
+  char ownOp[64];
+  char year[4];
+
   float messageRate;
   uint16_t messageRateAcc[MESSAGE_RATE_CALC_POINTS];
   int64_t nextMessageRateCalc;

From 78f140cc2f9b1ccb0dd622b27612cb7f9b7e991c Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Mon, 16 Sep 2024 17:10:33 +0200
Subject: [PATCH 45/90] couple of fixes to make ModeS beast work properly

make modesbeast read nonblocking
read returning zero not an error for serial clients

--devel=debugSerial for some debugging for serial clients

properly close serial client without unnecessary errors
---
 net_io.c    | 62 ++++++++++++++++++++++++++++++++++++++++-------------
 net_io.h    |  1 +
 readsb.c    |  3 +++
 readsb.h    |  1 +
 sdr_beast.c |  8 +++++--
 5 files changed, 58 insertions(+), 17 deletions(-)

diff --git a/net_io.c b/net_io.c
index 30e4fd81..aa09d4d6 100644
--- a/net_io.c
+++ b/net_io.c
@@ -318,6 +318,7 @@ static struct client *createSocketClient(struct net_service *service, int fd) {
     if ((c->fd == Modes.beast_fd) && (Modes.sdr_type == SDR_MODESBEAST || Modes.sdr_type == SDR_GNS)) {
         /* Message from a local connected Modes-S beast or GNS5894 are passed off the internet */
         c->remote = 0;
+        c->serial = 1;
     }
 
     //fprintf(stderr, "c->receiverId: %016"PRIx64"\n", c->receiverId);
@@ -1045,11 +1046,6 @@ void modesInitNet(void) {
     }
     serviceListen(Modes.beast_in_service, Modes.net_bind_address, Modes.net_input_beast_ports, Modes.net_epfd);
 
-    /* Beast input from local Modes-S Beast via USB */
-    if (Modes.sdr_type == SDR_MODESBEAST || Modes.sdr_type == SDR_GNS) {
-        Modes.serial_client = createSocketClient(Modes.beast_in_service, Modes.beast_fd);
-    }
-
     /* Planefinder input via network */
     planefinder_in = serviceInit(&Modes.services_in, "Planefinder TCP input", NULL, no_heartbeat, no_heartbeat, READ_MODE_PLANEFINDER, NULL, decodePfMessage);
     serviceListen(planefinder_in, Modes.net_bind_address, Modes.net_input_planefinder_ports, Modes.net_epfd);
@@ -1207,7 +1203,13 @@ static void modesCloseClient(struct client *c) {
     }
 
     epoll_ctl(Modes.net_epfd, EPOLL_CTL_DEL, c->fd, &c->epollEvent);
-    anetCloseSocket(c->fd);
+    if (c->serial) {
+        if (close(c->fd) < 0) {
+            fprintf(stderr, "Serial client close error: %s\n", strerror(errno));
+        }
+    } else {
+        anetCloseSocket(c->fd);
+    }
     c->service->connections--;
     Modes.modesClientCount--;
     if (c->service->writer) {
@@ -4438,7 +4440,15 @@ static int readClient(struct client *c, int64_t now) {
         nread = recv(c->fd, c->buf + c->buflen, left, 0);
     } else {
         // read instead of recv for modesbeast / gns-hulc ....
+        if (0 && Modes.debug_serial) {
+            fprintTimePrecise(stderr, mstime());
+            fprintf(stderr, " serial read ... fd: %d maxbytes: %d\n", c->fd, left);
+        }
         nread = read(c->fd, c->buf + c->buflen, left);
+        if (nread > 0 && Modes.debug_serial) {
+            fprintTimePrecise(stderr, mstime());
+            fprintf(stderr, " serial read return value: %d\n", nread);
+        }
     }
     int err = errno;
 
@@ -4456,6 +4466,9 @@ static int readClient(struct client *c, int64_t now) {
             return 0;
         }
         // Other errors
+        if (c->serial) {
+            fprintf(stderr, "Serial client read error: %s\n", strerror(err));
+        }
         if (Modes.debug_net) {
             fprintf(stderr, "%s: Socket Error: %s: %s port %s (fd %d, SendQ %d, RecvQ %d)\n",
                     c->service->descr, strerror(err), c->host, c->port,
@@ -4467,6 +4480,11 @@ static int readClient(struct client *c, int64_t now) {
 
     // End of file
     if (nread == 0) {
+        if (c->serial) {
+            // for serial this just means we're doing non-blocking reads and there are no bytes available
+            return 0;
+        }
+
         if (c->con) {
             if (Modes.synthetic_now) {
                 Modes.synthetic_now = 0;
@@ -5392,18 +5410,17 @@ void modesNetPeriodicWork(void) {
     dump_beast_check(now);
 
     int64_t wait_ms;
-    if (Modes.serial_client) {
-        wait_ms = 20;
-    } else if (Modes.sdr_type != SDR_NONE) {
+    if (Modes.sdr_type != SDR_NONE && Modes.sdr_type != SDR_MODESBEAST && Modes.sdr_type != SDR_GNS) {
         // NO WAIT WHEN USING AN SDR !! IMPORTANT !!
         wait_ms = 0;
-    } else if (Modes.net_only) {
+    } else {
         // wait in net-only mode (unless we get network packets, that wakes the wait immediately)
         wait_ms = imax(0, check_flush - now); // modify wait for next flush timer
         wait_ms = imin(wait_ms, Modes.next_reconnect_callback - now); // modify wait for reconnect callback timer
         wait_ms = imax(wait_ms, 0); // don't allow negative values
-    } else {
-        wait_ms = 0;
+        if (Modes.debug_serial) {
+            wait_ms = 1000;
+        }
     }
 
     // unlock decode mutex for waiting in handleEpoll
@@ -5416,7 +5433,9 @@ void modesNetPeriodicWork(void) {
     Modes.services_in.event_progress = 0;
     Modes.services_out.event_progress = 0;
 
-    //fprintTimePrecise(stderr, now); fprintf(stderr, " event count %d wait_ms %d\n", Modes.net_event_count, (int) wait_ms);
+    if (Modes.debug_serial && Modes.net_event_count > 0) {
+        fprintTimePrecise(stderr, mstime()); fprintf(stderr, " event count %d wait_ms %d\n", Modes.net_event_count, (int) wait_ms);
+    }
 
     if (0 && Modes.net_event_count > 0) {
         fprintTimePrecise(stderr, now); fprintf(stderr, " event count %d wait_ms %d\n", Modes.net_event_count, (int) wait_ms);
@@ -5456,10 +5475,23 @@ void modesNetPeriodicWork(void) {
         timespec_add_elapsed(&before, &after, &Modes.stats_current.background_cpu);
     }
 
-    if (Modes.serial_client) {
+    /* Beast input from local Modes-S Beast via USB */
+    if (Modes.sdrInitialized && (Modes.sdr_type == SDR_MODESBEAST || Modes.sdr_type == SDR_GNS)) {
+        if (!Modes.serial_client) {
+            if (Modes.debug_serial) {
+                fprintTimePrecise(stderr, mstime());
+                fprintf(stderr, " serial: creating socket client ... \n");
+            }
+            Modes.serial_client = createSocketClient(Modes.beast_in_service, Modes.beast_fd);
+            if (Modes.debug_serial) {
+                fprintTimePrecise(stderr, mstime());
+                fprintf(stderr, " serial: creating socket client ... done\n");
+            }
+        }
         if (Modes.serial_client->service) {
             modesReadFromClient(Modes.serial_client, mb);
-        } else {
+        }
+        if (!Modes.serial_client->service) {
             fprintf(stderr, "Serial client closed unexpectedly, exiting!\n");
             setExit(2);
         }
diff --git a/net_io.h b/net_io.h
index 344f1bad..7eb5e92f 100644
--- a/net_io.h
+++ b/net_io.h
@@ -100,6 +100,7 @@ struct client
     int fd; // File descriptor
     int8_t bufferToProcess;
     int8_t remote;
+    int8_t serial;
     int8_t bContinue;
     int8_t discard;
     int8_t processing;
diff --git a/readsb.c b/readsb.c
index 8292d919..4de6bafd 100644
--- a/readsb.c
+++ b/readsb.c
@@ -1972,6 +1972,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
                 if (strcasecmp(token[0], "debugGPS") == 0) {
                     Modes.debug_gps = 1;
                 }
+                if (strcasecmp(token[0], "debugSerial") == 0) {
+                    Modes.debug_serial = 1;
+                }
                 if (strcasecmp(token[0], "debugZstd") == 0) {
                     Modes.debug_zstd = 1;
                 }
diff --git a/readsb.h b/readsb.h
index 06064c11..533b59f1 100644
--- a/readsb.h
+++ b/readsb.h
@@ -639,6 +639,7 @@ struct _Modes
     int8_t mode_ac; // Enable decoding of SSR Modes A & C
     int8_t mode_ac_auto; // allow toggling of A/C by Beast commands
     int8_t debug_net;
+    int8_t debug_serial;
     int8_t debug_flush;
     int8_t debug_no_discard;
     int8_t debug_nextra;
diff --git a/sdr_beast.c b/sdr_beast.c
index d3e43056..2893f7d8 100644
--- a/sdr_beast.c
+++ b/sdr_beast.c
@@ -136,7 +136,9 @@ bool beastOpen(void) {
     struct termios tios;
     speed_t baud = B3000000;
 
-    Modes.beast_fd = open(Modes.beast_serial, O_RDWR | O_NOCTTY);
+    int flags = O_RDWR | O_NOCTTY;
+    //flags |= O_NONBLOCK;
+    Modes.beast_fd = open(Modes.beast_serial, flags);
     if (Modes.beast_fd < 0) {
         fprintf(stderr, "Failed to open serial device %s: %s\n",
                 Modes.beast_serial, strerror(errno));
@@ -153,7 +155,8 @@ bool beastOpen(void) {
     tios.c_oflag = 0;
     tios.c_lflag = 0;
     tios.c_cflag = CS8 | CRTSCTS;
-    tios.c_cc[VMIN] = 11;
+    //tios.c_cc[VMIN] = 11; // read returns when a minimum of 11 characters are available
+    tios.c_cc[VMIN] = 0; // polling read with vtime 0
     tios.c_cc[VTIME] = 0;
 
     if (Modes.sdr_type == SDR_GNS) {
@@ -190,6 +193,7 @@ bool beastOpen(void) {
 
     if (Modes.sdr_type == SDR_MODESBEAST) {
         /* set options */
+        beastSetOption('B'); /* set classic beast mode */
         beastSetOption('C'); /* use binary format */
         beastSetOption('H'); /* RTS enabled */
 

From 4b18685675aa0847e750a6a627a2d90cc08b2026 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Mon, 16 Sep 2024 17:11:25 +0200
Subject: [PATCH 46/90] incrementing version: 3.14.1640

---
 debian/changelog | 4 ++--
 version          | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index e56d49b5..c7411c57 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-readsb (3.14.1639) UNRELEASED; urgency=medium
+readsb (3.14.1640) UNRELEASED; urgency=medium
 
   * In development
 
- -- Matthias Wirth <matthias.wirth@gmail.com>  Sun, 25 Aug 2024 20:16:54 +0200
+ -- Matthias Wirth <matthias.wirth@gmail.com>  Mon, 16 Sep 2024 17:11:25 +0200
diff --git a/version b/version
index ec8858ea..99cb85bf 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.14.1639
+3.14.1640

From 2ab35ba545c65815fb742a0d4820f6b8c63d137c Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Tue, 17 Sep 2024 09:41:25 +0200
Subject: [PATCH 47/90] cleanup df18 exception handling

---
 aircraft.c | 268 +++++++++++++++++++++++++++--------------------------
 json_out.c |   8 +-
 track.c    |   2 +-
 track.h    |   7 +-
 4 files changed, 145 insertions(+), 140 deletions(-)

diff --git a/aircraft.c b/aircraft.c
index 07f842ab..ad4163aa 100644
--- a/aircraft.c
+++ b/aircraft.c
@@ -1,5 +1,7 @@
 #include "readsb.h"
 
+static int isMilRange(uint32_t i);
+
 static inline uint32_t dbHash(uint32_t addr) {
     return addrHash(addr, DB_HASH_BITS);
 }
@@ -413,6 +415,32 @@ static inline int nextToken(char delim, char **sot, char **eot, char **eol) {
     return 1;
 }
 
+static int is_df18_exception(uint32_t addr) {
+    switch (addr) {
+        case 0xa08508:
+        case 0xab33a0:
+        case 0xa7d24c:
+        case 0xa6e2cd:
+        case 0xaa8fca:
+        case 0xac808b:
+        case 0x48f6f7:
+        case 0x7cbc3d:
+        case 0x7c453a:
+        case 0x401cf9:
+        case 0x40206a:
+        case 0xa3227d:
+        case 0x478676:
+        case 0x40389d:
+        case 0x405acf:
+        case 0xc82452:
+        case 0x40334a:
+            return 1;
+        default:
+            return 0;
+    }
+}
+
+
 // meant to be used with this DB: https://raw.githubusercontent.com/wiedehopf/tar1090-db/csv/aircraft.csv.gz
 int dbUpdate(int64_t now) {
     static int64_t next_db_check;
@@ -470,11 +498,7 @@ int dbUpdate(int64_t now) {
         goto DBU0;
     }
 
-    static const uint32_t df18_exceptions[] = { 0xa08508, 0xab33a0, 0xa7d24c, 0xa6e2cd, 0xaa8fca, 0xac808b, 0x48f6f7, 0x7cbc3d, 0x7c453a, 0x401cf9, 0x40206a, 0xa3227d, 0x478676, 0x40389d, 0x405acf, 0xc82452, 0x40334a };
-    int num_df18_exceptions = sizeof(df18_exceptions) / sizeof(uint32_t);
-
     int alloc = 0;
-    alloc += num_df18_exceptions;
 
     if (1) {
         for (uint32_t i = 0; i < cb.len; i++) {
@@ -560,27 +584,6 @@ int dbUpdate(int64_t now) {
         dbPut(curr->addr, Modes.db2Index, curr);
     }
 
-    for (int k = 0; k < num_df18_exceptions; k++) {
-        uint32_t addr = df18_exceptions[k];
-        //fprintf(stderr, "df18_exceptions: %06x\n", addr);
-        dbEntry *curr = dbGet(addr, Modes.db2Index);
-        if (!curr) {
-            curr = &Modes.db2[i];
-            memset(curr, 0, sizeof(dbEntry));
-            curr->addr = addr;
-
-            i++; // increment db array index
-                 // add to hashtable
-            dbPut(curr->addr, Modes.db2Index, curr);
-        }
-        curr->dbFlags |= (1 << 7);
-
-        dbEntry *d = dbGet(addr, Modes.db2Index);
-        if (!d) {
-            fprintf(stderr, "<3>df18_exceptions %06x no db entry\n", addr);
-        }
-    }
-
     if (i < 1) {
         fprintf(stderr, "db update error: DB has no entries, maybe old / incorrect format?!\n");
         goto DBU0;
@@ -703,111 +706,116 @@ void updateTypeReg(struct aircraft *a) {
         memset(a->year, 0, sizeof(a->year));
         a->dbFlags = 0;
     }
-    uint32_t i = a->addr;
-    if (
-            false
-            // us military
-            //adf7c8-adf7cf = united states mil_5(uf)
-            //adf7d0-adf7df = united states mil_4(uf)
-            //adf7e0-adf7ff = united states mil_3(uf)
-            //adf800-adffff = united states mil_2(uf)
-            //ae0000-afffff = united states mil_1(uf)
-            || (i >= 0xadf7c8 && i <= 0xafffff)
-
-            //010070-01008f = egypt_mil
-            || (i >= 0x010070 && i <= 0x01008f)
-
-            //0a4000-0a4fff = algeria mil(ap)
-            || (i >= 0x0a4000 && i <= 0x0a4fff)
-
-            //33ff00-33ffff = italy mil(iy)
-            || (i >= 0x33ff00 && i <= 0x33ffff)
-
-            //350000-37ffff = spain mil(sp)
-            || (i >= 0x350000 && i <= 0x37ffff)
-
-            //3aa000-3affff = france mil_1(fs)
-            || (i >= 0x3aa000 && i <= 0x3affff)
-            //3b7000-3bffff = france mil_2(fs)
-            || (i >= 0x3b7000 && i <= 0x3bffff)
-
-            //3ea000-3ebfff = germany mil_1(df)
-            || (i >= 0x3ea000 && i <= 0x3ebfff)
-            //3f4000-3f7fff = germany mil_2(df)
-            //3f8000-3fbfff = germany mil_3(df)
-            || (i >= 0x3f4000 && i <= 0x3fbfff)
-
-            //400000-40003f = united kingdom mil_1(ra)
-            || (i >= 0x400000 && i <= 0x40003f)
-            //43c000-43cfff = united kingdom mil(ra)
-            || (i >= 0x43c000 && i <= 0x43cfff)
-
-            //444000-446fff = austria mil(aq)
-            || (i >= 0x444000 && i <= 0x446fff)
-
-            //44f000-44ffff = belgium mil(bc)
-            || (i >= 0x44f000 && i <= 0x44ffff)
-
-            //457000-457fff = bulgaria mil(bu)
-            || (i >= 0x457000 && i <= 0x457fff)
-
-            //45f400-45f4ff = denmark mil(dg)
-            || (i >= 0x45f400 && i <= 0x45f4ff)
-
-            //468000-4683ff = greece mil(gc)
-            || (i >= 0x468000 && i <= 0x4683ff)
-
-            //473c00-473c0f = hungary mil(hm)
-            || (i >= 0x473c00 && i <= 0x473c0f)
-
-            //478100-4781ff = norway mil(nn)
-            || (i >= 0x478100 && i <= 0x4781ff)
-            //480000-480fff = netherlands mil(nm)
-            || (i >= 0x480000 && i <= 0x480fff)
-            //48d800-48d87f = poland mil(po)
-            || (i >= 0x48d800 && i <= 0x48d87f)
-            //497c00-497cff = portugal mil(pu)
-            || (i >= 0x497c00 && i <= 0x497cff)
-            //498420-49842f = czech republic mil(ct)
-            || (i >= 0x498420 && i <= 0x49842f)
-
-            //4b7000-4b7fff = switzerland mil(su)
-            || (i >= 0x4b7000 && i <= 0x4b7fff)
-            //4b8200-4b82ff = turkey mil(tq)
-            || (i >= 0x4b8200 && i <= 0x4b82ff)
-
-            //506f32-506fff = slovenia mil(sj)
-            //|| (i >= 0x506f32 && i <= 0x506fff)
-
-            //70c070-70c07f = oman mil(on)
-            || (i >= 0x70c070 && i <= 0x70c07f)
-
-            //710258-71025f = saudi arabia mil_1(sx)
-            //710260-71027f = saudi arabia mil_2(sx)
-            //710280-71028f = saudi arabia mil_3(sx)
-            || (i >= 0x710258 && i <= 0x71028f)
-            //710380-71039f = saudi arabia mil_4(sx)
-            || (i >= 0x710380 && i <= 0x71039f)
-
-            //738a00-738aff = israel mil(iz)
-            || (i >= 0x738a00 && i <= 0x738aff)
-
-            //7cf800-7cfaff australia mil
-            || (i >= 0x7cf800 && i <= 0x7cfaff)
-
-            //800200-8002ff = india mil(im)
-            || (i >= 0x800200 && i <= 0x8002ff)
-
-            //c20000-c3ffff = canada mil(cb)
-            || (i >= 0xc20000 && i <= 0xc3ffff)
-
-            //e40000-e41fff = brazil mil(bq)
-            || (i >= 0xe40000 && i <= 0xe41fff)
-
-            //e80600-e806ff = chile mil(cq)
-            //|| (i >= 0xe80600 && i <= 0xe806ff)
-            // disabled due to civilian aircraft in hex range
-    ) {
+    if (is_df18_exception(a->addr)) {
+        a->is_df18_exception = 1;
+    }
+    if (isMilRange(a->addr)) {
         a->dbFlags |= 1;
     }
 }
+static int isMilRange(uint32_t i) {
+    return
+        false
+        // us military
+        //adf7c8-adf7cf = united states mil_5(uf)
+        //adf7d0-adf7df = united states mil_4(uf)
+        //adf7e0-adf7ff = united states mil_3(uf)
+        //adf800-adffff = united states mil_2(uf)
+        //ae0000-afffff = united states mil_1(uf)
+        || (i >= 0xadf7c8 && i <= 0xafffff)
+
+        //010070-01008f = egypt_mil
+        || (i >= 0x010070 && i <= 0x01008f)
+
+        //0a4000-0a4fff = algeria mil(ap)
+        || (i >= 0x0a4000 && i <= 0x0a4fff)
+
+        //33ff00-33ffff = italy mil(iy)
+        || (i >= 0x33ff00 && i <= 0x33ffff)
+
+        //350000-37ffff = spain mil(sp)
+        || (i >= 0x350000 && i <= 0x37ffff)
+
+        //3aa000-3affff = france mil_1(fs)
+        || (i >= 0x3aa000 && i <= 0x3affff)
+        //3b7000-3bffff = france mil_2(fs)
+        || (i >= 0x3b7000 && i <= 0x3bffff)
+
+        //3ea000-3ebfff = germany mil_1(df)
+        || (i >= 0x3ea000 && i <= 0x3ebfff)
+        //3f4000-3f7fff = germany mil_2(df)
+        //3f8000-3fbfff = germany mil_3(df)
+        || (i >= 0x3f4000 && i <= 0x3fbfff)
+
+        //400000-40003f = united kingdom mil_1(ra)
+        || (i >= 0x400000 && i <= 0x40003f)
+        //43c000-43cfff = united kingdom mil(ra)
+        || (i >= 0x43c000 && i <= 0x43cfff)
+
+        //444000-446fff = austria mil(aq)
+        || (i >= 0x444000 && i <= 0x446fff)
+
+        //44f000-44ffff = belgium mil(bc)
+        || (i >= 0x44f000 && i <= 0x44ffff)
+
+        //457000-457fff = bulgaria mil(bu)
+        || (i >= 0x457000 && i <= 0x457fff)
+
+        //45f400-45f4ff = denmark mil(dg)
+        || (i >= 0x45f400 && i <= 0x45f4ff)
+
+        //468000-4683ff = greece mil(gc)
+        || (i >= 0x468000 && i <= 0x4683ff)
+
+        //473c00-473c0f = hungary mil(hm)
+        || (i >= 0x473c00 && i <= 0x473c0f)
+
+        //478100-4781ff = norway mil(nn)
+        || (i >= 0x478100 && i <= 0x4781ff)
+        //480000-480fff = netherlands mil(nm)
+        || (i >= 0x480000 && i <= 0x480fff)
+        //48d800-48d87f = poland mil(po)
+        || (i >= 0x48d800 && i <= 0x48d87f)
+        //497c00-497cff = portugal mil(pu)
+        || (i >= 0x497c00 && i <= 0x497cff)
+        //498420-49842f = czech republic mil(ct)
+        || (i >= 0x498420 && i <= 0x49842f)
+
+        //4b7000-4b7fff = switzerland mil(su)
+        || (i >= 0x4b7000 && i <= 0x4b7fff)
+        //4b8200-4b82ff = turkey mil(tq)
+        || (i >= 0x4b8200 && i <= 0x4b82ff)
+
+        //506f32-506fff = slovenia mil(sj)
+        //|| (i >= 0x506f32 && i <= 0x506fff)
+
+        //70c070-70c07f = oman mil(on)
+        || (i >= 0x70c070 && i <= 0x70c07f)
+
+        //710258-71025f = saudi arabia mil_1(sx)
+        //710260-71027f = saudi arabia mil_2(sx)
+        //710280-71028f = saudi arabia mil_3(sx)
+        || (i >= 0x710258 && i <= 0x71028f)
+        //710380-71039f = saudi arabia mil_4(sx)
+        || (i >= 0x710380 && i <= 0x71039f)
+
+        //738a00-738aff = israel mil(iz)
+        || (i >= 0x738a00 && i <= 0x738aff)
+
+        //7cf800-7cfaff australia mil
+        || (i >= 0x7cf800 && i <= 0x7cfaff)
+
+        //800200-8002ff = india mil(im)
+        || (i >= 0x800200 && i <= 0x8002ff)
+
+        //c20000-c3ffff = canada mil(cb)
+        || (i >= 0xc20000 && i <= 0xc3ffff)
+
+        //e40000-e41fff = brazil mil(bq)
+        || (i >= 0xe40000 && i <= 0xe41fff)
+
+        //e80600-e806ff = chile mil(cq)
+        //|| (i >= 0xe80600 && i <= 0xe806ff)
+        // disabled due to civilian aircraft in hex range
+        ;
+}
diff --git a/json_out.c b/json_out.c
index eefbaa75..88deaffe 100644
--- a/json_out.c
+++ b/json_out.c
@@ -652,9 +652,7 @@ char *sprintAircraftObject(char *p, char *end, struct aircraft *a, int64_t now,
             if (a->typeCode[0])
                 p = safe_snprintf(p, end, ",\"t\":\"%.*s\"", (int) sizeof(a->typeCode), a->typeCode);
             if (a->dbFlags) {
-                uint32_t dbFlags = a->dbFlags;
-                dbFlags &= ~(1 << 7);
-                p = safe_snprintf(p, end, ",\"dbFlags\":%u", dbFlags);
+                p = safe_snprintf(p, end, ",\"dbFlags\":%u", a->dbFlags);
             }
 
             if (Modes.jsonLongtype) {
@@ -1763,9 +1761,7 @@ struct char_buffer generateTraceJson(struct aircraft *a, traceBuffer tb, int sta
             p = safe_snprintf(p, end, ",\n\"t\":\"%.*s\"", (int) sizeof(a->typeCode), a->typeCode);
         }
         if (a->typeCode[0] || a->registration[0] || a->dbFlags) {
-            uint32_t dbFlags = a->dbFlags;
-            dbFlags &= ~(1 << 7);
-            p = safe_snprintf(p, end, ",\n\"dbFlags\":%u", dbFlags);
+            p = safe_snprintf(p, end, ",\n\"dbFlags\":%u", a->dbFlags);
         }
 
         if (a->typeLong[0])
diff --git a/track.c b/track.c
index a3d2c1e2..731f1a99 100644
--- a/track.c
+++ b/track.c
@@ -1111,7 +1111,7 @@ static void setPosition(struct aircraft *a, struct modesMessage *mm, int64_t now
         if (
                 (valid_elapsed > 10 * MINUTES || override_elapsed < 10 * MINUTES)
                 && (mm->msgtype == 17 || (mm->addrtype == ADDR_ADSB_ICAO_NT && mm->cpr_type != CPR_SURFACE
-                        && !(a->dbFlags & (1 << 7)) && ((a->addr >= 0xa00000 && a->addr <= 0xafffff) || (a->dbFlags & (1 << 0))) ))
+                        && !a->is_df18_exception && ((a->addr >= 0xa00000 && a->addr <= 0xafffff) || (a->dbFlags & (1 << 0))) ))
                 && mm->cpr_valid
                 && status_elapsed > 5 * MINUTES
            ) {
diff --git a/track.h b/track.h
index 20574eeb..90d0df67 100644
--- a/track.h
+++ b/track.h
@@ -464,7 +464,8 @@ struct aircraft
   uint32_t localCPR_allow_ac_rel : 1; // allow local cpr relative to last known aircraft location
   // 24 bit
   uint32_t last_message_crc_fixed : 1;
-  uint32_t padding_b : 7;
+  uint32_t is_df18_exception : 1;
+  uint32_t padding_b : 6;
   // 32 bit !!
 
   // ----
@@ -516,8 +517,7 @@ struct aircraft
   double lonReliable; // last reliable position based on json_reliable threshold
   char typeCode[4];
   char registration[12];
-  char typeLong[63];
-  uint8_t dbFlags;
+  char typeLong[64];
   uint16_t receiverIds[RECEIVERIDBUFFER]; // RECEIVERIDBUFFER = 12
 
   int64_t next_reduce_forward_status;
@@ -569,6 +569,7 @@ struct aircraft
 
   char ownOp[64];
   char year[4];
+  uint16_t dbFlags;
 
   float messageRate;
   uint16_t messageRateAcc[MESSAGE_RATE_CALC_POINTS];

From 85badfbcff29893f12c13e57946257c70aae7f57 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Tue, 17 Sep 2024 12:39:57 +0200
Subject: [PATCH 48/90] streamline zlib gzbuffer things

---
 globe_index.c | 2 ++
 json_out.c    | 6 +-----
 util.c        | 8 +++++---
 util.h        | 2 +-
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/globe_index.c b/globe_index.c
index d85db377..6ece463a 100644
--- a/globe_index.c
+++ b/globe_index.c
@@ -3465,6 +3465,8 @@ void writeRangeDirs() {
     char pathbuf[PATH_MAX];
     snprintf(pathbuf, PATH_MAX, "%s/rangeDirs.gz", Modes.state_dir);
     gzFile gzfp = gzopen(pathbuf, "wb");
+    if (gzbuffer(gzfp, GZBUFFER_BIG) < 0)
+        fprintf(stderr, "gzbuffer fail");
     if (gzfp) {
         writeGz(gzfp, &Modes.lastRangeDirHour, sizeof(Modes.lastRangeDirHour), pathbuf);
         writeGz(gzfp, Modes.rangeDirs, sizeof(Modes.rangeDirs), pathbuf);
diff --git a/json_out.c b/json_out.c
index 88deaffe..04d776d1 100644
--- a/json_out.c
+++ b/json_out.c
@@ -2003,11 +2003,7 @@ static inline __attribute__((always_inline)) struct char_buffer writeJsonTo (con
             goto error_1;
         }
 
-        int gBufSize = 128 * 1024;
-        if (len < 16 * 1024) {
-            gBufSize = 16 * 1024;
-        }
-        gzbuffer(gzfp, gBufSize);
+        gzbuffer(gzfp, GZBUFFER_BIG);
 
         int name_len = strlen(file);
         if (name_len > 8 && strcmp("binCraft", file + (name_len - 8)) == 0) {
diff --git a/util.c b/util.c
index 11a347e9..45f13fd5 100644
--- a/util.c
+++ b/util.c
@@ -379,9 +379,8 @@ struct char_buffer readWholeGz(gzFile gzfp, char *errorContext) {
     struct char_buffer cb = {0};
     if (gzbuffer(gzfp, GZBUFFER_BIG) < 0) {
         fprintf(stderr, "reading %s: gzbuffer fail!\n", errorContext);
-        return cb;
     }
-    int alloc = 8 * 1024 * 1024;
+    int alloc = 1 * 1024 * 1024;
     cb.buffer = cmalloc(alloc);
     if (!cb.buffer) {
         fprintf(stderr, "reading %s: readWholeGz alloc fail!\n", errorContext);
@@ -397,7 +396,7 @@ struct char_buffer readWholeGz(gzFile gzfp, char *errorContext) {
         toRead -= res;
         if (toRead == 0) {
             toRead = alloc;
-            alloc += toRead;
+            alloc *= 2;
             char *oldBuffer = cb.buffer;
             cb.buffer = realloc(cb.buffer, alloc);
             if (!cb.buffer) {
@@ -821,6 +820,9 @@ void gzipFile(char *filename) {
         perror(fileGz);
         return;
     }
+
+    gzbuffer(gzfp, GZBUFFER_BIG);
+
     int res = gzsetparams(gzfp, 9, Z_DEFAULT_STRATEGY);
     if (res < 0) {
         fprintf(stderr, "gzsetparams fail: %d", res);
diff --git a/util.h b/util.h
index ddc82824..5362a877 100644
--- a/util.h
+++ b/util.h
@@ -26,7 +26,7 @@
 
 #define CHECK_APPROXIMATIONS (0)
 
-#define GZBUFFER_BIG (1 * 1024 * 1024)
+#define GZBUFFER_BIG (512 * 1024)
 
 #include <stdint.h>
 

From 85cab05e44e11f9b8fb56d9cbea6647fd82e8aa2 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Tue, 17 Sep 2024 12:57:38 +0200
Subject: [PATCH 49/90] prepare aircraft db change / cleanup

---
 aircraft.c | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/aircraft.c b/aircraft.c
index ad4163aa..4eb380e1 100644
--- a/aircraft.c
+++ b/aircraft.c
@@ -490,7 +490,7 @@ int dbUpdate(int64_t now) {
 
     cb = readWholeGz(gzfp, filename);
     if (!cb.buffer) {
-        fprintf(stderr, "readWholeGz failed.\n");
+        fprintf(stderr, "database read failed due to readWholeGz.\n");
         goto DBU0;
     }
     if (cb.len < 1000) {
@@ -498,23 +498,23 @@ int dbUpdate(int64_t now) {
         goto DBU0;
     }
 
+    // reallocate so we don't waste memory
+    char *oldBuffer = cb.buffer;
+    cb.len++; // for adding zero termination
+    cb.buffer = realloc(cb.buffer, cb.len);
+    if (!cb.buffer) {
+        fprintf(stderr, "database read failed due to realloc\n");
+        sfree(oldBuffer);
+        goto DBU0;
+    }
+    cb.buffer[cb.len - 1] = '\0'; // zero terminate for good measure
+
     int alloc = 0;
 
-    if (1) {
-        for (uint32_t i = 0; i < cb.len; i++) {
-            if (cb.buffer[i] == '\n')
-                alloc++;
-        }
-    } else {
-        // long live SSE and stuff, memchr reigns supreme
-        unsigned char *p = (unsigned char *) cb.buffer;
-        unsigned char *end = (unsigned char *) cb.buffer + cb.len;
-        size_t remaining = cb.len;
-        while ((p = memchr(p, '\n', remaining))) {
+    // memchr is not faster, seems the compiler is smart enough to optimize a simple loop that counts the newlines
+    for (uint32_t i = 0; i < cb.len; i++) {
+        if (cb.buffer[i] == '\n')
             alloc++;
-            p++;
-            remaining = end - p;
-        }
     }
 
     Modes.db2 = cmalloc(alloc * sizeof(dbEntry));

From ae09ee1151edc19bcac4e0187521b6425d959e17 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Tue, 17 Sep 2024 17:48:47 +0200
Subject: [PATCH 50/90] readwholeGz minor cleanup

---
 util.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/util.c b/util.c
index 45f13fd5..23377bd6 100644
--- a/util.c
+++ b/util.c
@@ -395,7 +395,6 @@ struct char_buffer readWholeGz(gzFile gzfp, char *errorContext) {
         cb.len += res;
         toRead -= res;
         if (toRead == 0) {
-            toRead = alloc;
             alloc *= 2;
             char *oldBuffer = cb.buffer;
             cb.buffer = realloc(cb.buffer, alloc);
@@ -404,6 +403,7 @@ struct char_buffer readWholeGz(gzFile gzfp, char *errorContext) {
                 fprintf(stderr, "reading %s: readWholeGz alloc fail!\n", errorContext);
                 return (struct char_buffer) {0};
             }
+            toRead = alloc - cb.len;
         }
     }
     if (res < 0) {

From 7086b20e8a335ff2c0307f7ed3182719f5baf871 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Tue, 17 Sep 2024 18:51:38 +0200
Subject: [PATCH 51/90] change aircraft db to use less memory

---
 aircraft.c | 233 ++++++++++++++++++++++++++++++++++-------------------
 aircraft.h |   7 +-
 readsb.h   |   2 +
 3 files changed, 154 insertions(+), 88 deletions(-)

diff --git a/aircraft.c b/aircraft.c
index 4eb380e1..828723ae 100644
--- a/aircraft.c
+++ b/aircraft.c
@@ -1,6 +1,7 @@
 #include "readsb.h"
 
 static int isMilRange(uint32_t i);
+static void updateDetails(struct aircraft *curr, struct char_buffer cb, uint32_t offset);
 
 static inline uint32_t dbHash(uint32_t addr) {
     return addrHash(addr, DB_HASH_BITS);
@@ -321,51 +322,68 @@ void toBinCraft(struct aircraft *a, struct binCraft *new, int64_t now) {
 #undef F
 }
 
+
 // rudimentary sanitization so the json output hopefully won't be invalid
-static inline void sanitize(char *str, unsigned len) {
+static inline void sanitize(char *str, int len) {
     unsigned char b2 = (1<<7) + (1<<6); // 2 byte code or more
     unsigned char b3 = (1<<7) + (1<<6) + (1<<5); // 3 byte code or more
     unsigned char b4 = (1<<7) + (1<<6) + (1<<5) + (1<<4); // 4 byte code
 
+    int debug = 0;
+
+    // UTF that goes beyond our string is cut off by terminating the string
+
     if (len >= 3 && (str[len - 3] & b4) == b4) {
-        //fprintf(stderr, "%c\n", str[len - 3]);
+        if (debug) { fprintf(stderr, "b4%d\n", str[len - 3]); }
         str[len - 3] = '\0';
     }
     if (len >= 2 && (str[len - 2] & b3) == b3) {
-        //fprintf(stderr, "%c\n", str[len - 2]);
+        if (debug) { fprintf(stderr, "b3%d\n", str[len - 2]); }
         str[len - 2] = '\0';
     }
     if (len >= 1 && (str[len - 1] & b2) == b2) {
-        //fprintf(stderr, "%c\n", str[len - 1]);
+        if (debug) { fprintf(stderr, "b2%d\n", str[len - 1]); }
         str[len - 1] = '\0';
     }
     char *p = str;
-    while(p < str + len && *p) {
-        if (*p == '"')
+    // careful str might be unterminated, use len
+    while(p - str < len) {
+        if (*p == '"') {
+            //if (debug) { fprintf(stderr, "quotation marks: %s\n", str); }
+            // replace with single quote
             *p = '\'';
-        if (*p > 0 && *p < 0x1f)
+        }
+        if (*p > 0 && *p < 0x1f) {
+            if (debug) { fprintf(stderr, "non-printable: %s\n", str); }
+            // replace with space
             *p = ' ';
+        }
         p++;
     }
     if (p - 1 >= str && *(p - 1) == '\\') {
         *(p - 1) = '\0';
     }
 }
-static char *sprintDB(char *p, char *end, dbEntry *d) {
-    p = safe_snprintf(p, end, "\n\"%s%06x\":{", (d->addr & MODES_NON_ICAO_ADDRESS) ? "~" : "", d->addr & 0xFFFFFF);
+static char *sprintDB2(char *p, char *end, dbEntry *d) {
+    struct aircraft aBack;
+    struct aircraft *a = &aBack;
+
+    updateDetails(a, Modes.db2Raw, d->rawOffset);
+    a->addr = d->addr;
+    p = safe_snprintf(p, end, "\n\"%s%06x\":{", (a->addr & MODES_NON_ICAO_ADDRESS) ? "~" : "", a->addr & 0xFFFFFF);
     char *regInfo = p;
-    if (d->registration[0])
-        p = safe_snprintf(p, end, "\"r\":\"%.*s\",", (int) sizeof(d->registration), d->registration);
-    if (d->typeCode[0])
-        p = safe_snprintf(p, end, "\"t\":\"%.*s\",", (int) sizeof(d->typeCode), d->typeCode);
-    if (d->typeLong[0])
-        p = safe_snprintf(p, end, "\"desc\":\"%.*s\",", (int) sizeof(d->typeLong), d->typeLong);
-    if (d->dbFlags)
-        p = safe_snprintf(p, end, "\"dbFlags\":%u,", d->dbFlags);
-    if (d->ownOp[0])
-        p = safe_snprintf(p, end, "\"ownOp\":\"%.*s\",", (int) sizeof(d->ownOp), d->ownOp);
-    if (d->year[0])
-        p = safe_snprintf(p, end, "\"year\":\"%.*s\",", (int) sizeof(d->year), d->year);
+    if (a->registration[0])
+        p = safe_snprintf(p, end, "\"r\":\"%.*s\",", (int) sizeof(a->registration), a->registration);
+    if (a->typeCode[0])
+        p = safe_snprintf(p, end, "\"t\":\"%.*s\",", (int) sizeof(a->typeCode), a->typeCode);
+    if (a->typeLong[0])
+        p = safe_snprintf(p, end, "\"desc\":\"%.*s\",", (int) sizeof(a->typeLong), a->typeLong);
+    if (a->dbFlags)
+        p = safe_snprintf(p, end, "\"dbFlags\":%u,", a->dbFlags);
+    if (a->ownOp[0])
+        p = safe_snprintf(p, end, "\"ownOp\":\"%.*s\",", (int) sizeof(a->ownOp), a->ownOp);
+    if (a->year[0])
+        p = safe_snprintf(p, end, "\"year\":\"%.*s\",", (int) sizeof(a->year), a->year);
     if (p == regInfo)
         p = safe_snprintf(p, end, "\"noRegData\":true,");
     if (*(p-1) == ',')
@@ -373,14 +391,17 @@ static char *sprintDB(char *p, char *end, dbEntry *d) {
     p = safe_snprintf(p, end, "},");
     return p;
 }
-static void dbToJson() {
+static void db2ToJson() {
     size_t buflen = 32 * 1024 * 1024;
     char *buf = (char *) cmalloc(buflen), *p = buf, *end = buf + buflen;
     p = safe_snprintf(p, end, "{");
 
     for (int j = 0; j < DB_BUCKETS; j++) {
+        if (0 && j % 1000 == 0) {
+            fprintf(stderr, "db print %d\n", j);
+        }
         for (dbEntry *d = Modes.db2Index[j]; d; d = d->next) {
-            p = sprintDB(p, end, d);
+            p = sprintDB2(p, end, d);
             if ((p + 1000) >= end) {
                 int used = p - buf;
                 buflen *= 2;
@@ -411,7 +432,6 @@ static inline int nextToken(char delim, char **sot, char **eot, char **eol) {
     if (!*eot)
         return 0;
 
-    **eot = '\0';
     return 1;
 }
 
@@ -498,16 +518,18 @@ int dbUpdate(int64_t now) {
         goto DBU0;
     }
 
-    // reallocate so we don't waste memory
-    char *oldBuffer = cb.buffer;
-    cb.len++; // for adding zero termination
-    cb.buffer = realloc(cb.buffer, cb.len);
-    if (!cb.buffer) {
-        fprintf(stderr, "database read failed due to realloc\n");
-        sfree(oldBuffer);
-        goto DBU0;
+    if (1) {
+        // reallocate so we don't waste memory
+        char *oldBuffer = cb.buffer;
+        cb.len++; // for adding zero termination
+        cb.buffer = realloc(cb.buffer, cb.len);
+        if (!cb.buffer) {
+            fprintf(stderr, "database read failed due to realloc\n");
+            sfree(oldBuffer);
+            goto DBU0;
+        }
+        cb.buffer[cb.len - 1] = '\0'; // zero terminate for good measure
     }
-    cb.buffer[cb.len - 1] = '\0'; // zero terminate for good measure
 
     int alloc = 0;
 
@@ -517,9 +539,19 @@ int dbUpdate(int64_t now) {
             alloc++;
     }
 
-    Modes.db2 = cmalloc(alloc * sizeof(dbEntry));
-    Modes.db2Index = cmalloc(DB_BUCKETS * sizeof(void*));
-    memset(Modes.db2Index, 0, DB_BUCKETS * sizeof(void*));
+    int indexSize = DB_BUCKETS * sizeof(void*);
+    int entriesSize = alloc * sizeof(dbEntry);
+
+    Modes.db2 = cmalloc(entriesSize);
+    Modes.db2Raw = cb;
+    Modes.db2Index = cmalloc(indexSize);
+    memset(Modes.db2Index, 0, indexSize);
+
+    if (0) {
+        fprintf(stderr, "db mem usage: total %d index %d entries %d text %d kB\n",
+                indexSize / 1024 +  entriesSize / 1024 +  (int) (cb.len / 1024),
+                indexSize / 1024, entriesSize / 1024, (int) (cb.len / 1024));
+    }
 
     if (!Modes.db2 || !Modes.db2Index) {
         fprintf(stderr, "db update error: malloc failure!\n");
@@ -545,39 +577,8 @@ int dbUpdate(int64_t now) {
         if (curr->addr == 0)
             continue;
 
-
-#define copyDetail(d) do { memcpy(curr->d , sot, imin(sizeof(curr->d ), eot - sot)); sanitize(curr->d , sizeof(curr->d )); } while (0)
-
-        if (!nextToken(';', &sot, &eot, &eol)) continue;
-        copyDetail(registration);
-
-        if (!nextToken(';', &sot, &eot, &eol)) continue;
-        copyDetail(typeCode);
-
-        if (!nextToken(';', &sot, &eot, &eol)) continue;
-        for (int j = 0; j < 8 * (int) sizeof(curr->dbFlags) && sot < eot; j++, sot++)
-            curr->dbFlags |= ((*sot == '1') << j);
-
-
-        if (!nextToken(';', &sot, &eot, &eol)) continue;
-        copyDetail(typeLong);
-
         if (!nextToken(';', &sot, &eot, &eol)) continue;
-        copyDetail(year);
-
-        if (!nextToken(';', &sot, &eot, &eol)) continue;
-        copyDetail(ownOp);
-
-#undef copyDetail
-
-        if (false) // debugging output
-            fprintf(stdout, "%06X;%.12s;%.4s;%c%c;%.54s\n",
-                    curr->addr,
-                    curr->registration,
-                    curr->typeCode,
-                    curr->dbFlags & 1 ? '1' : '0',
-                    curr->dbFlags & 2 ? '1' : '0',
-                    curr->typeLong);
+        curr->rawOffset = sot - cb.buffer;
 
         i++; // increment db array index
         // add to hashtable
@@ -593,7 +594,6 @@ int dbUpdate(int64_t now) {
     //fprintf(stderr, "dbUpdate() done\n");
 
     gzclose(gzfp);
-    free(cb.buffer);
     Modes.dbModificationTime = modTime;
     if (Modes.json_dir) {
         free(writeJsonToFile(Modes.json_dir, "receiver.json", generateReceiverJson()).buffer);
@@ -606,7 +606,7 @@ int dbUpdate(int64_t now) {
 
     // write database to json dir for testing
     if (Modes.json_dir && Modes.debug_dbJson) {
-        dbToJson();
+        db2ToJson();
     }
 
     return 1;
@@ -618,6 +618,8 @@ int dbUpdate(int64_t now) {
     free(Modes.db2Index);
     Modes.db2 = NULL;
     Modes.db2Index = NULL;
+    Modes.db2Raw.buffer = NULL;
+    Modes.db2Raw.len = 0;
     close(fd);
     return 1;
 }
@@ -634,7 +636,7 @@ static void updateTypeRegRange(void *arg, threadpool_threadbuffers_t *threadbuff
 }
 
 int dbFinishUpdate() {
-    if (!(Modes.db2 && Modes.db2Index)) {
+    if (!Modes.db2) {
         return 0;
     }
     // finish db update
@@ -644,12 +646,16 @@ int dbFinishUpdate() {
 
 
     free(Modes.dbIndex);
+    free(Modes.dbRaw.buffer);
     free(Modes.db);
 
     Modes.dbIndex = Modes.db2Index;
+    Modes.dbRaw = Modes.db2Raw;
     Modes.db = Modes.db2;
     Modes.db2Index = NULL;
     Modes.db2 = NULL;
+    Modes.db2Raw.buffer = NULL;
+    Modes.db2Raw.len = 0;
 
     /*
     for (int j = 0; j < AIRCRAFT_BUCKETS; j++) {
@@ -689,21 +695,84 @@ void dbPut(uint32_t addr, dbEntry **index, dbEntry *d) {
     index[hash] = d;
 }
 
+static void copyDetailFunc(char *sot, char *eot, char *target, int alloc) {
+    int len = imin(alloc, eot - sot);
+    memcpy(target, sot, len);
+    // terminate if string doesn't fill up alloc
+    // this mildly insane program can hopefully handle unterminated database strings
+    if (len < alloc) {
+        target[len] = '\0';
+    }
+    sanitize(target, len);
+}
+
+static void updateDetails(struct aircraft *curr, struct char_buffer cb, uint32_t offset) {
+    //fprintf(stdout, "%u %u\n", offset, (uint32_t) cb.len);
+    char *eob = cb.buffer + cb.len;
+    char *sol = cb.buffer + offset;
+    char *eol = memchr(sol, '\n', eob - sol);
+    char *sot;
+    char *eot = sol - 1; // this pointer must not be dereferenced, nextToken will increment it.
+
+    if (!eol) goto BAD_ENTRY;
+
+#define copyDetail(d) do { copyDetailFunc(sot, eot, curr->d, sizeof(curr->d)); } while (0)
+
+    if (!nextToken(';', &sot, &eot, &eol)) goto BAD_ENTRY;
+    copyDetail(registration);
+
+    if (!nextToken(';', &sot, &eot, &eol)) goto BAD_ENTRY;
+    copyDetail(typeCode);
+
+    if (!nextToken(';', &sot, &eot, &eol)) goto BAD_ENTRY;
+    curr->dbFlags = 0;
+    for (int j = 0; j < 8 * (int) sizeof(curr->dbFlags) && sot < eot; j++, sot++)
+        curr->dbFlags |= ((*sot == '1') << j);
+
+
+    if (!nextToken(';', &sot, &eot, &eol)) goto BAD_ENTRY;
+    copyDetail(typeLong);
+
+    if (!nextToken(';', &sot, &eot, &eol)) goto BAD_ENTRY;
+    copyDetail(year);
+
+    if (!nextToken(';', &sot, &eot, &eol)) goto BAD_ENTRY;
+    copyDetail(ownOp);
+
+#undef copyDetail
+
+    if (false) {
+        // debugging output
+        fprintf(stdout, "%06X;%.12s;%.4s;%c%c;%.54s\n",
+                curr->addr,
+                curr->registration,
+                curr->typeCode,
+                curr->dbFlags & 1 ? '1' : '0',
+                curr->dbFlags & 2 ? '1' : '0',
+                curr->typeLong);
+    }
+
+    return;
+
+BAD_ENTRY:
+    curr->registration[0] = '\0';
+    curr->typeCode[0] = '\0';
+    curr->typeLong[0] = '\0';
+    curr->ownOp[0] = '\0';
+    curr->year[0] = '\0';
+    curr->dbFlags = 0;
+}
+
 void updateTypeReg(struct aircraft *a) {
     dbEntry *d = dbGet(a->addr, Modes.dbIndex);
     if (d) {
-        memcpy(a->registration, d->registration, sizeof(a->registration));
-        memcpy(a->typeCode, d->typeCode, sizeof(a->typeCode));
-        memcpy(a->typeLong, d->typeLong, sizeof(a->typeLong));
-        memcpy(a->ownOp, d->ownOp, sizeof(a->ownOp));
-        memcpy(a->year, d->year, sizeof(a->year));
-        a->dbFlags = d->dbFlags;
+        updateDetails(a, Modes.dbRaw, d->rawOffset);
     } else {
-        memset(a->registration, 0, sizeof(a->registration));
-        memset(a->typeCode, 0, sizeof(a->typeCode));
-        memset(a->typeLong, 0, sizeof(a->typeLong));
-        memset(a->ownOp, 0, sizeof(a->ownOp));
-        memset(a->year, 0, sizeof(a->year));
+        a->registration[0] = '\0';
+        a->typeCode[0] = '\0';
+        a->typeLong[0] = '\0';
+        a->ownOp[0] = '\0';
+        a->year[0] = '\0';
         a->dbFlags = 0;
     }
     if (is_df18_exception(a->addr)) {
diff --git a/aircraft.h b/aircraft.h
index c2f233c3..c36ea29c 100644
--- a/aircraft.h
+++ b/aircraft.h
@@ -39,12 +39,7 @@ void freeAircraft(struct aircraft *a);
 typedef struct dbEntry {
     struct dbEntry *next;
     uint32_t addr;
-    uint16_t dbFlags;
-    char typeCode[4];
-    char registration[12];
-    char typeLong[64];
-    char ownOp[64];
-    char year[4];
+    uint32_t rawOffset;
 } dbEntry;
 
 dbEntry *dbGet(uint32_t addr, dbEntry **index);
diff --git a/readsb.h b/readsb.h
index 533b59f1..b58747a8 100644
--- a/readsb.h
+++ b/readsb.h
@@ -577,8 +577,10 @@ struct _Modes
     struct craftArray aircraftActive;
     dbEntry *db;
     dbEntry **dbIndex;
+    struct char_buffer dbRaw;
     dbEntry *db2;
     dbEntry **db2Index;
+    struct char_buffer db2Raw;
     int64_t dbModificationTime;
     int64_t receiverCount;
     struct net_writer raw_out; // Raw output

From 85b918727bb052b6ad49c79a765634e138d0165d Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Tue, 17 Sep 2024 18:51:49 +0200
Subject: [PATCH 52/90] incrementing version: 3.14.1641

---
 debian/changelog | 4 ++--
 version          | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index c7411c57..f4d4910e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-readsb (3.14.1640) UNRELEASED; urgency=medium
+readsb (3.14.1641) UNRELEASED; urgency=medium
 
   * In development
 
- -- Matthias Wirth <matthias.wirth@gmail.com>  Mon, 16 Sep 2024 17:11:25 +0200
+ -- Matthias Wirth <matthias.wirth@gmail.com>  Tue, 17 Sep 2024 18:51:49 +0200
diff --git a/version b/version
index 99cb85bf..cbfe147e 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.14.1640
+3.14.1641

From 6ee23d772674175e5babdbb654a296e415ac20f0 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sat, 28 Sep 2024 13:27:23 +0200
Subject: [PATCH 53/90] fixup uptime in stats

startup_time was changed to monotonic without properly changing all the
places it was used.
change startup_time back and add startup_time_mono
---
 readsb.c |  3 ++-
 readsb.h |  1 +
 stats.c  | 17 +++--------------
 util.c   |  2 +-
 4 files changed, 7 insertions(+), 16 deletions(-)

diff --git a/readsb.c b/readsb.c
index 4de6bafd..bd1a3618 100644
--- a/readsb.c
+++ b/readsb.c
@@ -2689,7 +2689,8 @@ int main(int argc, char **argv) {
     // Set sane defaults
     configSetDefaults();
 
-    Modes.startup_time = mono_milli_seconds();
+    Modes.startup_time_mono = mono_milli_seconds();
+    Modes.startup_time = mstime();
 
     if (lzo_init() != LZO_E_OK)
     {
diff --git a/readsb.h b/readsb.h
index b58747a8..434196ed 100644
--- a/readsb.h
+++ b/readsb.h
@@ -855,6 +855,7 @@ struct _Modes
     ALIGNED struct mag_buf mag_buffers[MODES_MAG_BUFFERS]; // Converted magnitude buffers from RTL or file input
 
     int64_t startup_time;
+    int64_t startup_time_mono;
     int64_t next_stats_update;
     int64_t next_stats_display;
     int64_t next_api_update;
diff --git a/stats.c b/stats.c
index a2041916..a668e0a0 100644
--- a/stats.c
+++ b/stats.c
@@ -660,12 +660,8 @@ struct char_buffer generateStatusProm(int64_t now) {
 
     struct statsCount *sC = &(Modes.globalStatsCount);
 
-    int64_t uptime = now - Modes.startup_time;
-    if (now < Modes.startup_time)
-        uptime = 0;
-
     p = safe_snprintf(p, end, "readsb_now %"PRIu64"\n", now);
-    p = safe_snprintf(p, end, "readsb_uptime %"PRIu64"\n", uptime);
+    p = safe_snprintf(p, end, "readsb_uptime %"PRIu64"\n", getUptime());
 
     p = safe_snprintf(p, end, "readsb_aircraft_with_position %u\n", sC->readsb_aircraft_with_position);
     p = safe_snprintf(p, end, "readsb_aircraft_without_position %u\n", sC->readsb_aircraft_total - sC->readsb_aircraft_with_position);
@@ -686,11 +682,7 @@ struct char_buffer generateStatusJson(int64_t now) {
     size_t buflen = 8192;
     char *buf = (char *) cmalloc(buflen), *p = buf, *end = buf + buflen;
 
-    int64_t uptime = now - Modes.startup_time;
-    if (now < Modes.startup_time)
-        uptime = 0;
-
-    p = safe_snprintf(p, end, "{ \"now\": %.1f, \"uptime\": %.1f", now / 1000.0, uptime / 1000.0);
+    p = safe_snprintf(p, end, "{ \"now\": %.1f, \"uptime\": %.1f", now / 1000.0, getUptime() / 1000.0);
 
     p = appendTypeCounts(p, end);
 
@@ -903,10 +895,7 @@ struct char_buffer generatePromFile(int64_t now) {
         p = safe_snprintf(p, end, "readsb_trace_chunk_memory %"PRIu64"\n", Modes.trace_chunk_size);
         p = safe_snprintf(p, end, "readsb_trace_cache_memory %"PRIu64"\n", Modes.trace_cache_size);
     }
-    int64_t uptime = now - Modes.startup_time;
-    if (now < Modes.startup_time)
-        uptime = 0;
-    p = safe_snprintf(p, end, "readsb_uptime %"PRIu64"\n", uptime);
+    p = safe_snprintf(p, end, "readsb_uptime %"PRIu64"\n", getUptime());
 
     if (p >= end)
         fprintf(stderr, "buffer overrun stats prom\n");
diff --git a/util.c b/util.c
index 23377bd6..7ae104d4 100644
--- a/util.c
+++ b/util.c
@@ -118,7 +118,7 @@ int64_t mono_milli_seconds() {
 }
 
 int64_t getUptime() {
-    return mono_milli_seconds() - Modes.startup_time;
+    return mono_milli_seconds() - Modes.startup_time_mono;
 }
 
 int snprintHMS(char *buf, size_t bufsize, int64_t now) {

From 9ef77b3424acd1aa466cbdd9b78c18e34a9eec6f Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Thu, 10 Oct 2024 21:47:39 +0200
Subject: [PATCH 54/90] fixup api thread count comment

---
 api.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/api.c b/api.c
index 1394aed2..83267284 100644
--- a/api.c
+++ b/api.c
@@ -1916,7 +1916,7 @@ static void *apiUpdateEntryPoint(void *arg) {
 }
 
 void apiBufferInit() {
-    // 1 api thread per 2 cores as we assume nginx running on the same box, better chances not swamping the CPU under high API load scenarios
+    // if --devel=apiThreads,<n> isn't given, use nproc - 1 api threads (nproc - 2 for nproc > 6)
     if (Modes.apiThreadCount <= 0) {
         Modes.apiThreadCount = imax(1, Modes.num_procs - (Modes.num_procs > 6 ? 2 : 1));
     }

From 87d62d3c9f3a348396b6fd8255960700c98c4360 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Thu, 10 Oct 2024 21:49:21 +0200
Subject: [PATCH 55/90] readsb service: add CAP_SYS_NICE

this enables the SDR thread to be run with high priority
in some cases this will help MLAT stability
---
 debian/readsb.service | 1 +
 1 file changed, 1 insertion(+)

diff --git a/debian/readsb.service b/debian/readsb.service
index 1ecb08d5..4ccc5f17 100644
--- a/debian/readsb.service
+++ b/debian/readsb.service
@@ -9,6 +9,7 @@ After=network.target
 [Service]
 EnvironmentFile=/etc/default/readsb
 User=readsb
+AmbientCapabilities=CAP_SYS_NICE
 RuntimeDirectory=readsb
 RuntimeDirectoryMode=0755
 ExecStart=/usr/bin/readsb \

From 431f5d8480c4247bd94ce133e5c868801029e13e Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sun, 13 Oct 2024 23:54:32 +0200
Subject: [PATCH 56/90] net io: make some code nicer to read

---
 net_io.c | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/net_io.c b/net_io.c
index aa09d4d6..acfb5a27 100644
--- a/net_io.c
+++ b/net_io.c
@@ -1466,14 +1466,13 @@ static int pongReceived(struct client *c, int64_t now) {
 static int flushClient(struct client *c, int64_t now) {
     if (!c->service) { fprintf(stderr, "report error: Ahlu8pie\n"); return -1; }
     int toWrite = c->sendq_len;
-    char *psendq = c->sendq;
 
     if (toWrite == 0) {
         c->last_flush = now;
         return 0;
     }
 
-    int bytesWritten = send(c->fd, psendq, toWrite, 0);
+    int bytesWritten = send(c->fd, c->sendq, toWrite, 0);
     int err = errno;
 
     // If we get -1, it's only fatal if it's not EAGAIN/EWOULDBLOCK
@@ -1499,7 +1498,6 @@ static int flushClient(struct client *c, int64_t now) {
     if (bytesWritten > 0) {
         Modes.stats_current.network_bytes_out += bytesWritten;
         // Advance buffer
-        psendq += bytesWritten;
         toWrite -= bytesWritten;
         c->sendq_len -= bytesWritten;
 
@@ -1510,13 +1508,13 @@ static int flushClient(struct client *c, int64_t now) {
             memmove((void*)c->sendq, c->sendq + bytesWritten, toWrite);
         }
     }
-    if (c->last_flush != now && !(c->epollEvent.events & EPOLLOUT)) {
+    if (toWrite > 0 && !(c->epollEvent.events & EPOLLOUT)) {
         // if we couldn't flush our buffer, make epoll tell us when we can write again
         c->epollEvent.events |= EPOLLOUT;
         if (epoll_ctl(Modes.net_epfd, EPOLL_CTL_MOD, c->fd, &c->epollEvent))
             perror("epoll_ctl fail:");
     }
-    if ((c->epollEvent.events & EPOLLOUT) && c->last_flush == now) {
+    if (toWrite == 0 && (c->epollEvent.events & EPOLLOUT)) {
         // if set, remove EPOLLOUT from epoll if flush was successful
         c->epollEvent.events ^= EPOLLOUT;
         if (epoll_ctl(Modes.net_epfd, EPOLL_CTL_MOD, c->fd, &c->epollEvent))

From e7dec69f0ad709dc7e6e5799c5998e571f03681e Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Thu, 17 Oct 2024 16:32:01 +0200
Subject: [PATCH 57/90] speed_check: decrement speedUnreliable speed is unknown

---
 track.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/track.c b/track.c
index 731f1a99..7f82b36d 100644
--- a/track.c
+++ b/track.c
@@ -598,7 +598,9 @@ static int speed_check(struct aircraft *a, datasource_t source, double lat, doub
         range += 250;
     }
 
-    if (distance > 2.5f && (track_diff < 70 || track_diff == -1)) {
+    if (transmitted_speed < 0) {
+            mm->speedUnreliable = -1;
+    } else if (distance > 2.5f && (track_diff < 70 || track_diff == -1)) {
         if (distance <= range + (((float) elapsed + 50.0f) * (1.0f / 1000.0f)) * (transmitted_speed * knots_to_meterpersecond)) {
             mm->speedUnreliable = -1;
         } else if (distance > range + (((float) elapsed + 400.0f) * (1.0f / 1000.0f)) * (transmitted_speed * knots_to_meterpersecond)) {

From 0c87e593256209f7fff8b27b428722863355a264 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Thu, 17 Oct 2024 16:35:27 +0200
Subject: [PATCH 58/90] incrementing version: 3.14.1642

---
 debian/changelog | 4 ++--
 version          | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index f4d4910e..55b091ba 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-readsb (3.14.1641) UNRELEASED; urgency=medium
+readsb (3.14.1642) UNRELEASED; urgency=medium
 
   * In development
 
- -- Matthias Wirth <matthias.wirth@gmail.com>  Tue, 17 Sep 2024 18:51:49 +0200
+ -- Matthias Wirth <matthias.wirth@gmail.com>  Thu, 17 Oct 2024 16:35:27 +0200
diff --git a/version b/version
index cbfe147e..379a3821 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.14.1641
+3.14.1642

From 4f8a0b4209f5942b2014248ac89d7ed4c7085246 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Tue, 29 Oct 2024 17:49:25 +0100
Subject: [PATCH 59/90] make --devel=enableClientsJson work

---
 mode_s.c |  2 +-
 readsb.c | 14 ++++++++------
 readsb.h |  2 +-
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/mode_s.c b/mode_s.c
index 4b6f965a..964b4080 100644
--- a/mode_s.c
+++ b/mode_s.c
@@ -785,7 +785,7 @@ int decodeModesMessage(struct modesMessage *mm) {
     }
 
     // these are messages of general bad quality, treat them as garbage when garbage_ports is in use.
-    if ((Modes.netIngest || Modes.garbage_ports) && mm->remote && mm->timestamp == 0 && mm->msgtype != 18) {
+    if ((Modes.netIngest || Modes.garbage_ports) && mm->remote && mm->timestamp == 0 && (mm->msgtype != 18 || mm->addrtype == ADDR_ADSB_ICAO_NT)) {
         mm->garbage = 1;
         mm->source = SOURCE_SBS;
         if (mm->addrtype >= ADDR_OTHER)
diff --git a/readsb.c b/readsb.c
index bd1a3618..4705bfed 100644
--- a/readsb.c
+++ b/readsb.c
@@ -1960,8 +1960,8 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
                 if (strcasecmp(token[0], "disableAcasJson") == 0) {
                     Modes.enableAcasJson = 0;
                 }
-                if (strcasecmp(token[0], "enableConnsJson") == 0) {
-                    Modes.enableConnsJson = 1;
+                if (strcasecmp(token[0], "enableClientsJson") == 0) {
+                    Modes.enableClientsJson = 1;
                 }
                 if (strcasecmp(token[0], "tar1090NoGlobe") == 0) {
                     Modes.tar1090_no_globe = 1;
@@ -2575,12 +2575,14 @@ static void miscStuff(int64_t now) {
     static int64_t next_clients_json;
     if (Modes.json_dir && now > next_clients_json) {
         next_clients_json = now + 10 * SECONDS;
-        if (Modes.enableConnsJson) {
-        }
-        if (Modes.netIngest)
+
+        if (Modes.netIngest || Modes.enableClientsJson) {
             free(writeJsonToFile(Modes.json_dir, "clients.json", generateClientsJson()).buffer);
-        if (Modes.netReceiverIdJson)
+        }
+
+        if (Modes.netReceiverIdJson) {
             free(writeJsonToFile(Modes.json_dir, "receivers.json", generateReceiversJson()).buffer);
+        }
 
         return;
     }
diff --git a/readsb.h b/readsb.h
index 434196ed..2f8385d1 100644
--- a/readsb.h
+++ b/readsb.h
@@ -694,7 +694,7 @@ struct _Modes
     int8_t netReceiverIdPrint;
     int8_t netReceiverIdJson;
     int8_t netIngest;
-    int8_t enableConnsJson;
+    int8_t enableClientsJson;
     int8_t forward_mlat; // forward beast mlat messages to beast output ports
     int8_t forward_mlat_sbs; // forward mlat messages to sbs output ports
     int8_t beast_forward_noforward;

From 207ec62065f9d933854c0ee76ca0ded94a47cfce Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Fri, 1 Nov 2024 11:10:29 +0100
Subject: [PATCH 60/90] incrementing version: 3.14.1643

---
 debian/changelog | 4 ++--
 version          | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 55b091ba..09f15f4d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-readsb (3.14.1642) UNRELEASED; urgency=medium
+readsb (3.14.1643) UNRELEASED; urgency=medium
 
   * In development
 
- -- Matthias Wirth <matthias.wirth@gmail.com>  Thu, 17 Oct 2024 16:35:27 +0200
+ -- Matthias Wirth <matthias.wirth@gmail.com>  Fri, 01 Nov 2024 11:10:29 +0100
diff --git a/version b/version
index 379a3821..a00e61cf 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.14.1642
+3.14.1643

From 6e2434f1f412bdaee89adf2a147c9482fea9e783 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Fri, 1 Nov 2024 11:07:06 +0100
Subject: [PATCH 61/90] make it simple to save traces without enabling
 globe-index

---
 globe_index.c | 10 ++++++----
 help.h        |  2 +-
 json_out.c    | 10 ++++++----
 readsb.c      |  7 ++++---
 readsb.h      |  1 +
 stats.c       |  4 ++--
 6 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/globe_index.c b/globe_index.c
index 6ece463a..35d04e33 100644
--- a/globe_index.c
+++ b/globe_index.c
@@ -566,6 +566,8 @@ void traceWrite(struct aircraft *a, threadpool_threadbuffers_t *buffer_group) {
         //if (Modes.debug_traceCount && ++count2 % 1000 == 0)
         //    fprintf(stderr, "recent trace write: %u\n", count2);
 
+        //fprintf(stderr, "traceWrite() recent for %06x uncompressed %4d bytes\n", a->addr, (int) recent.len);
+
         if (recent.len > 0) {
             snprintf(filename, 256, "traces/%02x/trace_recent_%s%06x.json", a->addr % 256, (a->addr & MODES_NON_ICAO_ADDRESS) ? "~" : "", a->addr & 0xFFFFFF);
 
@@ -1604,7 +1606,7 @@ void set_globe_index (struct aircraft *a, int new_index) {
 static void traceUnlink(struct aircraft *a) {
     char filename[PATH_MAX];
 
-    if (!Modes.json_globe_index || !Modes.json_dir)
+    if (!Modes.writeTraces || !Modes.json_dir)
         return;
 
     snprintf(filename, 1024, "%s/traces/%02x/trace_recent_%s%06x.json", Modes.json_dir, a->addr % 256, (a->addr & MODES_NON_ICAO_ADDRESS) ? "~" : "", a->addr & 0xFFFFFF);
@@ -2244,7 +2246,7 @@ void traceMaintenance(struct aircraft *a, int64_t now, threadpool_buffer_t *pass
         return;
     }
 
-    if (Modes.json_globe_index) {
+    if (Modes.writeTraces) {
         if (now > a->trace_next_perm)
             a->trace_write |= WPERM;
         if (now > a->trace_next_mw)
@@ -3349,7 +3351,7 @@ static void compressACAS(char *dateDir) {
 }
 
 void checkNewDay(int64_t now) {
-    if (!Modes.globe_history_dir || !Modes.json_globe_index)
+    if (!Modes.globe_history_dir || !Modes.writeTraces)
         return;
 
     static int64_t next_check;
@@ -3413,7 +3415,7 @@ void checkNewDay(int64_t now) {
 }
 
 void checkNewDayAcas(int64_t now) {
-    if (!Modes.globe_history_dir || !Modes.json_globe_index)
+    if (!Modes.globe_history_dir || !Modes.writeTraces)
         return;
 
     struct tm utc;
diff --git a/help.h b/help.h
index 17fae3bd..f2ec68d3 100644
--- a/help.h
+++ b/help.h
@@ -95,7 +95,7 @@ static struct argp_option optionsReadsb[] = {
     {"enable-biastee", OptBiasTee, 0, OPTION_HIDDEN, "Enable bias tee on supporting interfaces (default: disabled)", 1},
     {"write-json", OptJsonDir, "<dir>", 0, "Periodically write json output to <dir>", 1},
     {"write-prom", OptPromFile, "<filepath>", 0, "Periodically write prometheus output to <filepath>", 1},
-    {"write-globe-history", OptGlobeHistoryDir, "<dir>", 0, "Extended Globe History", 1},
+    {"write-globe-history", OptGlobeHistoryDir, "<dir>", 0, "Write traces to this directory, 1 gz compressed json per day and airframe", 1},
     {"write-state", OptStateDir, "<dir>", 0, "Write state to disk to have traces after a restart", 1},
     {"write-state-every", OptStateInterval, "<seconds>", 0, "Continuously write state to disk every X seconds (default: 3600)", 1},
     {"write-state-only-on-exit", OptStateOnlyOnExit, 0, 0, "Don't continously update state.", 1},
diff --git a/json_out.c b/json_out.c
index 04d776d1..d92b7e10 100644
--- a/json_out.c
+++ b/json_out.c
@@ -1726,8 +1726,8 @@ static void checkTraceCache(struct aircraft *a, traceBuffer tb, int64_t now) {
 
 struct char_buffer generateTraceJson(struct aircraft *a, traceBuffer tb, int start, int last, threadpool_buffer_t *buffer, int64_t referenceTs) {
     struct char_buffer cb = { 0 };
-    if (!Modes.json_globe_index) {
-        return cb;
+    if (!Modes.writeTraces) {
+        fprintf(stderr, "generateTraceJson called when Modes.writeTraces not set, this is a bug, please report with configuration!\n");
     }
     int64_t now = mstime();
 
@@ -1883,10 +1883,12 @@ struct char_buffer generateReceiverJson() {
         p = safe_snprintf(p, end, ", \"dbServer\": true");
     }
 
-    if (Modes.json_globe_index && !Modes.tar1090_no_globe) {
-
+    if (Modes.writeTraces) {
+        p = safe_snprintf(p, end, ", \"haveTraces\": true");
         p = safe_snprintf(p, end, ", \"json_trace_interval\": %.1f", ((double) Modes.json_trace_interval) / (1 * SECONDS));
+    }
 
+    if (Modes.json_globe_index && !Modes.tar1090_no_globe) {
         p = safe_snprintf(p, end, ", \"globeIndexGrid\": %d", GLOBE_INDEX_GRID);
 
         p = safe_snprintf(p, end, ", \"globeIndexSpecialTiles\": [ ");
diff --git a/readsb.c b/readsb.c
index 4705bfed..10f38f26 100644
--- a/readsb.c
+++ b/readsb.c
@@ -1138,7 +1138,7 @@ static void *upkeepEntryPoint(void *arg) {
 
         priorityTasksRun();
 
-        if (Modes.json_globe_index) {
+        if (Modes.writeTraces) {
             // writing a trace takes some time, to increase timing precision the priority tasks, allot a little less time than available
             // this isn't critical though
             int64_t time_alloted = ms_until_priority();
@@ -2212,8 +2212,9 @@ static void configAfterParse() {
     Modes.sdr_buf_samples = Modes.sdr_buf_size / 2;
     Modes.trackExpireMax = Modes.trackExpireJaero + TRACK_EXPIRE_LONG + 1 * MINUTES;
 
-    if (Modes.json_globe_index) {
+    if (Modes.json_globe_index || Modes.globe_history_dir) {
         Modes.keep_traces = 24 * HOURS + 60 * MINUTES; // include 60 minutes overlap
+        Modes.writeTraces = 1;
     } else if (Modes.heatmap || Modes.trace_focus != BADDR) {
         Modes.keep_traces = 35 * MINUTES; // heatmap is written every 30 minutes
     }
@@ -2753,7 +2754,7 @@ int main(int argc, char **argv) {
         mkdir_error(pathbuf, 0755, stderr);
     }
 
-    if (Modes.json_dir && Modes.json_globe_index) {
+    if (Modes.json_dir && Modes.writeTraces) {
         char pathbuf[PATH_MAX];
         snprintf(pathbuf, PATH_MAX, "%s/traces", Modes.json_dir);
         mkdir_error(pathbuf, 0755, stderr);
diff --git a/readsb.h b/readsb.h
index 2f8385d1..77a7ae22 100644
--- a/readsb.h
+++ b/readsb.h
@@ -802,6 +802,7 @@ struct _Modes
     int32_t dump_interval;
     int32_t dump_beast_index;
     uint64_t dump_lastReceiverId;
+    int8_t writeTraces;
     int8_t dump_reduce; // only dump beast that would be sent out according to reduce_interval
     int8_t state_only_on_exit;
     int8_t free_aircraft;
diff --git a/stats.c b/stats.c
index a668e0a0..d2a67657 100644
--- a/stats.c
+++ b/stats.c
@@ -890,7 +890,7 @@ struct char_buffer generatePromFile(int64_t now) {
             p = safe_snprintf(p, end, "readsb_demod_preambles %"PRIu32"\n", st->demod_preambles);
         }
     }
-    if (Modes.json_globe_index) {
+    if (Modes.keep_traces) {
         p = safe_snprintf(p, end, "readsb_trace_current_memory %"PRIu64"\n", Modes.trace_current_size);
         p = safe_snprintf(p, end, "readsb_trace_chunk_memory %"PRIu64"\n", Modes.trace_chunk_size);
         p = safe_snprintf(p, end, "readsb_trace_cache_memory %"PRIu64"\n", Modes.trace_cache_size);
@@ -936,7 +936,7 @@ void statsCountAircraft(int64_t now) {
         for (struct aircraft *a = Modes.aircraft[j]; a; a = a->next) {
             total_aircraft_count++;
 
-            if (Modes.json_globe_index) {
+            if (Modes.keep_traces) {
                 trace_current_size += stateBytes(a->trace_current_max);
                 trace_chunk_size += a->trace_chunk_overall_bytes;
                 struct traceCache *tCache = &a->traceCache;

From a0ad60b2a9576fb7472df9d5c701335c9cfe1db2 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Fri, 1 Nov 2024 13:36:17 +0100
Subject: [PATCH 62/90] incrementing version: 3.14.1644

---
 debian/changelog | 4 ++--
 version          | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 09f15f4d..3d05051d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-readsb (3.14.1643) UNRELEASED; urgency=medium
+readsb (3.14.1644) UNRELEASED; urgency=medium
 
   * In development
 
- -- Matthias Wirth <matthias.wirth@gmail.com>  Fri, 01 Nov 2024 11:10:29 +0100
+ -- Matthias Wirth <matthias.wirth@gmail.com>  Fri, 01 Nov 2024 13:36:17 +0100
diff --git a/version b/version
index a00e61cf..9f2334ec 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.14.1643
+3.14.1644

From f6c06e5817b19b6ca0dd144f3ec2b0c2d820ca31 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sat, 2 Nov 2024 01:16:26 +0100
Subject: [PATCH 63/90] first attempt at quick autogain

---
 demod_2400.c | 26 ++++++++++++++++++
 readsb.c     | 74 ++++++++++++++++++++++++++++++++++++++++++++++++----
 readsb.h     |  7 +++++
 sdr_rtlsdr.c | 48 +++++++++++++++++++++++++++++-----
 4 files changed, 143 insertions(+), 12 deletions(-)

diff --git a/demod_2400.c b/demod_2400.c
index 7ed1a299..01b51674 100644
--- a/demod_2400.c
+++ b/demod_2400.c
@@ -287,6 +287,16 @@ void demodulate2400(struct mag_buf *mag) {
     uint16_t *pa = m;
     uint16_t *stop = m + mlen;
 
+    uint16_t *statsProgress = m;
+
+    const uint32_t statsWindow = MODES_SHORT_MSG_SAMPLES / 2; // half a short message
+    uint32_t loudSamples = 0;
+    uint32_t quietSamples = 0;
+    uint32_t quiet2Samples = 0;
+    uint32_t loudThreshold = 55000 * statsWindow; // max 65536
+    uint32_t quietThreshold = 800 * statsWindow;
+    uint32_t quiet2Threshold = 2 * quietThreshold;
+
     for (; pa < stop; pa++) {
         int32_t pa_mag, base_noise, ref_level;
         int msglen;
@@ -308,6 +318,18 @@ void demodulate2400(struct mag_buf *mag) {
         // due to plenty room in the message buffer for decoding
         // we can with pa go beyond stop without a buffer overrun ...
 
+        if (Modes.autoGain && pa >= statsProgress) {
+            uint32_t magSum = 0;
+            for (uint32_t i = 0; i < statsWindow; i++) {
+                magSum += pa[i];
+            }
+            loudSamples += statsWindow * (magSum > loudThreshold);
+            quietSamples += statsWindow * (magSum < quietThreshold);
+            quiet2Samples += statsWindow * (magSum < quiet2Threshold);
+
+            statsProgress = pa + statsWindow;
+        }
+
         if (pa[1] > pa[7] && pa[12] > pa[14] && pa[12] > pa[15]) { goto after_pre; }
         pa++; if (pa[1] > pa[7] && pa[12] > pa[14] && pa[12] > pa[15]) { goto after_pre; }
         pa++; if (pa[1] > pa[7] && pa[12] > pa[14] && pa[12] > pa[15]) { goto after_pre; }
@@ -471,6 +493,10 @@ void demodulate2400(struct mag_buf *mag) {
         netUseMessage(mm);
     }
 
+    mag->loudSamples = loudSamples;
+    mag->quietSamples = quietSamples;
+    mag->quiet2Samples = quiet2Samples;
+
     /* update noise power */
     {
         double sum_signal_power = sum_scaled_signal_power / 65535.0 / 65535.0;
diff --git a/readsb.c b/readsb.c
index 10f38f26..69b1a22f 100644
--- a/readsb.c
+++ b/readsb.c
@@ -751,6 +751,65 @@ static void *globeBinEntryPoint(void *arg) {
     return NULL;
 }
 
+static void gainStatistics(struct mag_buf *buf) {
+    static uint64_t loudSamples;
+    static uint64_t quietSamples;
+    static uint64_t quiet2Samples;
+    static uint64_t totalSamples;
+    static int slowRise;
+    static int lastGain;
+
+    loudSamples += buf->loudSamples;
+    quietSamples += buf->quietSamples;
+    quiet2Samples += buf->quiet2Samples;
+    totalSamples += buf->length;
+
+    if (totalSamples < 2 * Modes.sample_rate) {
+        return;
+    }
+
+    double loudPercent = loudSamples / (double) totalSamples * 100.0;
+    double quietPercent = quietSamples / (double) totalSamples * 100.0;
+    double quiet2Percent = quiet2Samples / (double) totalSamples * 100.0;
+
+    // reset
+    loudSamples = 0;
+    quietSamples = 0;
+    quiet2Samples = 0;
+    totalSamples = 0;
+
+    if (!Modes.autoGain) {
+        // don't adjust anything
+        return;
+    }
+
+    if (loudPercent > 0.05 || quiet2Percent < 0.05) {
+        Modes.lowerGain = 1;
+    } else if (
+            quietPercent > 10.0
+            ) {
+        if (getUptime() < 1 * MINUTES || slowRise > 10) {
+            slowRise = 0;
+            Modes.increaseGain = 1;
+        } else {
+            slowRise++;
+        }
+    }
+    if (Modes.increaseGain || Modes.lowerGain) {
+        if (getUptime() < 1 * MINUTES) {
+            Modes.lowerGain *= 2;
+            Modes.increaseGain *= 2;
+        }
+        if (Modes.gain != lastGain) {
+            lastGain = Modes.gain;
+            fprintf(stderr, "loud: %8.4f %% quiet: %8.4f %% quiet2 %8.4f %%\n", loudPercent, quietPercent, quiet2Percent);
+        }
+        sdrSetGain();
+    }
+
+}
+
+
 static void timingStatistics(struct mag_buf *buf) {
     static int64_t last_ts;
 
@@ -873,6 +932,10 @@ static void *decodeEntryPoint(void *arg) {
                     demodulate2400AC(buf);
                 }
 
+                gainStatistics(buf);
+                timingStatistics(buf);
+
+                Modes.stats_current.samples_lost += Modes.sdr_buf_samples - buf->length;
                 Modes.stats_current.samples_processed += buf->length;
                 Modes.stats_current.samples_dropped += buf->dropped;
                 end_cpu_timing(&start_time, &Modes.stats_current.demod_cpu);
@@ -883,10 +946,6 @@ static void *decodeEntryPoint(void *arg) {
                 pthread_cond_signal(&Threads.reader.cond);
                 unlockReader();
 
-                Modes.stats_current.samples_lost += Modes.sdr_buf_samples - buf->length;
-
-                timingStatistics(buf);
-
                 watchdogCounter = 100; // roughly 10 seconds
             } else {
                 // Nothing to process this time around.
@@ -1446,7 +1505,12 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
             Modes.dev_name = strdup(arg);
             break;
         case OptGain:
-            Modes.gain = (int) (atof(arg)*10); // Gain is in tens of DBs
+            if (strcmp(arg, "auto") == 0) {
+                Modes.autoGain = 1;
+                Modes.gain = 439;
+            } else {
+                Modes.gain = (int) (atof(arg)*10); // Gain is in tens of DBs
+            }
             break;
         case OptFreq:
             Modes.freq = (int) strtoll(arg, NULL, 10);
diff --git a/readsb.h b/readsb.h
index 77a7ae22..a02e894a 100644
--- a/readsb.h
+++ b/readsb.h
@@ -456,6 +456,10 @@ struct mag_buf
     unsigned length; // Number of valid samples _after_ overlap. Total buffer length is buf->length + Modes.trailing_samples.
     int64_t sysTimestamp; // Estimated system time at start of block
     int64_t sysMicroseconds; // sysTimestamp in microseconds
+    uint32_t loudSamples;
+    uint32_t quietSamples;
+    uint32_t quiet2Samples;
+    uint32_t padding2;
     uint16_t *data; // Magnitude data. Starts with Modes.trailing_samples worth of overlap from the previous block
 #if defined(__arm__)
     /*padding 4 bytes*/
@@ -529,6 +533,9 @@ struct _Modes
     pthread_mutex_t sdrControlMutex;
     int8_t sdrInitialized;
     int8_t sdrOpenFailed;
+    int8_t increaseGain;
+    int8_t lowerGain;
+    int8_t autoGain;
     int gain;
     int dc_filter; // should we apply a DC filter?
     int enable_agc;
diff --git a/sdr_rtlsdr.c b/sdr_rtlsdr.c
index 587ac332..5123a583 100644
--- a/sdr_rtlsdr.c
+++ b/sdr_rtlsdr.c
@@ -91,7 +91,47 @@ void rtlsdrInitConfig() {
     RTLSDR.tunerAgcEnabled = 0;
 }
 
+static int getClosestGainIndex(int target) {
+    target = (target == MODES_MAX_GAIN ? 9999 : target);
+    int closest = 0;
+
+    for (int i = 0; i < RTLSDR.numgains; ++i) {
+        if (abs(RTLSDR.gains[i] - target) < abs(RTLSDR.gains[closest] - target)) {
+            closest = i;
+        }
+    }
+    return closest;
+}
+
 void rtlsdrSetGain() {
+    if (Modes.increaseGain || Modes.lowerGain) {
+        int closest = getClosestGainIndex(Modes.gain);
+        if (Modes.increaseGain) {
+            closest += Modes.increaseGain;
+        } else if (Modes.lowerGain) {
+            closest -= Modes.lowerGain;
+        }
+        if (closest >= RTLSDR.numgains) {
+            closest = RTLSDR.numgains - 1;
+        }
+        if (closest < 0) {
+            closest = 0;
+        }
+        Modes.increaseGain = 0;
+        Modes.lowerGain = 0;
+
+        if (Modes.gain == RTLSDR.gains[closest]) {
+            // same gain, nothing to do
+            return;
+        }
+
+        // change gain
+        Modes.gain = RTLSDR.gains[closest];
+    }
+
+    if (Modes.gain < 0) {
+        Modes.gain = 0;
+    }
     if (Modes.gain == MODES_AUTO_GAIN || Modes.gain >= 520) {
         Modes.gain = 590;
 
@@ -104,13 +144,7 @@ void rtlsdrSetGain() {
         }
     } else {
 
-        int target = (Modes.gain == MODES_MAX_GAIN ? 9999 : Modes.gain);
-        int closest = -1;
-
-        for (int i = 0; i < RTLSDR.numgains; ++i) {
-            if (closest == -1 || abs(RTLSDR.gains[i] - target) < abs(RTLSDR.gains[closest] - target))
-                closest = i;
-        }
+        int closest = getClosestGainIndex(Modes.gain);
         int newGain = RTLSDR.gains[closest];
 
         if (RTLSDR.tunerAgcEnabled) {

From e1b2cecc409027ce73d907f18c3ede30c6a9a622 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sat, 2 Nov 2024 01:31:35 +0100
Subject: [PATCH 64/90] default to --gain=auto

---
 debian/readsb.default | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/readsb.default b/debian/readsb.default
index 7ede7bdf..4f9324da 100644
--- a/debian/readsb.default
+++ b/debian/readsb.default
@@ -2,7 +2,7 @@
 # This is sourced by /etc/systemd/system/default.target.wants/readsb.service as
 # daemon startup configuration.
 
-RECEIVER_OPTIONS="--device 0 --device-type rtlsdr --gain -10 --ppm 0"
+RECEIVER_OPTIONS="--device 0 --device-type rtlsdr --gain auto --ppm 0"
 DECODER_OPTIONS="--max-range 450 --write-json-every 1"
 NET_OPTIONS="--net --net-heartbeat 60 --net-ro-size 1250 --net-ro-interval 0.05 --net-ri-port 30001 --net-ro-port 30002 --net-sbs-port 30003 --net-bi-port 30004,30104 --net-bo-port 30005"
 JSON_OPTIONS="--json-location-accuracy 2 --range-outline-hours 24"

From 739f6af7a51a11c1d45bade7069658a88c9a7507 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sat, 2 Nov 2024 09:46:46 +0100
Subject: [PATCH 65/90] tweak autogain

---
 demod_2400.c | 24 +++++++-------
 readsb.c     | 92 ++++++++++++++++++++++++++++++++++++----------------
 readsb.h     |  9 +++--
 sdr_rtlsdr.c |  3 +-
 4 files changed, 84 insertions(+), 44 deletions(-)

diff --git a/demod_2400.c b/demod_2400.c
index 01b51674..765328d8 100644
--- a/demod_2400.c
+++ b/demod_2400.c
@@ -290,12 +290,12 @@ void demodulate2400(struct mag_buf *mag) {
     uint16_t *statsProgress = m;
 
     const uint32_t statsWindow = MODES_SHORT_MSG_SAMPLES / 2; // half a short message
-    uint32_t loudSamples = 0;
-    uint32_t quietSamples = 0;
-    uint32_t quiet2Samples = 0;
-    uint32_t loudThreshold = 55000 * statsWindow; // max 65536
-    uint32_t quietThreshold = 800 * statsWindow;
-    uint32_t quiet2Threshold = 2 * quietThreshold;
+    uint32_t loudEvents = 0;
+    uint32_t noiseLowSamples = 0;
+    uint32_t noiseHighSamples = 0;
+    const uint32_t loudThreshold = Modes.loudThreshold * Modes.loudThreshold * statsWindow;
+    const uint32_t noiseLowThreshold = Modes.noiseLowThreshold * Modes.noiseLowThreshold * statsWindow;
+    const uint32_t noiseHighThreshold = Modes.noiseHighThreshold * Modes.noiseHighThreshold * statsWindow;
 
     for (; pa < stop; pa++) {
         int32_t pa_mag, base_noise, ref_level;
@@ -323,9 +323,9 @@ void demodulate2400(struct mag_buf *mag) {
             for (uint32_t i = 0; i < statsWindow; i++) {
                 magSum += pa[i];
             }
-            loudSamples += statsWindow * (magSum > loudThreshold);
-            quietSamples += statsWindow * (magSum < quietThreshold);
-            quiet2Samples += statsWindow * (magSum < quiet2Threshold);
+            loudEvents += (magSum > loudThreshold);
+            noiseLowSamples += statsWindow * (magSum < noiseLowThreshold);
+            noiseHighSamples += statsWindow * (magSum < noiseHighThreshold);
 
             statsProgress = pa + statsWindow;
         }
@@ -493,9 +493,9 @@ void demodulate2400(struct mag_buf *mag) {
         netUseMessage(mm);
     }
 
-    mag->loudSamples = loudSamples;
-    mag->quietSamples = quietSamples;
-    mag->quiet2Samples = quiet2Samples;
+    mag->loudEvents = loudEvents;
+    mag->noiseLowSamples = noiseLowSamples;
+    mag->noiseHighSamples = noiseHighSamples;
 
     /* update noise power */
     {
diff --git a/readsb.c b/readsb.c
index 69b1a22f..2eac7a27 100644
--- a/readsb.c
+++ b/readsb.c
@@ -116,6 +116,12 @@ static void configSetDefaults(void) {
 
     // Now initialise things that should not be 0/NULL to their defaults
     Modes.gain = MODES_MAX_GAIN;
+
+    // 8 bit autogain defaults, will be squared and compared against magnitude data
+    Modes.loudThreshold = 245;
+    Modes.noiseLowThreshold = 25;
+    Modes.noiseHighThreshold = 35;
+
     Modes.freq = MODES_DEFAULT_FREQ;
     Modes.check_crc = 1;
     Modes.net_heartbeat_interval = MODES_NET_HEARTBEAT_INTERVAL;
@@ -752,61 +758,76 @@ static void *globeBinEntryPoint(void *arg) {
 }
 
 static void gainStatistics(struct mag_buf *buf) {
-    static uint64_t loudSamples;
-    static uint64_t quietSamples;
-    static uint64_t quiet2Samples;
+    static uint64_t loudEvents;
+    static uint64_t noiseLowSamples;
+    static uint64_t noiseHighSamples;
     static uint64_t totalSamples;
     static int slowRise;
     static int lastGain;
 
-    loudSamples += buf->loudSamples;
-    quietSamples += buf->quietSamples;
-    quiet2Samples += buf->quiet2Samples;
+    loudEvents += buf->loudEvents;
+    noiseLowSamples += buf->noiseLowSamples;
+    noiseHighSamples += buf->noiseHighSamples;
     totalSamples += buf->length;
 
-    if (totalSamples < 2 * Modes.sample_rate) {
+    double interval = 0.5;
+
+    if (totalSamples < interval * Modes.sample_rate) {
         return;
     }
 
-    double loudPercent = loudSamples / (double) totalSamples * 100.0;
-    double quietPercent = quietSamples / (double) totalSamples * 100.0;
-    double quiet2Percent = quiet2Samples / (double) totalSamples * 100.0;
-
-    // reset
-    loudSamples = 0;
-    quietSamples = 0;
-    quiet2Samples = 0;
-    totalSamples = 0;
+    double noiseLowPercent = noiseLowSamples / (double) totalSamples * 100.0;
+    double noiseHighPercent = noiseHighSamples / (double) totalSamples * 100.0;
 
     if (!Modes.autoGain) {
-        // don't adjust anything
-        return;
+        goto reset;
     }
 
-    if (loudPercent > 0.05 || quiet2Percent < 0.05) {
+    // 29 gain values for typical rtl-sdr
+    // allow startup to sweep entire range quickly, half it for double steps
+    int starting = getUptime() < (29 / 2.0 * interval) * SECONDS;
+
+    char *action = "";
+    int noiseLow = noiseLowPercent > 5; // too many samples < noiseLowThreshold
+    int noiseHigh = noiseHighPercent < 0.2; // too few samples < noiseHighThreshold
+    int loud = loudEvents > 1;
+    if (loud || noiseHigh) {
+        action = "decreased";
         Modes.lowerGain = 1;
-    } else if (
-            quietPercent > 10.0
-            ) {
-        if (getUptime() < 1 * MINUTES || slowRise > 10) {
+    } else if (noiseLow) {
+        if (starting || slowRise > 15 / interval) {
             slowRise = 0;
             Modes.increaseGain = 1;
+            action = "increased";
         } else {
             slowRise++;
         }
     }
+
     if (Modes.increaseGain || Modes.lowerGain) {
-        if (getUptime() < 1 * MINUTES) {
+        if (starting) {
             Modes.lowerGain *= 2;
             Modes.increaseGain *= 2;
         }
+        sdrSetGain();
         if (Modes.gain != lastGain) {
+            if (loud) {
+                fprintf(stderr, "%9s gain. loudEvents: %4lld\n", action, (long long) loudEvents);
+            } else if (noiseHigh) {
+                fprintf(stderr, "%9s gain. noise high.\n", action);
+            } else if (noiseLow) {
+                fprintf(stderr, "%9s gain. noise low.\n", action);
+            }
+            //fprintf(stderr, "%9s gain.  noiseLow: %5.2f %%  noiseHigh: %5.2f %%  loudEvents: %4lld\n", action, noiseLowPercent, noiseHighPercent, (long long) loudEvents);
             lastGain = Modes.gain;
-            fprintf(stderr, "loud: %8.4f %% quiet: %8.4f %% quiet2 %8.4f %%\n", loudPercent, quietPercent, quiet2Percent);
         }
-        sdrSetGain();
     }
 
+reset:
+    loudEvents = 0;
+    noiseLowSamples = 0;
+    noiseHighSamples = 0;
+    totalSamples = 0;
 }
 
 
@@ -1505,9 +1526,24 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
             Modes.dev_name = strdup(arg);
             break;
         case OptGain:
-            if (strcmp(arg, "auto") == 0) {
+            if (strcasestr(arg, "auto") == arg) {
                 Modes.autoGain = 1;
-                Modes.gain = 439;
+                char *argdup = strdup(arg);
+                tokenize(&argdup, ",", token, maxTokens);
+                if (token[1]) {
+                    Modes.gain = (int) (atof(token[1])*10); // Gain is in tens of DBs
+                } else {
+                    Modes.gain = 439;
+                }
+                if (token[2]) {
+                    Modes.noiseLowThreshold = atoi(token[2]);
+                }
+                if (token[3]) {
+                    Modes.noiseHighThreshold = atoi(token[3]);
+                }
+                if (token[4]) {
+                    Modes.loudThreshold = atoi(token[4]);
+                }
             } else {
                 Modes.gain = (int) (atof(arg)*10); // Gain is in tens of DBs
             }
diff --git a/readsb.h b/readsb.h
index a02e894a..03c81085 100644
--- a/readsb.h
+++ b/readsb.h
@@ -456,9 +456,9 @@ struct mag_buf
     unsigned length; // Number of valid samples _after_ overlap. Total buffer length is buf->length + Modes.trailing_samples.
     int64_t sysTimestamp; // Estimated system time at start of block
     int64_t sysMicroseconds; // sysTimestamp in microseconds
-    uint32_t loudSamples;
-    uint32_t quietSamples;
-    uint32_t quiet2Samples;
+    uint32_t loudEvents;
+    uint32_t noiseLowSamples;
+    uint32_t noiseHighSamples;
     uint32_t padding2;
     uint16_t *data; // Magnitude data. Starts with Modes.trailing_samples worth of overlap from the previous block
 #if defined(__arm__)
@@ -536,6 +536,9 @@ struct _Modes
     int8_t increaseGain;
     int8_t lowerGain;
     int8_t autoGain;
+    uint32_t loudThreshold;
+    uint32_t noiseLowThreshold;
+    uint32_t noiseHighThreshold;
     int gain;
     int dc_filter; // should we apply a DC filter?
     int enable_agc;
diff --git a/sdr_rtlsdr.c b/sdr_rtlsdr.c
index 5123a583..66fe7f13 100644
--- a/sdr_rtlsdr.c
+++ b/sdr_rtlsdr.c
@@ -108,7 +108,8 @@ void rtlsdrSetGain() {
         int closest = getClosestGainIndex(Modes.gain);
         if (Modes.increaseGain) {
             closest += Modes.increaseGain;
-        } else if (Modes.lowerGain) {
+        }
+        if (Modes.lowerGain) {
             closest -= Modes.lowerGain;
         }
         if (closest >= RTLSDR.numgains) {

From f40bd618d50589864f842bf85d32ba1c72e0b7ca Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sat, 2 Nov 2024 12:56:53 +0100
Subject: [PATCH 66/90] allow rtl tuner agc for autogain

only log 1 line per gain change
---
 readsb.c     | 43 +++++++++++++++++++++++--------------------
 readsb.h     |  1 +
 sdr.c        |  4 ++--
 sdr.h        |  2 +-
 sdr_rtlsdr.c | 18 +++++++++++-------
 sdr_rtlsdr.h |  2 +-
 6 files changed, 39 insertions(+), 31 deletions(-)

diff --git a/readsb.c b/readsb.c
index 2eac7a27..3737458f 100644
--- a/readsb.c
+++ b/readsb.c
@@ -763,7 +763,7 @@ static void gainStatistics(struct mag_buf *buf) {
     static uint64_t noiseHighSamples;
     static uint64_t totalSamples;
     static int slowRise;
-    static int lastGain;
+    static int64_t nextRaiseAgc;
 
     loudEvents += buf->loudEvents;
     noiseLowSamples += buf->noiseLowSamples;
@@ -771,6 +771,7 @@ static void gainStatistics(struct mag_buf *buf) {
     totalSamples += buf->length;
 
     double interval = 0.5;
+    double riseTime = 15;
 
     if (totalSamples < interval * Modes.sample_rate) {
         return;
@@ -784,43 +785,45 @@ static void gainStatistics(struct mag_buf *buf) {
     }
 
     // 29 gain values for typical rtl-sdr
-    // allow startup to sweep entire range quickly, half it for double steps
-    int starting = getUptime() < (29 / 2.0 * interval) * SECONDS;
+    // allow startup to sweep entire range quickly, almost half it for double steps
+    int starting = getUptime() < (29 / 1.5 * interval) * SECONDS;
 
-    char *action = "";
     int noiseLow = noiseLowPercent > 5; // too many samples < noiseLowThreshold
-    int noiseHigh = noiseHighPercent < 0.2; // too few samples < noiseHighThreshold
+    int noiseHigh = noiseHighPercent < 1; // too few samples < noiseHighThreshold
     int loud = loudEvents > 1;
     if (loud || noiseHigh) {
-        action = "decreased";
         Modes.lowerGain = 1;
     } else if (noiseLow) {
-        if (starting || slowRise > 15 / interval) {
+        if (starting || slowRise > riseTime / interval) {
             slowRise = 0;
             Modes.increaseGain = 1;
-            action = "increased";
         } else {
             slowRise++;
         }
     }
 
+    if (Modes.increaseGain && Modes.gain == 496 && buf->sysTimestamp < nextRaiseAgc) {
+        goto reset;
+    }
     if (Modes.increaseGain || Modes.lowerGain) {
         if (starting) {
             Modes.lowerGain *= 2;
             Modes.increaseGain *= 2;
         }
-        sdrSetGain();
-        if (Modes.gain != lastGain) {
-            if (loud) {
-                fprintf(stderr, "%9s gain. loudEvents: %4lld\n", action, (long long) loudEvents);
-            } else if (noiseHigh) {
-                fprintf(stderr, "%9s gain. noise high.\n", action);
-            } else if (noiseLow) {
-                fprintf(stderr, "%9s gain. noise low.\n", action);
-            }
-            //fprintf(stderr, "%9s gain.  noiseLow: %5.2f %%  noiseHigh: %5.2f %%  loudEvents: %4lld\n", action, noiseLowPercent, noiseHighPercent, (long long) loudEvents);
-            lastGain = Modes.gain;
+        char *reason = "";
+        if (loud) {
+            reason = "decreasing gain, strong signal found: ";
+        } else if (noiseHigh) {
+            reason = "decreasing gain, noise too high:      ";
+        } else if (noiseLow) {
+            reason = "increasing gain, noise too low:       ";
+        }
+        sdrSetGain(reason);
+        if (Modes.gain == MODES_RTL_AGC) {
+            // switching to AGC is only done every 5 minutes to avoid oscillations due to the large step
+            nextRaiseAgc = buf->sysTimestamp + 5 * MINUTES;
         }
+        //fprintf(stderr, "%9s gain.  noiseLow: %5.2f %%  noiseHigh: %5.2f %%  loudEvents: %4lld\n", action, noiseLowPercent, noiseHighPercent, (long long) loudEvents);
     }
 
 reset:
@@ -2609,7 +2612,7 @@ static void checkSetGain() {
     double newGain = atof(tmp);
     Modes.gain = (int) (newGain * 10); // Gain is in tens of DBs
 
-    sdrSetGain();
+    sdrSetGain("");
 
     //fprintf(stderr, "Modes.gain (tens of dB): %d\n", Modes.gain);
 }
diff --git a/readsb.h b/readsb.h
index 03c81085..ec19f7ce 100644
--- a/readsb.h
+++ b/readsb.h
@@ -113,6 +113,7 @@
 #define MODES_MAG_BUFFERS       12                         // Number of magnitude buffers (should be smaller than RTL_BUFFERS for flowcontrol to work)
 #define MODES_AUTO_GAIN         -100                       // Use automatic gain
 #define MODES_MAX_GAIN          999999                     // Use max available gain
+#define MODES_RTL_AGC           590                        // Use rtl tuner agc
 #define MODEAC_MSG_BYTES        2
 
 #define MODES_PREAMBLE_US       8   // microseconds = bits
diff --git a/sdr.c b/sdr.c
index 588162ec..ff73bfc2 100644
--- a/sdr.c
+++ b/sdr.c
@@ -211,11 +211,11 @@ void sdrClose() {
     pthread_mutex_unlock(&Modes.sdrControlMutex);
 }
 
-void sdrSetGain(){
+void sdrSetGain(char *reason){
     pthread_mutex_lock(&Modes.sdrControlMutex);
 
     if (Modes.sdrInitialized) {
-        current_handler()->setGain();
+        current_handler()->setGain(reason);
     }
 
     pthread_mutex_unlock(&Modes.sdrControlMutex);
diff --git a/sdr.h b/sdr.h
index 53d3f088..fcd50ee3 100644
--- a/sdr.h
+++ b/sdr.h
@@ -34,7 +34,7 @@ void sdrRun ();
 bool sdrHasRun();
 void sdrCancel ();
 void sdrClose ();
-void sdrSetGain ();
+void sdrSetGain (char *reason);
 
 void lockReader();
 void unlockReader();
diff --git a/sdr_rtlsdr.c b/sdr_rtlsdr.c
index 66fe7f13..05aaea20 100644
--- a/sdr_rtlsdr.c
+++ b/sdr_rtlsdr.c
@@ -103,7 +103,7 @@ static int getClosestGainIndex(int target) {
     return closest;
 }
 
-void rtlsdrSetGain() {
+void rtlsdrSetGain(char *reason) {
     if (Modes.increaseGain || Modes.lowerGain) {
         int closest = getClosestGainIndex(Modes.gain);
         if (Modes.increaseGain) {
@@ -134,11 +134,10 @@ void rtlsdrSetGain() {
         Modes.gain = 0;
     }
     if (Modes.gain == MODES_AUTO_GAIN || Modes.gain >= 520) {
-        Modes.gain = 590;
+        Modes.gain = MODES_RTL_AGC;
 
         RTLSDR.tunerAgcEnabled = 1;
-
-        fprintf(stderr, "rtlsdr: enabling tuner AGC\n");
+        fprintf(stderr, "%srtlsdr: enabling tuner AGC\n", reason);
         if (rtlsdr_set_tuner_gain_mode(RTLSDR.dev, 0)) {
             fprintf(stderr, "rtlsdr: enabling tuner AGC failed\n");
             return;
@@ -160,7 +159,7 @@ void rtlsdrSetGain() {
             fprintf(stderr, "rtlsdr: setting tuner gain failed\n");
             return;
         } else {
-            fprintf(stderr, "rtlsdr: tuner gain set to %.1f dB\n", newGain / 10.0);
+            fprintf(stderr, "%srtlsdr: tuner gain set to %4.1f dB\n", reason, newGain / 10.0);
             Modes.gain = newGain;
         }
     }
@@ -276,15 +275,20 @@ bool rtlsdrOpen(void) {
         fprintf(stderr, "FATAL: rtlsdr: error getting tuner gains\n");
         return false;
     }
+    // one extra step for tuner agc / max
+    RTLSDR.numgains++;
 
     RTLSDR.gains = cmalloc(RTLSDR.numgains * sizeof (int));
-    if (rtlsdr_get_tuner_gains(RTLSDR.dev, RTLSDR.gains) != RTLSDR.numgains) {
+    if (rtlsdr_get_tuner_gains(RTLSDR.dev, RTLSDR.gains) != RTLSDR.numgains - 1) {
         fprintf(stderr, "FATAL: rtlsdr: error getting tuner gains\n");
         free(RTLSDR.gains);
         return false;
     }
 
-    rtlsdrSetGain();
+    // set agc at 590 (MODES_RTL_AGC)
+    RTLSDR.gains[RTLSDR.numgains - 1] = MODES_RTL_AGC;
+
+    rtlsdrSetGain("");
 
     if (RTLSDR.digital_agc) {
         fprintf(stderr, "rtlsdr: enabling digital AGC\n");
diff --git a/sdr_rtlsdr.h b/sdr_rtlsdr.h
index a9bd1675..7ae7b4f6 100644
--- a/sdr_rtlsdr.h
+++ b/sdr_rtlsdr.h
@@ -31,6 +31,6 @@ void rtlsdrRun ();
 void rtlsdrCancel();
 void rtlsdrClose ();
 bool rtlsdrHandleOption (int argc, char *argv);
-void rtlsdrSetGain();
+void rtlsdrSetGain(char *reason);
 
 #endif

From e4faf7fa55faf5736f81700fcb8c518309ca0c9d Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sat, 2 Nov 2024 13:15:05 +0100
Subject: [PATCH 67/90] incrementing version: 3.14.1645

---
 debian/changelog | 4 ++--
 version          | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 3d05051d..da2a062f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-readsb (3.14.1644) UNRELEASED; urgency=medium
+readsb (3.14.1645) UNRELEASED; urgency=medium
 
   * In development
 
- -- Matthias Wirth <matthias.wirth@gmail.com>  Fri, 01 Nov 2024 13:36:17 +0100
+ -- Matthias Wirth <matthias.wirth@gmail.com>  Sat, 02 Nov 2024 13:15:05 +0100
diff --git a/version b/version
index 9f2334ec..acf32846 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-3.14.1644
+3.14.1645

From 1e276882afc44097f53aee38f9639fe3e57432a6 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sat, 2 Nov 2024 19:58:04 +0100
Subject: [PATCH 68/90] autogain: silent by default

---
 readsb.c     | 76 ++++++++++++++++++++++++++++++++++++----------------
 readsb.h     |  2 ++
 sdr_rtlsdr.c |  8 ++++--
 3 files changed, 61 insertions(+), 25 deletions(-)

diff --git a/readsb.c b/readsb.c
index 3737458f..b36bac63 100644
--- a/readsb.c
+++ b/readsb.c
@@ -1520,6 +1520,48 @@ static int parseLongs(char *p, long long *results, int result_size) {
     return count;
 }
 
+static void parseGainOpt(char *arg) {
+    int maxTokens = 128;
+    char* token[maxTokens];
+    if (!arg) {
+        fprintf(stderr, "parseGainOpt called wiht argument null\n");
+        return;
+    }
+    if (strcasestr(arg, "auto") == arg) {
+        if (Modes.sdr_type != SDR_RTLSDR) {
+            fprintf(stderr, "autogain not supported for non rtl-sdr devices\n");
+        }
+        if (strcasestr(arg, "auto-verbose") == arg) {
+            fprintf(stderr, "autogain enabled, verbose mode\n");
+            Modes.gainQuiet = 0;
+        } else {
+            fprintf(stderr, "autogain enabled, silent mode, suppressing gain changing messages\n");
+            Modes.gainQuiet = 1;
+        }
+        Modes.autoGain = 1;
+        char *argdup = strdup(arg);
+        tokenize(&argdup, ",", token, maxTokens);
+        if (token[1]) {
+            Modes.gain = (int) (atof(token[1])*10); // Gain is in tens of DBs
+        } else {
+            Modes.gain = 439;
+        }
+        if (token[2]) {
+            Modes.noiseLowThreshold = atoi(token[2]);
+        }
+        if (token[3]) {
+            Modes.noiseHighThreshold = atoi(token[3]);
+        }
+        if (token[4]) {
+            Modes.loudThreshold = atoi(token[4]);
+        }
+    } else {
+        Modes.gain = (int) (atof(arg)*10); // Gain is in tens of DBs
+        Modes.autoGain = 0;
+        Modes.gainQuiet = 0;
+    }
+}
+
 static error_t parse_opt(int key, char *arg, struct argp_state *state) {
     //fprintf(stderr, "parse_opt(%d, %s, argp_state*)\n", key, arg);
     int maxTokens = 128;
@@ -1529,27 +1571,8 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
             Modes.dev_name = strdup(arg);
             break;
         case OptGain:
-            if (strcasestr(arg, "auto") == arg) {
-                Modes.autoGain = 1;
-                char *argdup = strdup(arg);
-                tokenize(&argdup, ",", token, maxTokens);
-                if (token[1]) {
-                    Modes.gain = (int) (atof(token[1])*10); // Gain is in tens of DBs
-                } else {
-                    Modes.gain = 439;
-                }
-                if (token[2]) {
-                    Modes.noiseLowThreshold = atoi(token[2]);
-                }
-                if (token[3]) {
-                    Modes.noiseHighThreshold = atoi(token[3]);
-                }
-                if (token[4]) {
-                    Modes.loudThreshold = atoi(token[4]);
-                }
-            } else {
-                Modes.gain = (int) (atof(arg)*10); // Gain is in tens of DBs
-            }
+            sfree(Modes.gainArg);
+            Modes.gainArg = strdup(arg);
             break;
         case OptFreq:
             Modes.freq = (int) strtoll(arg, NULL, 10);
@@ -2315,6 +2338,13 @@ static void configAfterParse() {
     Modes.sdr_buf_samples = Modes.sdr_buf_size / 2;
     Modes.trackExpireMax = Modes.trackExpireJaero + TRACK_EXPIRE_LONG + 1 * MINUTES;
 
+    if (Modes.sdr_type == SDR_RTLSDR && !Modes.gainArg) {
+        parseGainOpt("auto");
+    } else if (Modes.gainArg) {
+        parseGainOpt(Modes.gainArg);
+        sfree(Modes.gainArg);
+    }
+
     if (Modes.json_globe_index || Modes.globe_history_dir) {
         Modes.keep_traces = 24 * HOURS + 60 * MINUTES; // include 60 minutes overlap
         Modes.writeTraces = 1;
@@ -2609,8 +2639,8 @@ static void checkSetGain() {
 
     tmp[len] = '\0';
 
-    double newGain = atof(tmp);
-    Modes.gain = (int) (newGain * 10); // Gain is in tens of DBs
+
+    parseGainOpt(tmp);
 
     sdrSetGain("");
 
diff --git a/readsb.h b/readsb.h
index ec19f7ce..b1346b3a 100644
--- a/readsb.h
+++ b/readsb.h
@@ -537,6 +537,8 @@ struct _Modes
     int8_t increaseGain;
     int8_t lowerGain;
     int8_t autoGain;
+    int8_t gainQuiet;
+    char *gainArg;
     uint32_t loudThreshold;
     uint32_t noiseLowThreshold;
     uint32_t noiseHighThreshold;
diff --git a/sdr_rtlsdr.c b/sdr_rtlsdr.c
index 05aaea20..a78aa0c5 100644
--- a/sdr_rtlsdr.c
+++ b/sdr_rtlsdr.c
@@ -137,7 +137,9 @@ void rtlsdrSetGain(char *reason) {
         Modes.gain = MODES_RTL_AGC;
 
         RTLSDR.tunerAgcEnabled = 1;
-        fprintf(stderr, "%srtlsdr: enabling tuner AGC\n", reason);
+        if (!Modes.gainQuiet) {
+            fprintf(stderr, "%srtlsdr: enabling tuner AGC\n", reason);
+        }
         if (rtlsdr_set_tuner_gain_mode(RTLSDR.dev, 0)) {
             fprintf(stderr, "rtlsdr: enabling tuner AGC failed\n");
             return;
@@ -159,7 +161,9 @@ void rtlsdrSetGain(char *reason) {
             fprintf(stderr, "rtlsdr: setting tuner gain failed\n");
             return;
         } else {
-            fprintf(stderr, "%srtlsdr: tuner gain set to %4.1f dB\n", reason, newGain / 10.0);
+            if (!Modes.gainQuiet) {
+                fprintf(stderr, "%srtlsdr: tuner gain set to %4.1f dB\n", reason, newGain / 10.0);
+            }
             Modes.gain = newGain;
         }
     }

From 2c51b73c5f2e4128063a48c6603df35d72b8538c Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sat, 2 Nov 2024 20:17:05 +0100
Subject: [PATCH 69/90] make quiet autogain the default

---
 README.md | 17 +++++++++++++++++
 help.h    |  2 +-
 readsb.c  |  5 ++++-
 readsb.h  |  1 +
 4 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 340ae775..b9fa151f 100644
--- a/README.md
+++ b/README.md
@@ -91,6 +91,23 @@ The difference of using -Ofast or -O3 over the default of -O2 is likely very min
 
 If required, edit `/etc/default/readsb` to set the service options, device type, network ports etc.
 
+## Autogain
+
+For rtl-sdr devices a software gain algorithm is the default, it's optimized for ADS-B.
+On the command line it's activated using `--gain=auto` an is silent by default.
+`--gain=auto-verbose` can be used to enable log messages for gain changes.
+To tweak the internals, more parameters can be passed:
+```
+--gain=auto-verbose,<startingGain>,<noiseLowThreshold>,<noiseHighThreshold>,<loudThreshold>
+```
+The defaults are:
+```
+--gain=auto-verbose,43.9,25,35,245
+```
+The thresholds are numbers 0 to 256, tweaking them requires some understanding of how it works.
+One option would be to change the noise thresholds up or down and then observe the log.
+There should be no need to tweak these parameters.
+
 ## rtl-sdr bias tee
 
 Use this utility independent of readsb:
diff --git a/help.h b/help.h
index f2ec68d3..517968f2 100644
--- a/help.h
+++ b/help.h
@@ -68,7 +68,7 @@ static struct argp_option optionsReadsb[] = {
     {"filter-DF", OptFilterDF, "<type>", 0, "When displaying decoded ModeS messages on stdout only show this DF type", 1},
     {"aggressive", OptAggressive, 0, OPTION_HIDDEN, "Enable two-bit CRC error correction", 1},
     {"device-type", OptDeviceType, "<type>", 0, "Select SDR type (this needs to be placed on the command line before any SDR type specific options)", 1},
-    {"gain", OptGain, "<db>", 0, "Set gain (default: max gain, possible values for rtl-sdr devices: 0.0 0.9 1.4 2.7 3.7 7.7 8.7 12.5 14.4 15.7 16.6 19.7 20.7 22.9 25.4 28.0 29.7 32.8 33.8 36.4 37.2 38.6 40.2 42.1 43.4 43.9 44.5 48.0 49.6 58)", 1},
+    {"gain", OptGain, "<db>", 0, "Set gain (default: auto gain, possible values for rtl-sdr devices: auto auto-verbose 0.0 0.9 1.4 2.7 3.7 7.7 8.7 12.5 14.4 15.7 16.6 19.7 20.7 22.9 25.4 28.0 29.7 32.8 33.8 36.4 37.2 38.6 40.2 42.1 43.4 43.9 44.5 48.0 49.6 58)", 1},
     {"freq", OptFreq, "<hz>", 0, "Set frequency (default: 1090 MHz)", 1},
     {"interactive", OptInteractive, 0, 0, "Interactive mode refreshing data on screen. Implies --throttle", 1},
     {"raw", OptRaw, 0, 0, "Show only messages hex values", 1},
diff --git a/readsb.c b/readsb.c
index b36bac63..ede2e6e5 100644
--- a/readsb.c
+++ b/readsb.c
@@ -116,6 +116,8 @@ static void configSetDefaults(void) {
 
     // Now initialise things that should not be 0/NULL to their defaults
     Modes.gain = MODES_MAX_GAIN;
+    Modes.autoGain = 1;
+    Modes.gainQuiet = 1;
 
     // 8 bit autogain defaults, will be squared and compared against magnitude data
     Modes.loudThreshold = 245;
@@ -786,7 +788,7 @@ static void gainStatistics(struct mag_buf *buf) {
 
     // 29 gain values for typical rtl-sdr
     // allow startup to sweep entire range quickly, almost half it for double steps
-    int starting = getUptime() < (29 / 1.5 * interval) * SECONDS;
+    int starting = mono_milli_seconds() - Modes.gainStartupTime < (29 / 1.5 * interval) * SECONDS;
 
     int noiseLow = noiseLowPercent > 5; // too many samples < noiseLowThreshold
     int noiseHigh = noiseHighPercent < 1; // too few samples < noiseHighThreshold
@@ -1521,6 +1523,7 @@ static int parseLongs(char *p, long long *results, int result_size) {
 }
 
 static void parseGainOpt(char *arg) {
+    Modes.gainStartupTime = mono_milli_seconds();
     int maxTokens = 128;
     char* token[maxTokens];
     if (!arg) {
diff --git a/readsb.h b/readsb.h
index b1346b3a..af295563 100644
--- a/readsb.h
+++ b/readsb.h
@@ -539,6 +539,7 @@ struct _Modes
     int8_t autoGain;
     int8_t gainQuiet;
     char *gainArg;
+    int64_t gainStartupTime;
     uint32_t loudThreshold;
     uint32_t noiseLowThreshold;
     uint32_t noiseHighThreshold;

From 31378c2ee444197643f77402ef68a48721c91bc4 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sat, 2 Nov 2024 20:26:06 +0100
Subject: [PATCH 70/90] just log 59 dB instead of tuner AGC

---
 sdr_rtlsdr.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdr_rtlsdr.c b/sdr_rtlsdr.c
index a78aa0c5..5872e587 100644
--- a/sdr_rtlsdr.c
+++ b/sdr_rtlsdr.c
@@ -138,7 +138,7 @@ void rtlsdrSetGain(char *reason) {
 
         RTLSDR.tunerAgcEnabled = 1;
         if (!Modes.gainQuiet) {
-            fprintf(stderr, "%srtlsdr: enabling tuner AGC\n", reason);
+            fprintf(stderr, "%srtlsdr: tuner gain set to 59.0 dB (tuner AGC)\n", reason);
         }
         if (rtlsdr_set_tuner_gain_mode(RTLSDR.dev, 0)) {
             fprintf(stderr, "rtlsdr: enabling tuner AGC failed\n");

From 80fc54619e7217f1541ec4385f3569e0b66e588d Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sun, 3 Nov 2024 09:32:19 +0100
Subject: [PATCH 71/90] autogain: change default noise thresholds

---
 README.md | 2 +-
 readsb.c  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index b9fa151f..3b121c58 100644
--- a/README.md
+++ b/README.md
@@ -102,7 +102,7 @@ To tweak the internals, more parameters can be passed:
 ```
 The defaults are:
 ```
---gain=auto-verbose,43.9,25,35,245
+--gain=auto-verbose,43.9,24,28,245
 ```
 The thresholds are numbers 0 to 256, tweaking them requires some understanding of how it works.
 One option would be to change the noise thresholds up or down and then observe the log.
diff --git a/readsb.c b/readsb.c
index ede2e6e5..965221b4 100644
--- a/readsb.c
+++ b/readsb.c
@@ -121,8 +121,8 @@ static void configSetDefaults(void) {
 
     // 8 bit autogain defaults, will be squared and compared against magnitude data
     Modes.loudThreshold = 245;
-    Modes.noiseLowThreshold = 25;
-    Modes.noiseHighThreshold = 35;
+    Modes.noiseLowThreshold = 24;
+    Modes.noiseHighThreshold = 28;
 
     Modes.freq = MODES_DEFAULT_FREQ;
     Modes.check_crc = 1;

From 8fd5acf09fbf3a8b5273f5e9ac90e0cd80723d5d Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sun, 3 Nov 2024 09:46:36 +0100
Subject: [PATCH 72/90] autogain: print autogain settings

---
 readsb.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/readsb.c b/readsb.c
index 965221b4..67804ada 100644
--- a/readsb.c
+++ b/readsb.c
@@ -1558,6 +1558,8 @@ static void parseGainOpt(char *arg) {
         if (token[4]) {
             Modes.loudThreshold = atoi(token[4]);
         }
+        fprintf(stderr, "startingGain: %4.1f noiseLowThreshold: %3d noiseHighThreshold: %3d loudThreshold: %3d\n",
+                Modes.gain / 10.0, Modes.noiseLowThreshold, Modes.noiseHighThreshold, Modes.loudThreshold);
     } else {
         Modes.gain = (int) (atof(arg)*10); // Gain is in tens of DBs
         Modes.autoGain = 0;

From 89fc77451a611d7db426308edd0c0c8019c69fb7 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sun, 3 Nov 2024 10:57:13 +0100
Subject: [PATCH 73/90] autogain defaults tweak

---
 README.md | 2 +-
 readsb.c  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 3b121c58..b4fe126b 100644
--- a/README.md
+++ b/README.md
@@ -102,7 +102,7 @@ To tweak the internals, more parameters can be passed:
 ```
 The defaults are:
 ```
---gain=auto-verbose,43.9,24,28,245
+--gain=auto-verbose,43.9,25,31,245
 ```
 The thresholds are numbers 0 to 256, tweaking them requires some understanding of how it works.
 One option would be to change the noise thresholds up or down and then observe the log.
diff --git a/readsb.c b/readsb.c
index 67804ada..188a83c0 100644
--- a/readsb.c
+++ b/readsb.c
@@ -121,8 +121,8 @@ static void configSetDefaults(void) {
 
     // 8 bit autogain defaults, will be squared and compared against magnitude data
     Modes.loudThreshold = 245;
-    Modes.noiseLowThreshold = 24;
-    Modes.noiseHighThreshold = 28;
+    Modes.noiseLowThreshold = 25;
+    Modes.noiseHighThreshold = 31;
 
     Modes.freq = MODES_DEFAULT_FREQ;
     Modes.check_crc = 1;

From b4bd2f5c632dbc585fc29939851c2c8491e8fefa Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sun, 3 Nov 2024 14:40:19 +0100
Subject: [PATCH 74/90] autogain: reduce default loudThreshold

---
 README.md | 2 +-
 readsb.c  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index b4fe126b..3bce11ef 100644
--- a/README.md
+++ b/README.md
@@ -102,7 +102,7 @@ To tweak the internals, more parameters can be passed:
 ```
 The defaults are:
 ```
---gain=auto-verbose,43.9,25,31,245
+--gain=auto-verbose,43.9,25,31,243
 ```
 The thresholds are numbers 0 to 256, tweaking them requires some understanding of how it works.
 One option would be to change the noise thresholds up or down and then observe the log.
diff --git a/readsb.c b/readsb.c
index 188a83c0..b885451f 100644
--- a/readsb.c
+++ b/readsb.c
@@ -120,7 +120,7 @@ static void configSetDefaults(void) {
     Modes.gainQuiet = 1;
 
     // 8 bit autogain defaults, will be squared and compared against magnitude data
-    Modes.loudThreshold = 245;
+    Modes.loudThreshold = 243;
     Modes.noiseLowThreshold = 25;
     Modes.noiseHighThreshold = 31;
 

From 7be4ddf30d29c68cd33ef350ff238602e96c20ed Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sun, 3 Nov 2024 19:16:13 +0100
Subject: [PATCH 75/90] autogain: faster startup, faster reaction to very loud
 signals

---
 readsb.c | 29 ++++++++++++++++++-----------
 readsb.h |  2 +-
 2 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/readsb.c b/readsb.c
index b885451f..fba354ae 100644
--- a/readsb.c
+++ b/readsb.c
@@ -788,15 +788,18 @@ static void gainStatistics(struct mag_buf *buf) {
 
     // 29 gain values for typical rtl-sdr
     // allow startup to sweep entire range quickly, almost half it for double steps
-    int starting = mono_milli_seconds() - Modes.gainStartupTime < (29 / 1.5 * interval) * SECONDS;
 
     int noiseLow = noiseLowPercent > 5; // too many samples < noiseLowThreshold
     int noiseHigh = noiseHighPercent < 1; // too few samples < noiseHighThreshold
     int loud = loudEvents > 1;
+    int veryLoud = loudEvents > 5;
     if (loud || noiseHigh) {
         Modes.lowerGain = 1;
+        if (veryLoud && !Modes.gainStartup) {
+            Modes.lowerGain = 2;
+        }
     } else if (noiseLow) {
-        if (starting || slowRise > riseTime / interval) {
+        if (Modes.gainStartup || slowRise > riseTime / interval) {
             slowRise = 0;
             Modes.increaseGain = 1;
         } else {
@@ -804,21 +807,24 @@ static void gainStatistics(struct mag_buf *buf) {
         }
     }
 
+
     if (Modes.increaseGain && Modes.gain == 496 && buf->sysTimestamp < nextRaiseAgc) {
         goto reset;
     }
     if (Modes.increaseGain || Modes.lowerGain) {
-        if (starting) {
-            Modes.lowerGain *= 2;
-            Modes.increaseGain *= 2;
+        if (Modes.gainStartup) {
+            Modes.lowerGain *= Modes.gainStartup;
+            Modes.increaseGain *= Modes.gainStartup;
         }
         char *reason = "";
-        if (loud) {
-            reason = "decreasing gain, strong signal found: ";
+        if (veryLoud) {
+            reason = "decreasing gain, many strong signals found: ";
+        } else if (loud) {
+            reason = "decreasing gain, strong signal found:       ";
         } else if (noiseHigh) {
-            reason = "decreasing gain, noise too high:      ";
+            reason = "decreasing gain, noise too high:            ";
         } else if (noiseLow) {
-            reason = "increasing gain, noise too low:       ";
+            reason = "increasing gain, noise too low:             ";
         }
         sdrSetGain(reason);
         if (Modes.gain == MODES_RTL_AGC) {
@@ -829,6 +835,7 @@ static void gainStatistics(struct mag_buf *buf) {
     }
 
 reset:
+    Modes.gainStartup /= 2;
     loudEvents = 0;
     noiseLowSamples = 0;
     noiseHighSamples = 0;
@@ -1523,7 +1530,7 @@ static int parseLongs(char *p, long long *results, int result_size) {
 }
 
 static void parseGainOpt(char *arg) {
-    Modes.gainStartupTime = mono_milli_seconds();
+    Modes.gainStartup = 8;
     int maxTokens = 128;
     char* token[maxTokens];
     if (!arg) {
@@ -1547,7 +1554,7 @@ static void parseGainOpt(char *arg) {
         if (token[1]) {
             Modes.gain = (int) (atof(token[1])*10); // Gain is in tens of DBs
         } else {
-            Modes.gain = 439;
+            Modes.gain = 300;
         }
         if (token[2]) {
             Modes.noiseLowThreshold = atoi(token[2]);
diff --git a/readsb.h b/readsb.h
index af295563..8bbf2014 100644
--- a/readsb.h
+++ b/readsb.h
@@ -538,8 +538,8 @@ struct _Modes
     int8_t lowerGain;
     int8_t autoGain;
     int8_t gainQuiet;
+    int8_t gainStartup;
     char *gainArg;
-    int64_t gainStartupTime;
     uint32_t loudThreshold;
     uint32_t noiseLowThreshold;
     uint32_t noiseHighThreshold;

From 15823dd47fbd8b4346334317b77ff4dc02912078 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sun, 3 Nov 2024 19:34:26 +0100
Subject: [PATCH 76/90] autogain: replace startingGain with lowestGain
 parameter

---
 README.md |  4 ++--
 readsb.c  | 20 +++++++++++---------
 readsb.h  |  1 +
 3 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index 3bce11ef..e6fc8468 100644
--- a/README.md
+++ b/README.md
@@ -98,11 +98,11 @@ On the command line it's activated using `--gain=auto` an is silent by default.
 `--gain=auto-verbose` can be used to enable log messages for gain changes.
 To tweak the internals, more parameters can be passed:
 ```
---gain=auto-verbose,<startingGain>,<noiseLowThreshold>,<noiseHighThreshold>,<loudThreshold>
+--gain=auto-verbose,<lowestGain>,<noiseLowThreshold>,<noiseHighThreshold>,<loudThreshold>
 ```
 The defaults are:
 ```
---gain=auto-verbose,43.9,25,31,243
+--gain=auto-verbose,0,25,31,243
 ```
 The thresholds are numbers 0 to 256, tweaking them requires some understanding of how it works.
 One option would be to change the noise thresholds up or down and then observe the log.
diff --git a/readsb.c b/readsb.c
index fba354ae..6cbfbb5a 100644
--- a/readsb.c
+++ b/readsb.c
@@ -116,13 +116,6 @@ static void configSetDefaults(void) {
 
     // Now initialise things that should not be 0/NULL to their defaults
     Modes.gain = MODES_MAX_GAIN;
-    Modes.autoGain = 1;
-    Modes.gainQuiet = 1;
-
-    // 8 bit autogain defaults, will be squared and compared against magnitude data
-    Modes.loudThreshold = 243;
-    Modes.noiseLowThreshold = 25;
-    Modes.noiseHighThreshold = 31;
 
     Modes.freq = MODES_DEFAULT_FREQ;
     Modes.check_crc = 1;
@@ -1549,21 +1542,29 @@ static void parseGainOpt(char *arg) {
             Modes.gainQuiet = 1;
         }
         Modes.autoGain = 1;
+        Modes.gain = 300;
+
         char *argdup = strdup(arg);
         tokenize(&argdup, ",", token, maxTokens);
         if (token[1]) {
-            Modes.gain = (int) (atof(token[1])*10); // Gain is in tens of DBs
+            Modes.minGain = (int) (atof(token[1])*10); // Gain is in tens of DBs
         } else {
-            Modes.gain = 300;
+            Modes.minGain = 0;
         }
         if (token[2]) {
             Modes.noiseLowThreshold = atoi(token[2]);
+        } else {
+            Modes.noiseLowThreshold = 25;
         }
         if (token[3]) {
             Modes.noiseHighThreshold = atoi(token[3]);
+        } else {
+            Modes.noiseHighThreshold = 31;
         }
         if (token[4]) {
             Modes.loudThreshold = atoi(token[4]);
+        } else {
+            Modes.loudThreshold = 243;
         }
         fprintf(stderr, "startingGain: %4.1f noiseLowThreshold: %3d noiseHighThreshold: %3d loudThreshold: %3d\n",
                 Modes.gain / 10.0, Modes.noiseLowThreshold, Modes.noiseHighThreshold, Modes.loudThreshold);
@@ -1571,6 +1572,7 @@ static void parseGainOpt(char *arg) {
         Modes.gain = (int) (atof(arg)*10); // Gain is in tens of DBs
         Modes.autoGain = 0;
         Modes.gainQuiet = 0;
+        Modes.minGain = 0;
     }
 }
 
diff --git a/readsb.h b/readsb.h
index 8bbf2014..b1e9557c 100644
--- a/readsb.h
+++ b/readsb.h
@@ -543,6 +543,7 @@ struct _Modes
     uint32_t loudThreshold;
     uint32_t noiseLowThreshold;
     uint32_t noiseHighThreshold;
+    int minGain;
     int gain;
     int dc_filter; // should we apply a DC filter?
     int enable_agc;

From c67e6710eb38ebb77a256288967df990c06400d1 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Mon, 4 Nov 2024 14:54:06 +0100
Subject: [PATCH 77/90] remove adsbexchange hardcoding

to send uuid, use beast_reduce_plus_out with uuid in net-connector
or use beast_reduce_plus_out with --uuid-file
---
 README.md | 10 +++++++++-
 help.h    |  2 +-
 net_io.c  | 13 ++++---------
 readsb.c  |  1 -
 track.c   |  4 ++++
 5 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index e6fc8468..2da38275 100644
--- a/README.md
+++ b/README.md
@@ -56,6 +56,14 @@ Selectively forwards beast messages if the received data hasn't been forwarded i
 Data not related to the physical aircraft state are only forwarded every 500 ms (4 * `--net-beast-reduce-interval`).The messages of
 this output are normal beast messages and compatible with every program able to receive beast messages.
 
+This is used by some aggregators to aggregate ADS-B data, an example net connector would be:
+```
+--net-connector=feed.airplanes.live,30004,beast_reduce_plus_out,uuid=0033062d-e17e-4389-91a9-79ebb967fb4c
+```
+The uuid is optional, if none is given, the uuid from --uuid-file is used, if that isn't present no uuid is sent.
+The beast_reduce_out net-connector will never send an uuid.
+The aggregator enables --net-receiver-id and --net-ingest on their readsb server, it's made to work with beast_reduce_plus_out.
+
 ### Debian package
 
 - Build package with no additional receiver library dependencies: `dpkg-buildpackage -b`.
@@ -121,9 +129,9 @@ For that purpose it's used in conjunction with tar1090 with some extra options t
 Websites using this software:
 
 - https://adsb.lol/
-- https://globe.adsbexchange.com/
 - https://globe.airplanes.live/
 - https://globe.adsb.fi/
+- https://globe.adsbexchange.com/
 
 Projects that use or have used data generated by this software:
 
diff --git a/help.h b/help.h
index 517968f2..c4e8a33d 100644
--- a/help.h
+++ b/help.h
@@ -46,7 +46,7 @@ static struct argp_option optionsViewadsb[] = {
     {"quiet", OptQuiet, 0, 0, "Disable output (default)", 1},
     {"debug", OptDebug, "<flags>", 0, "Debug mode (verbose), n: network, P: CPR, S: speed check", 1},
     {0,0,0,0, "Network options:", 2},
-    {"net-connector", OptNetConnector, "<ip,port,protocol>", 0, "Establish connection, can be specified multiple times (viewadsb default: --net-connector 127.0.0.1,30005,beast_in viewadsb first usage overrides default, second usage adds another input/output) Protocols: beast_out, beast_in, raw_out, raw_in, sbs_in, sbs_in_jaero, sbs_out, sbs_out_jaero, vrs_out, json_out, gpsd_in, uat_in, uat_replay_out, planefinder_in, asterix_in, asterix_out (one failover ip/address,port can be specified: primary-address,primary-port,protocol,failover-address,failover-port) (any position in the comma separated list can also be either silent_fail or uuid=<uuid>)", 2},
+    {"net-connector", OptNetConnector, "<ip,port,protocol>", 0, "Establish connection, can be specified multiple times (viewadsb default: --net-connector 127.0.0.1,30005,beast_in viewadsb first usage overrides default, second usage adds another input/output) Protocols: beast_out, beast_reduce_out, beast_reduce_plus_out, beast_in, raw_out, raw_in, sbs_in, sbs_in_jaero, sbs_out, sbs_out_jaero, vrs_out, json_out, gpsd_in, uat_in, uat_replay_out, planefinder_in, asterix_in, asterix_out (one failover ip/address,port can be specified: primary-address,primary-port,protocol,failover-address,failover-port) (any position in the comma separated list can also be either silent_fail or uuid=<uuid>)", 2},
     {0,0,0,0, "Help options:", 100},
     { 0 }
 };
diff --git a/net_io.c b/net_io.c
index acfb5a27..098c5c82 100644
--- a/net_io.c
+++ b/net_io.c
@@ -371,23 +371,19 @@ static struct client *createSocketClient(struct net_service *service, int fd) {
 
 static int sendUUID(struct client *c, int64_t now) {
     struct net_connector *con = c->con;
-    // sending UUID if hostname matches adsbexchange or for beast_reduce_plus output
+    // sending UUID for beast_reduce_plus output
     char uuid[150];
     uuid[0] = '\0';
     if ((c->sendq && c->sendq_len + 256 < c->sendq_max) && con
-            && (con->enable_uuid_ping || (strstr(con->address, "feed") && strstr(con->address, ".adsbexchange.com")) || Modes.debug_ping || Modes.debug_send_uuid)) {
+            && (con->enable_uuid_ping || Modes.debug_ping || Modes.debug_send_uuid)) {
 
         int res = -1;
 
         if (con->uuid) {
             strncpy(uuid, con->uuid, 135);
             res = strlen(uuid);
-        } else {
+        } else if (Modes.uuidFile) {
             int fd = open(Modes.uuidFile, O_RDONLY);
-            // try legacy / adsbexchange image path as hardcoded fallback
-            if (fd == -1) {
-                fd = open("/boot/adsbx-uuid", O_RDONLY);
-            }
             if (fd != -1) {
                 res = read(fd, uuid, 128);
                 close(fd);
@@ -408,9 +404,8 @@ static int sendUUID(struct client *c, int64_t now) {
             strncpy(c->sendq + c->sendq_len, uuid, res);
             c->sendq_len += 36;
         } else {
+            fprintf(stderr, "ERROR: Not a valid UUID: '%s' (to generate a valid uuid use this command: cat /proc/sys/kernel/random/uuid)\n", uuid);
             uuid[0] = '\0';
-            fprintf(stderr, "ERROR: Not a valid UUID: %s\n", Modes.uuidFile);
-            fprintf(stderr, "Use this command to fix: sudo uuidgen > %s\n", Modes.uuidFile);
         }
 
         // enable ping stuff
diff --git a/readsb.c b/readsb.c
index 6cbfbb5a..46e63a7b 100644
--- a/readsb.c
+++ b/readsb.c
@@ -157,7 +157,6 @@ static void configSetDefaults(void) {
     Modes.net_output_flush_interval_beast_reduce = -1; // default to net_output_flush_interval after config parse if not configured
     Modes.netReceiverId = 0;
     Modes.netIngest = 0;
-    Modes.uuidFile = strdup("/usr/local/share/adsbexchange/adsbx-uuid");
     Modes.json_trace_interval = 20 * 1000;
     Modes.state_write_interval = 1 * HOURS;
     Modes.heatmap_current_interval = -15;
diff --git a/track.c b/track.c
index 7f82b36d..8962c6c5 100644
--- a/track.c
+++ b/track.c
@@ -2620,6 +2620,10 @@ struct aircraft *trackUpdateFromMessage(struct modesMessage *mm) {
         }
     }
 
+    if (0 && a->addr == Modes.cpr_focus && mm->cpr_valid) {
+        displayModesMessage(mm);
+    }
+
     if (cpr_new) {
         a->last_cpr_type = mm->cpr_type;
     }

From 7c34ff39b17cbc358fae46684cca23b44a162799 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Tue, 5 Nov 2024 08:15:52 +0100
Subject: [PATCH 78/90] fixup autogain min gain setting

---
 readsb.c     | 5 ++++-
 sdr_rtlsdr.c | 4 ++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/readsb.c b/readsb.c
index 46e63a7b..217296f2 100644
--- a/readsb.c
+++ b/readsb.c
@@ -1550,6 +1550,9 @@ static void parseGainOpt(char *arg) {
         } else {
             Modes.minGain = 0;
         }
+        if (Modes.gain < Modes.minGain) {
+            Modes.gain = Modes.minGain;
+        }
         if (token[2]) {
             Modes.noiseLowThreshold = atoi(token[2]);
         } else {
@@ -1565,7 +1568,7 @@ static void parseGainOpt(char *arg) {
         } else {
             Modes.loudThreshold = 243;
         }
-        fprintf(stderr, "startingGain: %4.1f noiseLowThreshold: %3d noiseHighThreshold: %3d loudThreshold: %3d\n",
+        fprintf(stderr, "lowestGain: %4.1f noiseLowThreshold: %3d noiseHighThreshold: %3d loudThreshold: %3d\n",
                 Modes.gain / 10.0, Modes.noiseLowThreshold, Modes.noiseHighThreshold, Modes.loudThreshold);
     } else {
         Modes.gain = (int) (atof(arg)*10); // Gain is in tens of DBs
diff --git a/sdr_rtlsdr.c b/sdr_rtlsdr.c
index 5872e587..fb3bb470 100644
--- a/sdr_rtlsdr.c
+++ b/sdr_rtlsdr.c
@@ -121,6 +121,10 @@ void rtlsdrSetGain(char *reason) {
         Modes.increaseGain = 0;
         Modes.lowerGain = 0;
 
+        if (RTLSDR.gains[closest] < Modes.minGain) {
+            // new gain less than minimum, nothing to do
+            return;
+        }
         if (Modes.gain == RTLSDR.gains[closest]) {
             // same gain, nothing to do
             return;

From e8de536829ae5c69b3f8e2e6d13333c5cbeec378 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Fri, 15 Nov 2024 10:45:04 +0100
Subject: [PATCH 79/90] add range outline reset via setGain file

echo resetRangeOutline > /run/readsb/setGain
---
 readsb.c | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/readsb.c b/readsb.c
index 217296f2..d79e9a69 100644
--- a/readsb.c
+++ b/readsb.c
@@ -2634,6 +2634,13 @@ static void checkReplaceState() {
     checkReplaceStateDir(Modes.json_dir);
 }
 
+static void writeOutlineJson() {
+    if (!Modes.json_dir) {
+        return;
+    }
+    free(writeJsonToFile(Modes.json_dir, "outline.json", generateOutlineJson()).buffer);
+}
+
 static void checkSetGain() {
     if (!Modes.json_dir) {
         return;
@@ -2655,6 +2662,12 @@ static void checkSetGain() {
 
     tmp[len] = '\0';
 
+    if (strcasestr(tmp, "resetRangeOutline")) {
+        fprintf(stderr, "resetting range outline\n");
+        memset(Modes.rangeDirs, 0, sizeof(Modes.rangeDirs));
+        writeOutlineJson();
+        return;
+    }
 
     parseGainOpt(tmp);
 
@@ -2670,7 +2683,7 @@ static void miscStuff(int64_t now) {
     if (Modes.outline_json) {
         static int64_t nextOutlineWrite;
         if (now > nextOutlineWrite) {
-            free(writeJsonToFile(Modes.json_dir, "outline.json", generateOutlineJson()).buffer);
+            writeOutlineJson();
             nextOutlineWrite = now + 15 * SECONDS;
         }
 

From 079fda30a6756cf770ad88e03be16c448d50f956 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Mon, 18 Nov 2024 22:23:37 +0100
Subject: [PATCH 80/90] fixup --gain -10

was broken while implementing autogain
bad setting regardless, use 58 instead of -10 to set maximum gain on
rtl-sdr
---
 sdr_rtlsdr.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/sdr_rtlsdr.c b/sdr_rtlsdr.c
index fb3bb470..18de7d2d 100644
--- a/sdr_rtlsdr.c
+++ b/sdr_rtlsdr.c
@@ -104,6 +104,10 @@ static int getClosestGainIndex(int target) {
 }
 
 void rtlsdrSetGain(char *reason) {
+    if (Modes.gain == MODES_AUTO_GAIN || Modes.gain >= 520) {
+        Modes.gain = MODES_RTL_AGC;
+    }
+
     if (Modes.increaseGain || Modes.lowerGain) {
         int closest = getClosestGainIndex(Modes.gain);
         if (Modes.increaseGain) {
@@ -137,8 +141,7 @@ void rtlsdrSetGain(char *reason) {
     if (Modes.gain < 0) {
         Modes.gain = 0;
     }
-    if (Modes.gain == MODES_AUTO_GAIN || Modes.gain >= 520) {
-        Modes.gain = MODES_RTL_AGC;
+    if (Modes.gain == MODES_RTL_AGC) {
 
         RTLSDR.tunerAgcEnabled = 1;
         if (!Modes.gainQuiet) {

From c3214b36f7962793917e5830500bf611c1a04060 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Tue, 19 Nov 2024 16:39:37 +0100
Subject: [PATCH 81/90] longer dwell time for aircraft when writing traces
 without globe index

---
 track.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/track.c b/track.c
index 8962c6c5..39c694fb 100644
--- a/track.c
+++ b/track.c
@@ -2841,7 +2841,7 @@ static void removeStaleRange(void *arg, threadpool_threadbuffers_t * buffer_grou
     // timeout for non-ICAO aircraft with position
     int64_t nonIcaoPosTimeout = now - 30 * MINUTES;
 
-    if (Modes.json_globe_index) {
+    if (Modes.writeTraces) {
         posTimeout = now - 26 * HOURS;
         nonIcaoPosTimeout = now - 26 * HOURS;
     }

From 2b633849664885266e8e8e1ed5c37afc12b409e1 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Tue, 10 Dec 2024 22:31:21 +0000
Subject: [PATCH 82/90] update --net-only help text

---
 help.h   | 2 +-
 readsb.c | 7 +++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/help.h b/help.h
index c4e8a33d..a5098d4d 100644
--- a/help.h
+++ b/help.h
@@ -119,7 +119,7 @@ static struct argp_option optionsReadsb[] = {
     {0,0,0,0, "Network options:", 2},
     {"net-connector", OptNetConnector, "<ip,port,protocol>", 0, "Establish connection, can be specified multiple times (e.g. 127.0.0.1,23004,beast_out) Protocols: beast_out, beast_in, raw_out, raw_in, sbs_in, sbs_in_jaero, sbs_out, sbs_out_jaero, vrs_out, json_out, gpsd_in, uat_in, uat_replay_out, planefinder_in, asterix_in, asterix_out (one failover ip/address,port can be specified: primary-address,primary-port,protocol,failover-address,failover-port) (any position in the comma separated list can also be either silent_fail or uuid=<uuid>)", 2},
     {"net", OptNet, 0, 0, "Enable networking", 2},
-    {"net-only", OptNetOnly, 0, 0, "Enable just networking, no RTL device or file used", 2},
+    {"net-only", OptNetOnly, 0, 0, "Legacy Option, Enable networking, use --net instead", 2},
     {"net-bind-address", OptNetBindAddr, "<ip>", 0, "IP address to bind to (default: Any; Use 127.0.0.1 for private)", 2},
     {"net-bo-port", OptNetBoPorts, "<ports>", 0, "TCP Beast output listen ports (default: 0)", 2},
     {"net-bi-port", OptNetBiPorts, "<ports>", 0, "TCP Beast input listen ports  (default: 0)", 2},
diff --git a/readsb.c b/readsb.c
index d79e9a69..527a93a7 100644
--- a/readsb.c
+++ b/readsb.c
@@ -1617,16 +1617,15 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
         case OptNet:
             Modes.net = 1;
             break;
+        case OptNetOnly:
+            Modes.net = 1;
+            break;
         case OptModeAc:
             Modes.mode_ac = 1;
             break;
         case OptModeAcAuto:
             Modes.mode_ac_auto = 1;
             break;
-        case OptNetOnly:
-            Modes.net = 1;
-            Modes.net_only = 1;
-            break;
         case OptQuiet:
             Modes.quiet = 1;
             break;

From fb844975428d1cf9f944348446168a7874d9be5e Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Wed, 20 Nov 2024 21:42:27 +0100
Subject: [PATCH 83/90] non rtl-sdr: use default gain when gain=auto passed

---
 readsb.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/readsb.c b/readsb.c
index 527a93a7..5e3b385a 100644
--- a/readsb.c
+++ b/readsb.c
@@ -1532,6 +1532,7 @@ static void parseGainOpt(char *arg) {
     if (strcasestr(arg, "auto") == arg) {
         if (Modes.sdr_type != SDR_RTLSDR) {
             fprintf(stderr, "autogain not supported for non rtl-sdr devices\n");
+            return;
         }
         if (strcasestr(arg, "auto-verbose") == arg) {
             fprintf(stderr, "autogain enabled, verbose mode\n");

From 9cf1af0a1869119e0d944fff72ccc8e8fe91ab24 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Tue, 26 Nov 2024 14:53:24 +0000
Subject: [PATCH 84/90] readProxy setting

---
 net_io.c | 2 +-
 readsb.c | 3 +++
 readsb.h | 1 +
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/net_io.c b/net_io.c
index 098c5c82..c926df2b 100644
--- a/net_io.c
+++ b/net_io.c
@@ -5074,7 +5074,7 @@ static int processClient(struct client *c, int64_t now, struct messageBuffer *mb
 
     if (now - c->connectedSince < 5 * SECONDS) {
         // check for PROXY v1 header if connection is new / low bytes received
-        if (Modes.netIngest && c->som == c->buf) {
+        if ((Modes.netIngest || Modes.readProxy) && c->som == c->buf) {
             if (c->eod - c->som >= 6 && c->som[0] == 'P' && c->som[1] == 'R') {
                 int res = readProxy(c);
                 if (res != 0) {
diff --git a/readsb.c b/readsb.c
index 5e3b385a..46b39b98 100644
--- a/readsb.c
+++ b/readsb.c
@@ -2041,6 +2041,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
                 if (strcasecmp(token[0], "legacy_history") == 0) {
                     Modes.legacy_history = 1;
                 }
+                if (strcasecmp(token[0], "readProxy") == 0) {
+                    Modes.readProxy = 1;
+                }
                 if (strcasecmp(token[0], "beast_forward_noforward") == 0) {
                     Modes.beast_forward_noforward = 1;
                 }
diff --git a/readsb.h b/readsb.h
index b1e9557c..ad7d739c 100644
--- a/readsb.h
+++ b/readsb.h
@@ -709,6 +709,7 @@ struct _Modes
     int8_t netReceiverIdPrint;
     int8_t netReceiverIdJson;
     int8_t netIngest;
+    int8_t readProxy;
     int8_t enableClientsJson;
     int8_t forward_mlat; // forward beast mlat messages to beast output ports
     int8_t forward_mlat_sbs; // forward mlat messages to sbs output ports

From 038b4857678b5f7a25fc08d04f7052c4be01d322 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Tue, 31 Dec 2024 17:21:42 +0000
Subject: [PATCH 85/90] support non-icao addresses in SBS input/output

as in the json, non-icao addresses are prepended with ~
---
 net_io.c | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/net_io.c b/net_io.c
index c926df2b..8f226430 100644
--- a/net_io.c
+++ b/net_io.c
@@ -3009,10 +3009,15 @@ static int decodeSbsLine(struct client *c, char *line, int remote, int64_t now,
 
     mm->sbsMsgType = atoi(t[2]);
 
-    if (!t[5] || strlen(t[5]) != 6) // icao must be 6 characters
+    if (!t[5] || strlen(t[5]) < 6 || strlen(t[5]) > 7) // icao must be 6 characters
         goto basestation_invalid;
 
     char *icao = t[5];
+    int non_icao = 0;
+    if (icao[0] == '~') {
+        icao++;
+        non_icao = 1;
+    }
     unsigned char *chars = (unsigned char *) &(mm->addr);
     for (int j = 0; j < 6; j += 2) {
         int high = hexDigitVal(icao[j]);
@@ -3024,6 +3029,10 @@ static int decodeSbsLine(struct client *c, char *line, int remote, int64_t now,
         chars[2 - j / 2] = (high << 4) | low;
     }
 
+    if (non_icao) {
+        mm->addr |= MODES_NON_ICAO_ADDRESS;
+    }
+
     //fprintf(stderr, "%x type %s: ", mm->addr, t[2]);
     //fprintf(stderr, "%x: %d, %0.5f, %0.5f\n", mm->addr, mm->baro_alt, mm->decoded_lat, mm->decoded_lon);
     //field 11, callsign
@@ -3182,10 +3191,6 @@ static void modesSendSBSOutput(struct modesMessage *mm, struct aircraft *a, stru
     struct tm stTime_receive, stTime_now;
     int msgType;
 
-    // For now, suppress non-ICAO addresses
-    if (mm->addr & MODES_NON_ICAO_ADDRESS)
-        return;
-
     p = prepareWrite(writer, 200);
     if (!p)
         return;
@@ -3241,7 +3246,7 @@ static void modesSendSBSOutput(struct modesMessage *mm, struct aircraft *a, stru
     }
 
     // Fields 1 to 6 : SBS message type and ICAO address of the aircraft and some other stuff
-    p += sprintf(p, "MSG,%d,1,1,%06X,1,", msgType, mm->addr);
+    p += sprintf(p, "MSG,%d,1,1,%s%06X,1,", msgType, (a->addr & MODES_NON_ICAO_ADDRESS) ? "~" : "", a->addr & 0xFFFFFF);
 
     // Find current system time
     clock_gettime(CLOCK_REALTIME, &now);

From ae987a904965ffa13e906aa613f3b8a18ce57600 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Tue, 31 Dec 2024 21:55:57 +0000
Subject: [PATCH 86/90] add comment with SBS sample message

---
 net_io.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/net_io.c b/net_io.c
index 8f226430..37b98f7c 100644
--- a/net_io.c
+++ b/net_io.c
@@ -2991,6 +2991,9 @@ static int decodeSbsLine(struct client *c, char *line, int remote, int64_t now,
     mm->signalLevel = 0;
     mm->sbs_in = 1;
 
+    // SBS fields:
+    // MSG,3,1,1,icaoHex,1,messageDate,messageTime,currentDate,currentTime,callsign_8char,altitude_ft,groundspeed_kts,track,lat,lon,vert_rate_fpm,squawk,squawkChangeAlert,squawkEmergencyFlag,squawkIdentFlag,groundFlag_0airborne_-1ground\r\n
+
     // sample message from mlat-client basestation output
     //MSG,3,1,1,4AC8B3,1,2019/12/10,19:10:46.320,2019/12/10,19:10:47.789,,36017,,,51.1001,10.1915,,,,,,
     //
@@ -3199,6 +3202,8 @@ static void modesSendSBSOutput(struct modesMessage *mm, struct aircraft *a, stru
     // SBS BS style output checked against the following reference
     // http://www.homepages.mcb.net/bones/SBS/Article/Barebones42_Socket_Data.htm - seems comprehensive
     //
+    // SBS fields:
+    // MSG,3,1,1,icaoHex,1,messageDate,messageTime,currentDate,currentTime,callsign_8char,altitude_ft,groundspeed_kts,track,lat,lon,vert_rate_fpm,squawk,squawkChangeAlert,squawkEmergencyFlag,squawkIdentFlag,groundFlag_0airborne_-1ground\r\n
 
     if (mm->sbs_in) {
         msgType = mm->sbsMsgType;

From 7211f3f4cd468a6383896b401092194f86879028 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sat, 4 Jan 2025 18:29:26 +0000
Subject: [PATCH 87/90] improve receiver count logic

set to zero when no position has been received for 5 seconds
---
 track.c | 35 +++++++++++++++++++++++++----------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/track.c b/track.c
index 39c694fb..8fe601f7 100644
--- a/track.c
+++ b/track.c
@@ -1058,6 +1058,8 @@ static void setPosition(struct aircraft *a, struct modesMessage *mm, int64_t now
         }
     }
 
+    int64_t elapsed_pos = now - a->seen_pos;
+
     // Update aircraft state
     a->prev_lat = a->lat;
     a->prev_lon = a->lon;
@@ -1072,22 +1074,32 @@ static void setPosition(struct aircraft *a, struct modesMessage *mm, int64_t now
 
     a->pos_surface = trackDataValid(&a->airground_valid) && a->airground == AG_GROUND;
 
-    // due to accept_data / beast_reduce forwarding we can't put receivers
+    // due to the position deduplication logic we won't put receivers
     // into the receiver list which aren't the first ones to send us the position
     if (!Modes.netReceiverId) {
         a->receiverCount = 1;
     } else {
-        a->receiverIdsNext = (a->receiverIdsNext + 1) % RECEIVERIDBUFFER;
-        a->receiverIds[a->receiverIdsNext] = simpleHash(mm->receiverId);
-        if (0 && a->addr == Modes.cpr_focus) {
-            fprintf(stderr, "%u\n", simpleHash(mm->receiverId));
-        }
-
         if (mm->source == SOURCE_MLAT && mm->receiverCountMlat) {
             a->receiverCount = mm->receiverCountMlat;
         } else if (a->position_valid.source < SOURCE_TISB) {
             a->receiverCount = 1;
         } else {
+            a->receiverIdsNext = (a->receiverIdsNext + 1) % RECEIVERIDBUFFER;
+            a->receiverIds[a->receiverIdsNext] = simpleHash(mm->receiverId);
+            int64_t advance = imin(RECEIVERIDBUFFER * 500, elapsed_pos);
+            while (advance > 750) {
+                // ADS-B positions nominally come in every 500 ms receiver id
+                // buffer has 12 positions, if the positions don't come in
+                // often enough we zero them so it's only roughly the receiver
+                // ids for the last 6 seconds
+                advance -= 500;
+                a->receiverIdsNext = (a->receiverIdsNext + 1) % RECEIVERIDBUFFER;
+                a->receiverIds[a->receiverIdsNext] = 0;
+            }
+            if (0 && a->addr == Modes.cpr_focus) {
+                fprintf(stderr, "%u\n", simpleHash(mm->receiverId));
+            }
+
             uint16_t *set1 = a->receiverIds;
             uint16_t set2[RECEIVERIDBUFFER] = { 0 };
             int div = 0;
@@ -3557,10 +3569,13 @@ static const char *source_string(datasource_t source) {
 void updateValidities(struct aircraft *a, int64_t now) {
 
     int64_t elapsed_seen_global = now - a->seenPosGlobal;
+    int64_t elapsed_pos = now - a->seen_pos;
 
-    if (Modes.json_globe_index && elapsed_seen_global < 45 * MINUTES) {
-        a->receiverIdsNext = (a->receiverIdsNext + 1) % RECEIVERIDBUFFER;
-        a->receiverIds[a->receiverIdsNext] = 0;
+    if (elapsed_pos > 5 * SECONDS && a->receiverCount > 0) {
+        a->receiverCount = 0;
+        for (int i = 0; i < RECEIVERIDBUFFER; i++) {
+            a->receiverIds[i] = 0;
+        }
     }
 
     if (a->globe_index >= 0 && now > a->seen_pos + Modes.trackExpireMax) {

From 2f7a72ed238d27e9af676433d2b665964f8b86fd Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sat, 4 Jan 2025 19:00:23 +0000
Subject: [PATCH 88/90] add print to make sure we don't do too many iterations

---
 track.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/track.c b/track.c
index 8fe601f7..74e29ea7 100644
--- a/track.c
+++ b/track.c
@@ -1087,7 +1087,9 @@ static void setPosition(struct aircraft *a, struct modesMessage *mm, int64_t now
             a->receiverIdsNext = (a->receiverIdsNext + 1) % RECEIVERIDBUFFER;
             a->receiverIds[a->receiverIdsNext] = simpleHash(mm->receiverId);
             int64_t advance = imin(RECEIVERIDBUFFER * 500, elapsed_pos);
+            int iterations = 0;
             while (advance > 750) {
+                iterations++;
                 // ADS-B positions nominally come in every 500 ms receiver id
                 // buffer has 12 positions, if the positions don't come in
                 // often enough we zero them so it's only roughly the receiver
@@ -1096,6 +1098,9 @@ static void setPosition(struct aircraft *a, struct modesMessage *mm, int64_t now
                 a->receiverIdsNext = (a->receiverIdsNext + 1) % RECEIVERIDBUFFER;
                 a->receiverIds[a->receiverIdsNext] = 0;
             }
+            if (iterations > 20) {
+                fprintf(stderr, "%06x\n", a->addr);
+            }
             if (0 && a->addr == Modes.cpr_focus) {
                 fprintf(stderr, "%u\n", simpleHash(mm->receiverId));
             }

From 48ef3c0bdacd0eefee25f41ba90fc0b04fcabdc9 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Sat, 4 Jan 2025 19:09:58 +0000
Subject: [PATCH 89/90] github action: fix readsb version to include git commit

---
 .github/workflows/docker.yaml | 2 ++
 Dockerfile                    | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml
index 8a17b8dc..bc3d001d 100644
--- a/.github/workflows/docker.yaml
+++ b/.github/workflows/docker.yaml
@@ -61,3 +61,5 @@ jobs:
         tags: ${{ steps.docker_meta.outputs.tags }}
         labels: ${{ steps.docker_meta.outputs.labels }}
         platforms: linux/amd64,linux/arm64
+        build-args: |
+          BUILDKIT_CONTEXT_KEEP_GIT_DIR=true
diff --git a/Dockerfile b/Dockerfile
index ac2a94cc..72642e38 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,7 +4,7 @@ SHELL ["/bin/bash", "-x", "-o", "pipefail", "-c"]
 RUN --mount=type=bind,source=.,target=/app/git \
     cd /app/git && \
     READSB_BUILD_DIR=$(mktemp -d) && \
-    cp -r /app/git/* $READSB_BUILD_DIR && \
+    cp -aT /app/git $READSB_BUILD_DIR && \
     cd $READSB_BUILD_DIR && \
     [[ $(uname -m) == x86_64 ]] && MARCH=" -march=nehalem" || MARCH="" && \
     make -j$(nproc) RTLSDR=yes OPTIMIZE="-O2 $MARCH" && \

From 3efa2b1ecf3b6b3abf5821d7acaba475a7d97d96 Mon Sep 17 00:00:00 2001
From: Matthias Wirth <matthias.wirth@gmail.com>
Date: Mon, 6 Jan 2025 13:51:38 +0000
Subject: [PATCH 90/90] traces: make sure the air ground transition is well
 recorded

---
 globe_index.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/globe_index.c b/globe_index.c
index 35d04e33..a499589c 100644
--- a/globe_index.c
+++ b/globe_index.c
@@ -2453,6 +2453,7 @@ int traceAdd(struct aircraft *a, struct modesMessage *mm, int64_t now, int stale
 
     // record ground air state changes precisely
     if (on_ground != last->on_ground) {
+        traceUsePosBuffered(a); // save the previous position as well if it's not already saved
         goto save_state;
     }