Pending latest changes from Widehopf: Dev

.github/workflows/docker.yaml

@@ -2,15 +2,16 @@ name: docker build
 
 on:
   schedule:
-  - cron: "0 10 * * *"
+    - cron: "0 5 * * 1"
   push:
     branches:
-    - "**"
+      - "dev"
+      - "sid"
     tags:
-    - "v*.*.*"
+      - "v*.*.*"
   pull_request:
     branches:
-    - "dev"
+      - "dev"
 
 jobs:
   docker:
@@ -21,8 +22,8 @@ jobs:
     - name: Set variables useful for later
       id: useful_vars
       run: |-
-        echo "::set-output name=timestamp::$(date +%s)"
-        echo "::set-output name=short_sha::${GITHUB_SHA::8}"
+        echo "name=timestamp::$(date +%s)" >> $GITHUB_OUTPUT
+        echo "name=short_sha::${GITHUB_SHA::8}" >> $GITHUB_OUTPUT
     - name: Checkout
       uses: actions/checkout@v3
     - name: Docker meta
@@ -52,13 +53,6 @@ jobs:
         registry: ghcr.io
         username: ${{ github.repository_owner }}
         password: ${{ secrets.GITHUB_TOKEN }}
-    - name: Cache Docker layers
-      uses: actions/cache@v3
-      with:
-        path: /tmp/.buildx-cache
-        key: ${{ runner.os }}-buildx-${{ github.sha }}
-        restore-keys: |
-          ${{ runner.os }}-buildx-
     - name: Build and push
       uses: docker/build-push-action@v4
       with:
@@ -67,5 +61,5 @@ jobs:
         tags: ${{ steps.docker_meta.outputs.tags }}
         labels: ${{ steps.docker_meta.outputs.labels }}
         platforms: linux/amd64,linux/arm64
-        cache-from: type=local,src=/tmp/.buildx-cache
-        cache-to: type=local,dest=/tmp/.buildx-cache,mode=max
+        build-args: |
+          BUILDKIT_CONTEXT_KEEP_GIT_DIR=true

Dockerfile

@@ -1,30 +1,18 @@
-FROM debian:bookworm-20240311 AS builder
-RUN apt-get update && \
-    apt-get install -y git wget pkg-config autoconf gcc make libusb-1.0-0-dev librtlsdr-dev librtlsdr0 libncurses-dev zlib1g-dev zlib1g libzstd-dev libzstd1
+FROM ghcr.io/wiedehopf/readsb-builder:latest AS builder
 
-# install jemalloc
-RUN JEMALLOC_BDIR=$(mktemp -d) && \
-    git clone --depth 1 https://github.com/jemalloc/jemalloc $JEMALLOC_BDIR && \
-    cd $JEMALLOC_BDIR && \
-    ./autogen.sh && \
-    ./configure --with-lg-page=14 && \
-    make -j$(nproc) && \
-    make install && \
-    rm -rf $JEMALLOC_BDIR
-
-# install readsb
-RUN mkdir -p /app/git
-COPY . /app/git
-RUN cd /app/git && \
+SHELL ["/bin/bash", "-x", "-o", "pipefail", "-c"]
+RUN --mount=type=bind,source=.,target=/app/git \
+    cd /app/git && \
     READSB_BUILD_DIR=$(mktemp -d) && \
-    cp -r /app/git/* $READSB_BUILD_DIR && \
+    cp -aT /app/git $READSB_BUILD_DIR && \
     cd $READSB_BUILD_DIR && \
-    make -j$(nproc) RTLSDR=yes OPTIMIZE="-O2" && \
+    [[ $(uname -m) == x86_64 ]] && MARCH=" -march=nehalem" || MARCH="" && \
+    make -j$(nproc) RTLSDR=yes OPTIMIZE="-O2 $MARCH" && \
     mv readsb /usr/local/bin && \
     mv viewadsb /usr/local/bin && \
     chmod +x /usr/local/bin/viewadsb /usr/local/bin/readsb && \
     make clean && \
-    make -j$(nproc) PRINT_UUIDS=yes TRACKS_UUID=yes OPTIMIZE="-O2" && \
+    make -j$(nproc) PRINT_UUIDS=yes TRACKS_UUID=yes OPTIMIZE="-O2 $MARCH" && \
     mv readsb /usr/local/bin/readsb-uuid && \
     mv viewadsb /usr/local/bin/viewadsb-uuid && \
     chmod +x /usr/local/bin/viewadsb-uuid && \
@@ -35,18 +23,21 @@ RUN cd /app/git && \
     true
 
 FROM debian:bookworm-slim
-RUN apt-get update && \
+RUN \
+    --mount=type=bind,from=builder,source=/,target=/builder/ \
+    apt-get update && \
     apt-get -y install --no-install-recommends \
-    wget \
     librtlsdr0 libncurses6 zlib1g libzstd1 && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
-    mkdir -p /run/readsb
+    mkdir -p /run/readsb && \
+    cp /builder/usr/local/bin/readsb* /usr/local/bin/ && \
+    cp /builder/usr/local/bin/viewadsb* /usr/local/bin/ && \
+    mkdir -p  /usr/local/share/tar1090 && \
+    cp /builder/usr/local/share/tar1090/aircraft.csv.gz /usr/local/share/tar1090/aircraft.csv.gz && \
+    cp /builder/usr/local/lib/libjemalloc.so.2 /usr/local/lib/libjemalloc.so.2 && \
+    true
 
 ENV LD_PRELOAD=/usr/local/lib/libjemalloc.so.2
-COPY --from=builder /usr/local/bin/readsb* /usr/local/bin/
-COPY --from=builder /usr/local/bin/viewadsb* /usr/local/bin/
-COPY --from=builder /usr/local/share/tar1090/aircraft.csv.gz /usr/local/share/tar1090/aircraft.csv.gz
-COPY --from=builder /usr/local/lib/libjemalloc.so.2 /usr/local/lib/libjemalloc.so.2
 
 ENTRYPOINT ["/usr/local/bin/readsb"]

README-json.md

@@ -24,7 +24,6 @@ The keys are:
 
  * version: the version of readsb in use
  * refresh: how often aircraft.json is updated (for the file version), in milliseconds. the webmap uses this to control its refresh interval.
- * history: the current number of valid history files (see below)
  * lat: the latitude of the receiver in decimal degrees. Optional, may not be present.
  * lon: the longitude of the receiver in decimal degrees. Optional, may not be present.
 
@@ -202,18 +201,6 @@ The keys are:
   ```
   * Status code 200 during normal operation
 
-## history_0.json, history_1.json, ..., history_119.json
-
-These files are historical copies of aircraft.json at (by default) 30 second intervals. They follow exactly the
-same format as aircraft.json. To know how many are valid, see receiver.json ("history" value). They are written in
-a cycle, with history_0 being overwritten after history_119 is generated, so history_0.json is not necessarily the
-oldest history entry. To load history, you should:
-
- * read "history" from receiver.json.
- * load that many history_N.json files
- * sort the resulting files by their "now" values
- * process the files in order
-
 ## trace jsons
 
  * overall structure

README.md

@@ -20,7 +20,7 @@ Or build the package yourself:
 sudo apt update
 sudo apt install --no-install-recommends --no-install-suggests -y \
     git build-essential debhelper libusb-1.0-0-dev \
-    librtlsdr-dev librtlsdr0 pkg-config \
+    librtlsdr-dev librtlsdr0 pkg-config fakeroot \
     libncurses-dev zlib1g-dev zlib1g libzstd-dev libzstd1
 git clone --depth 20 https://github.com/wiedehopf/readsb.git
 cd readsb
@@ -56,6 +56,14 @@ Selectively forwards beast messages if the received data hasn't been forwarded i
 Data not related to the physical aircraft state are only forwarded every 500 ms (4 * `--net-beast-reduce-interval`).The messages of
 this output are normal beast messages and compatible with every program able to receive beast messages.
 
+This is used by some aggregators to aggregate ADS-B data, an example net connector would be:
+```
+--net-connector=feed.airplanes.live,30004,beast_reduce_plus_out,uuid=0033062d-e17e-4389-91a9-79ebb967fb4c
+```
+The uuid is optional, if none is given, the uuid from --uuid-file is used, if that isn't present no uuid is sent.
+The beast_reduce_out net-connector will never send an uuid.
+The aggregator enables --net-receiver-id and --net-ingest on their readsb server, it's made to work with beast_reduce_plus_out.
+
 ### Debian package
 
 - Build package with no additional receiver library dependencies: `dpkg-buildpackage -b`.
@@ -91,6 +99,23 @@ The difference of using -Ofast or -O3 over the default of -O2 is likely very min
 
 If required, edit `/etc/default/readsb` to set the service options, device type, network ports etc.
 
+## Autogain
+
+For rtl-sdr devices a software gain algorithm is the default, it's optimized for ADS-B.
+On the command line it's activated using `--gain=auto` an is silent by default.
+`--gain=auto-verbose` can be used to enable log messages for gain changes.
+To tweak the internals, more parameters can be passed:
+```
+--gain=auto-verbose,<lowestGain>,<noiseLowThreshold>,<noiseHighThreshold>,<loudThreshold>
+```
+The defaults are:
+```
+--gain=auto-verbose,0,25,31,243
+```
+The thresholds are numbers 0 to 256, tweaking them requires some understanding of how it works.
+One option would be to change the noise thresholds up or down and then observe the log.
+There should be no need to tweak these parameters.
+
 ## rtl-sdr bias tee
 
 Use this utility independent of readsb:
@@ -104,9 +129,9 @@ For that purpose it's used in conjunction with tar1090 with some extra options t
 Websites using this software:
 
 - https://adsb.lol/
-- https://globe.adsbexchange.com/
 - https://globe.airplanes.live/
 - https://globe.adsb.fi/
+- https://globe.adsbexchange.com/
 
 Projects that use or have used data generated by this software: