Merge remote-tracking branch 'upstream/master' into master-3.3.x

RELEASE-NOTES

@@ -1,3 +1,12 @@
+# Release 3.3.8 - (July 11, 2019)
+
+ * fix some bugs with recent bump fee (RBF) improvements (#5483, #5502)
+ * fix #5491: watch-only wallets could not bump fee in some cases
+ * appimage: URLs could not be opened on some desktop environments (#5425)
+ * faster tx signing for segwit inputs for really large txns (#5494)
+ * A few other minor bugfixes and usability improvements.
+
+
 # Release 3.3.7 - (July 3, 2019)
 
  * The AppImage Linux x86_64 binary and the Windows setup.exe

contrib/build-linux/appimage/Dockerfile

@@ -21,7 +21,7 @@ RUN apt-get update -q && \
         libudev-dev=204-5ubuntu20.31 \
         gettext=0.18.3.1-1ubuntu3.1 \
         libzbar0=0.10+doc-9build1  \
-        faketime=0.9.5-2 \
+        libdbus-1-3=1.6.18-0ubuntu4.5 \
         && \
     rm -rf /var/lib/apt/lists/* && \
     apt-get autoremove -y && \

contrib/build-linux/appimage/README.md

@@ -25,7 +25,7 @@ see [issue #5159](https://github.com/spesmilo/electrum/issues/5159).
 2. Build image
 
     ```
-    $ sudo docker build --no-cache -t electrum-appimage-builder-img contrib/build-linux/appimage
+    $ sudo docker build -t electrum-appimage-builder-img contrib/build-linux/appimage
     ```
 
 3. Build binary

contrib/build-linux/appimage/build.sh

@@ -132,10 +132,10 @@ info "preparing electrum-locale."
 
 info "installing electrum and its dependencies."
 mkdir -p "$CACHEDIR/pip_cache"
-"$python" -m pip install --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements.txt"
-"$python" -m pip install --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-binaries.txt"
-"$python" -m pip install --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-hw.txt"
-"$python" -m pip install --cache-dir "$CACHEDIR/pip_cache" "$PROJECT_ROOT"
+"$python" -m pip install --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements.txt"
+"$python" -m pip install --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-binaries.txt"
+"$python" -m pip install --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-hw.txt"
+"$python" -m pip install --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" "$PROJECT_ROOT"
 
 
 info "copying zbar"
@@ -157,12 +157,8 @@ info "finalizing AppDir."
 
     cd "$APPDIR"
     # copy system dependencies
-    # note: temporarily move PyQt5 out of the way so
-    # we don't try to bundle its system dependencies.
-    mv "$APPDIR/usr/lib/python3.6/site-packages/PyQt5" "$BUILDDIR"
     copy_deps; copy_deps; copy_deps
     move_lib
-    mv "$BUILDDIR/PyQt5" "$APPDIR/usr/lib/python3.6/site-packages"
 
     # apply global appimage blacklist to exclude stuff
     # move usr/include out of the way to preserve usr/include/python3.6m.
@@ -171,10 +167,12 @@ info "finalizing AppDir."
     mv usr/include.tmp usr/include
 ) || fail "Could not finalize AppDir"
 
-# copy libusb here because it is on the AppImage excludelist and it can cause problems if we use system libusb
-info "Copying libusb"
-cp -f /usr/lib/x86_64-linux-gnu/libusb-1.0.so "$APPDIR/usr/lib/libusb-1.0.so" || fail "Could not copy libusb"
-
+# We copy some libraries here that are on the AppImage excludelist
+info "Copying additional libraries"
+(
+    # On some systems it can cause problems to use the system libusb
+    cp -f /usr/lib/x86_64-linux-gnu/libusb-1.0.so "$APPDIR/usr/lib/libusb-1.0.so" || fail "Could not copy libusb"
+)
 
 info "stripping binaries from debug symbols."
 # "-R .note.gnu.build-id" also strips the build id
@@ -231,8 +229,11 @@ find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} +
 info "creating the AppImage."
 (
     cd "$BUILDDIR"
-    chmod +x "$CACHEDIR/appimagetool"
-    "$CACHEDIR/appimagetool" --appimage-extract
+    cp "$CACHEDIR/appimagetool" "$CACHEDIR/appimagetool_copy"
+    # zero out "appimage" magic bytes, as on some systems they confuse the linker
+    sed -i 's|AI\x02|\x00\x00\x00|' "$CACHEDIR/appimagetool_copy"
+    chmod +x "$CACHEDIR/appimagetool_copy"
+    "$CACHEDIR/appimagetool_copy" --appimage-extract
     # We build a small wrapper for mksquashfs that removes the -mkfs-fixed-time option
     # that mksquashfs from squashfskit does not support. It is not needed for squashfskit.
     cat > ./squashfs-root/usr/lib/appimagekit/mksquashfs << EOF

contrib/build-wine/README.md

@@ -32,7 +32,7 @@ folder.
 
     ```
     $ FRESH_CLONE=contrib/build-wine/fresh_clone && \
-        rm -rf $FRESH_CLONE && \
+        sudo rm -rf $FRESH_CLONE && \
         mkdir -p $FRESH_CLONE && \
         cd $FRESH_CLONE  && \
         git clone https://github.com/bitzeny-electrum/electrum-zny.git && \

contrib/build-wine/build-electrum-git.sh

@@ -4,6 +4,7 @@ NAME_ROOT=electrum-zny
 
 # These settings probably don't need any change
 export WINEPREFIX=/opt/wine64
+export WINEDEBUG=-all
 export PYTHONDONTWRITEBYTECODE=1
 export PYTHONHASHSEED=22
 
@@ -41,14 +42,14 @@ find -exec touch -d '2000-11-11T11:11:11+00:00' {} +
 popd
 
 # Install frozen dependencies
-$PYTHON -m pip install -r "$CONTRIB"/deterministic-build/requirements.txt
+$PYTHON -m pip install --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements.txt
 
-$PYTHON -m pip install -r "$CONTRIB"/deterministic-build/requirements-hw.txt
+$PYTHON -m pip install --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-hw.txt
 
 pushd $WINEPREFIX/drive_c/electrum-zny
 # see https://github.com/pypa/pip/issues/2195 -- pip makes a copy of the entire directory
 info "Pip installing Electrum. This might take a long time if the project folder is large."
-$PYTHON -m pip install .
+$PYTHON -m pip install --no-warn-script-location .
 popd
 
 
@@ -71,4 +72,46 @@ cd dist
 mv electrum-zny-setup.exe $NAME_ROOT-$VERSION-setup.exe
 cd ..
 
+info "Padding binaries to 8-byte boundaries, and fixing COFF image checksum in PE header"
+# note: 8-byte boundary padding is what osslsigncode uses:
+#       https://github.com/mtrojnar/osslsigncode/blob/6c8ec4427a0f27c145973450def818e35d4436f6/osslsigncode.c#L3047
+(
+    cd dist
+    for binary_file in ./*.exe; do
+        info ">> fixing $binary_file..."
+        # code based on https://github.com/erocarrera/pefile/blob/bbf28920a71248ed5c656c81e119779c131d9bd4/pefile.py#L5877
+        python3 <<EOF
+pe_file = "$binary_file"
+with open(pe_file, "rb") as f:
+    binary = bytearray(f.read())
+pe_offset = int.from_bytes(binary[0x3c:0x3c+4], byteorder="little")
+checksum_offset = pe_offset + 88
+checksum = 0
+
+# Pad data to 8-byte boundary.
+remainder = len(binary) % 8
+binary += bytes(8 - remainder)
+
+for i in range(len(binary) // 4):
+    if i == checksum_offset // 4:  # Skip the checksum field
+        continue
+    dword = int.from_bytes(binary[i*4:i*4+4], byteorder="little")
+    checksum = (checksum & 0xffffffff) + dword + (checksum >> 32)
+    if checksum > 2 ** 32:
+        checksum = (checksum & 0xffffffff) + (checksum >> 32)
+
+checksum = (checksum & 0xffff) + (checksum >> 16)
+checksum = (checksum) + (checksum >> 16)
+checksum = checksum & 0xffff
+checksum += len(binary)
+
+# Set the checksum
+binary[checksum_offset : checksum_offset + 4] = int.to_bytes(checksum, byteorder="little", length=4)
+
+with open(pe_file, "wb") as f:
+    f.write(binary)
+EOF
+    done
+)
+
 sha256sum dist/electrum*.exe

contrib/build-wine/prepare-wine.sh

@@ -25,7 +25,7 @@ PYTHON_VERSION=3.6.8
 
 ## These settings probably don't need change
 export WINEPREFIX=/opt/wine64
-#export WINEARCH='win32'
+export WINEDEBUG=-all
 
 PYTHON_FOLDER="python3"
 PYHOME="c:/$PYTHON_FOLDER"
@@ -62,7 +62,7 @@ done
 
 info "Installing dependencies specific to binaries."
 # note that this also installs pinned versions of both pip and setuptools
-$PYTHON -m pip install -r "$CONTRIB"/deterministic-build/requirements-binaries.txt
+$PYTHON -m pip install --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-binaries.txt
 
 info "Installing ZBar."
 download_if_not_exist "$CACHEDIR/$ZBAR_FILENAME" "$ZBAR_URL"
@@ -117,6 +117,6 @@ info "Building PyInstaller."
     [[ -e PyInstaller/bootloader/Windows-32bit/runw.exe ]] || fail "Could not find runw.exe in target dir!"
 ) || fail "PyInstaller build failed"
 info "Installing PyInstaller."
-$PYTHON -m pip install ./pyinstaller
+$PYTHON -m pip install --no-warn-script-location ./pyinstaller
 
 info "Wine is configured."

contrib/build-wine/unsign.sh

@@ -24,28 +24,8 @@ for mine in $(ls dist/*.exe); do
     echo "Downloading https://zny.electrum-jp.net/$version/$f"
     wget -q https://zny.electrum-jp.net/$version/$f -O signed/$f
     out="signed/stripped/$f"
-    size=$( wc -c < $mine )
-    # Step 1: Remove PE signature from signed binary
+    # Remove PE signature from signed binary
     osslsigncode remove-signature -in signed/$f -out $out > /dev/null 2>&1
-    # Step 2: Remove checksum and padding from signed binary
-    python3 <<EOF
-pe_file = "$out"
-size= $size
-with open(pe_file, "rb") as f:
-    binary = bytearray(f.read())
-pe_offset = int.from_bytes(binary[0x3c:0x3c+4], byteorder="little")
-checksum_offset = pe_offset + 88
-for b in range(4):
-    binary[checksum_offset + b] = 0
-l = len(binary)
-n = l - size
-if n > 0:
-   if binary[-n:] != bytearray(n):
-       print('expecting failure for', str(pe_file))
-   binary = binary[:size]
-with open(pe_file, "wb") as f:
-    f.write(binary)
-EOF
     chmod +x $out
     if cmp -s $out $mine; then
 	echo "Success: $f"

contrib/make_download

@@ -2,8 +2,15 @@
 import re
 import os
 import sys
+import importlib
 
-from electrum.version import ELECTRUM_VERSION, APK_VERSION
+# load version.py; needlessly complicated alternative to "imp.load_source":
+version_spec = importlib.util.spec_from_file_location('version', 'electrum/version.py')
+version_module = importlib.util.module_from_spec(version_spec)
+version_spec.loader.exec_module(version_module)
+
+ELECTRUM_VERSION = version_module.ELECTRUM_VERSION
+APK_VERSION = version_module.APK_VERSION
 print("version", ELECTRUM_VERSION)
 
 dirname = sys.argv[1]

contrib/osx/README.md

@@ -13,6 +13,9 @@ This needs to be done on a system running macOS or OS X. We use El Capitan (10.1
 on High Sierra (or later)
 makes the binaries [incompatible with older versions](https://github.com/pyinstaller/pyinstaller/issues/1191).
 
+Another factor for the minimum supported macOS version is the
+[bundled Qt version](https://github.com/spesmilo/electrum/issues/3685).
+
 Before starting, make sure that the Xcode command line tools are installed (e.g. you have `git`).
 
 #### 1.1a Get Xcode

contrib/upload

@@ -2,16 +2,17 @@
 
 set -e
 
+host=$1
 version=`git describe --tags`
 echo $version
 
 here=$(dirname "$0")
 cd $here/../dist
 
-sftp -oBatchMode=no -b - thomasv@download.electrum.org << !
+sftp -oBatchMode=no -b - thomasv@$host << !
    cd electrum-downloads
    mkdir $version
    cd $version
    mput *
    bye
-!
+!

electrum/base_crash_reporter.py

@@ -131,3 +131,20 @@ def get_user_description(self):
 
     def get_wallet_type(self):
         raise NotImplementedError
+
+
+def trigger_crash():
+    # note: do not change the type of the exception, the message,
+    # or the name of this method. All reports generated through this
+    # method will be grouped together by the crash reporter, and thus
+    # don't spam the issue tracker.
+
+    class TestingException(Exception):
+        pass
+
+    def crash_test():
+        raise TestingException("triggered crash for testing purposes")
+
+    import threading
+    t = threading.Thread(target=crash_test)
+    t.start()

electrum/coinchooser.py

@@ -151,7 +151,7 @@ def make_Bucket(desc, coins):
     def penalty_func(self, base_tx, *, tx_from_buckets) -> Callable[[List[Bucket]], ScoredCandidate]:
         raise NotImplementedError
 
-    def _change_amounts(self, tx, count, fee_estimator_numchange):
+    def _change_amounts(self, tx, count, fee_estimator_numchange) -> List[int]:
         # Break change up if bigger than max_change
         output_amounts = [o.value for o in tx.outputs()]
         # Don't split change of less than 0.02 BTC
@@ -197,7 +197,7 @@ def trailing_zeroes(val):
         # no more than 10**max_dp_to_round_for_privacy
         # e.g. a max of 2 decimal places means losing 100 satoshis to fees
         max_dp_to_round_for_privacy = 2 if self.enable_output_value_rounding else 0
-        N = pow(10, min(max_dp_to_round_for_privacy, zeroes[0]))
+        N = int(pow(10, min(max_dp_to_round_for_privacy, zeroes[0])))
         amount = (remaining // N) * N
         amounts.append(amount)
 
@@ -209,6 +209,7 @@ def _change_outputs(self, tx, change_addrs, fee_estimator_numchange, dust_thresh
         amounts = self._change_amounts(tx, len(change_addrs), fee_estimator_numchange)
         assert min(amounts) >= 0
         assert len(change_addrs) >= len(amounts)
+        assert all([isinstance(amt, int) for amt in amounts])
         # If change is above dust threshold after accounting for the
         # size of the change output, add it to the transaction.
         amounts = [amount for amount in amounts if amount >= dust_threshold]

electrum/constants.py

@@ -39,6 +39,10 @@ def read_json(filename, default):
     return r
 
 
+GIT_REPO_URL = "https://github.com/bitzeny-electrum/electrum-zny"
+GIT_REPO_ISSUES_URL = "https://github.com/bitzeny-electrum/electrum-zny/issues"
+
+
 class AbstractNet:
 
     @classmethod

electrum/exchange_rate.py

@@ -423,7 +423,8 @@ def set_exchange(self, name):
         self.logger.info(f"using exchange {name}")
         if self.config_exchange() != name:
             self.config.set_key('use_exchange', name, True)
-        self.exchange = class_(self.on_quotes, self.on_history)
+        assert issubclass(class_, ExchangeBase), f"unexpected type {class_} for {name}"
+        self.exchange = class_(self.on_quotes, self.on_history)  # type: ExchangeBase
         # A new exchange means new fx quotes, initially empty.  Force
         # a quote refresh
         self.trigger_update()
@@ -476,6 +477,8 @@ def history_rate(self, d_t):
         # Use spot quotes in that case
         if rate == 'NaN' and (datetime.today().date() - d_t.date()).days <= 2:
             rate = self.exchange.quotes.get(self.ccy, 'NaN')
+            if rate is None:
+                rate = 'NaN'
             self.history_used_spot = True
         return Decimal(rate)