Apple M1 cmake failure in cloning subrepository. See #2326.

- installer adds python3.12 and use cc, c++ instead of clang,clang++
- Migrate  macosx pkg build to use notarytool (instead of altool) (#2584)
  * Migrate to notarytool
  * Update notarization documentation.
  * Test basic functionality before notarization.

bldnrnmacpkgcmake.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -ex
 
-default_pythons="python3.8 python3.9 python3.10 python3.11"
+default_pythons="python3.9 python3.10 python3.11 python3.12"
 # distribution built with
 # bash bldnrnmacpkgcmake.sh
 # without args, default are the pythons above.
@@ -18,6 +18,10 @@ fi
 # All pythons must have the same macos version and that will become
 # the MACOSX_DEPLOYMENT_TARGET
 
+# On my machine, to build nrn-x.x.x-macosx-10.9-universal2-py-38-39-310-311.pkg
+# I built my own versions of 3.8 in $HOME/soft/python3.8, and
+export PATH=$HOME/soft/python3.8/bin:$PATH
+
 CPU=`uname -m`
 
 universal="yes" # changes to "no" if any python not universal
@@ -52,11 +56,15 @@ if test "$archs" != "universal2" ; then
   universal=no
 fi
 
+# Arrgh. Recent changes to nrn source require at least 10.15!
+# macosver=10.15
+mac_platform=macosx-$macosver-$archs
+
 export MACOSX_DEPLOYMENT_TARGET=$macosver
 echo "MACOSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET"
 
 if test "$NRN_SRC" == "" ; then
-  NRN_SRC=$HOME/neuron/nrn
+  NRN_SRC=`pwd`
 fi
 NRN_BLD=$NRN_SRC/build
 NSRC=$NRN_SRC
@@ -71,7 +79,7 @@ mkdir -p $NRN_BLD
 rm -r -f $NRN_BLD/*
 cd $NRN_BLD
 
-PYVS="py" # will be part of package file name, eg. py-37-38-39-310
+PYVS="py" # will be part of package file name, eg. py-38-39-310-311
 pythons="" # will be arg value of NRN_PYTHON_DYNAMIC
 for i in $args ; do
   PYVER=`$i -c 'from sys import version_info as v ; print (str(v.major) + str(v.minor)); quit()'`
@@ -90,18 +98,18 @@ fi
 # from brew install gedit). User installations are expected to have the
 # former and would only accidentally have the latter.
 
-cmake .. -DCMAKE_INSTALL_PREFIX=$NRN_INSTALL \
+cmake .. -G Ninja -DCMAKE_INSTALL_PREFIX=$NRN_INSTALL \
   -DNRN_ENABLE_MPI_DYNAMIC=ON \
   -DPYTHON_EXECUTABLE=`which python3` -DNRN_ENABLE_PYTHON_DYNAMIC=ON \
   -DNRN_PYTHON_DYNAMIC="$pythons" \
   -DIV_ENABLE_X11_DYNAMIC=ON \
   -DNRN_ENABLE_CORENEURON=OFF \
-  -DNRN_RX3D_OPT_LEVEL=2 \
+  -DNRN_RX3D_OPT_LEVEL=1 \
   $archs_cmake \
   -DCMAKE_PREFIX_PATH=/usr/X11 \
-  -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++
+  -DCMAKE_C_COMPILER=cc -DCMAKE_CXX_COMPILER=c++
 
-make -j install
+ninja install
 
 if test "$universal" = "yes" ; then
   _temp="`lipo -archs $NRN_INSTALL/share/nrn/demo/release/$CPU/special`"
@@ -128,32 +136,29 @@ chk () {
   )
 }
 
+# test basic functionality
+for i in $args ; do
+  chk $i
+done
+
 #/Applications/Packages.app from
 # http://s.sudre.free.fr/Software/Packages/about.html
 # For mac to do a productsign, need my developerID_installer.cer
 # and Neurondev.p12 file. To add to the keychain, double click each
 # of those files. By default, I added my certificates to the login keychain.
-make macpkg # will sign the binaries, construct below
-            # mentioned PACKAGE_FILE_NAME, and request notarization from
-            # Apple. At the end it will print a stapling request that you
-            # should run manually after receiving a "success" email from
-            # Apple.
-
-# test basic functionality
-for i in $args ; do
-  chk $i
-done
+ninja macpkg # will sign the binaries, construct below
+            # mentioned PACKAGE_FILE_NAME, request notarization from
+            # Apple, and staple the package.
 
-# upload package to neuron.yale.edu
-ALPHADIR='[email protected]:/home/htdocs/ftp/neuron/versions/alpha'
+# Copy the package to $HOME/$PACKAGE_FULL_NAME
+# You should then manually upload that to github.
 describe="`sh $NRN_SRC/nrnversion.sh describe`"
 macos=macos${MACOSX_DEPLOYMENT_TARGET}
 PACKAGE_FULL_NAME=nrn-${describe}-${mac_platform}-${PYVS}.pkg
-PACKATE_DOWNLOAD_NAME=$ALPHADIR/$PACKAGE_FULL_NAME
 PACKAGE_FILE_NAME=$NRN_BLD/src/mac/build/NEURON.pkg
+
+cp $PACKAGE_FILE_NAME $HOME/$PACKAGE_FULL_NAME
+
 echo "
-  Until we figure out how to automatically staple the notarization
-  the following two commands must be executed manually.
-  xcrun stapler staple $PACKAGE_FILE_NAME
-  cp $PACKAGE_FILE_NAME $HOME/$PACKAGE_FULL_NAME
+  Manually upload $HOME/$PACKAGE_FULL_NAME to github
 "

cmake/ExternalProjectHelper.cmake

@@ -42,7 +42,8 @@ function(nrn_add_external_project name)
   find_path(
     ${name}_PATH
     NAMES CMakeLists.txt
-    PATHS "${THIRD_PARTY_DIRECTORY}/${name}")
+    PATHS "${THIRD_PARTY_DIRECTORY}/${name}"
+    NO_DEFAULT_PATH)
   if(NOT EXISTS ${${name}_PATH})
     nrn_submodule_file_not_found("${THIRD_PARTY_DIRECTORY}/${name}")
     nrn_initialize_submodule("${THIRD_PARTY_DIRECTORY}/${name}")

docs/install/mac_pkg.md

@@ -17,7 +17,7 @@ On an Apple M1 or x86_64,
 the script, by default, creates, e.g.,
 ```nrn-8.0a-726-gb9a811a32-macosx-11-universal2-py-38-39-310.pkg```
 where the information between nrn and macosx comes from ```git describe```,
-the number after macos refers to the ```MACOSX_DEPLOYMENT_TARGET=11```
+the number after macosx refers to the ```MACOSX_DEPLOYMENT_TARGET=11```
 the next item(s) before py indicate the architectures on which
 the program can run (i.e. ```arm64```, ```x86_64```, or ```universal2```
 for both)
@@ -32,7 +32,7 @@ the same MACOSX_DEPLOYMENT_TARGET. You can check both of these with
 
 A space separated list of python executable arguments can be used in
 place of the internal default lists. ```$NRN_SRC``` is the location of the
-repository, default ```$HOME/neuron/nrn```. The script makes sure ```$NRN_SRC/build```
+repository, default is the current working directory (hopefully you launch at the location of this script, e.g. ```$HOME/neuron/nrn```). The script makes sure ```$NRN_SRC/build```
 exists and uses that folder to configure and build the software. The
 software is installed in ```/Applications/NEURON```.
 
@@ -50,9 +50,12 @@ cmake .. -DCMAKE_INSTALL_PREFIX=$NRN_INSTALL \
   -DCMAKE_PREFIX_PATH=/usr/X11 \
   -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++
 ```
+[^1]
+[^1]: NRN_RX3D_OPT_LEVEL=2 can build VERY slowly (cython translated cpp file can take a half hour or more). So for testing, we generally copy the script to temp.sh and modify to NRN_RX3D_OPT_LEVEL=0
+
 The default variables above will be
 ```
-pythons="python3.8 python3.9 python3.10"
+pythons="python3.9;python3.10;python3.11;python3.12"
 archs_cmake='-DCMAKE_OSX_ARCHITECTURES=arm64;x86_64'
 ```
 
@@ -62,7 +65,11 @@ The MPI library does not have to be universal if configuring with
 ```-DNRN_ENABLE_MPI_DYNAMIC=ON``` as the library is not linked against
 during the build.
 
-```make -j install``` is used to build and install in ```/Applications/NEURON```
+```make -j install``` [^2] is used to build and install in ```/Applications/NEURON```
+
+[^2]: Instead of make, the script uses Ninja. ```ninja install``` and the cmake line begins with ```cmake .. -G Ninja ...```
+
+Runs some basic tests to verify a successful build
 
 ```make macpkg``` ( see ```src/mac/CMakeLists.txt``` ) is used to:
 
@@ -90,42 +97,47 @@ during the build.
 
 - request Apple to notarize NEURON.pkg ```src/macnrn_notarize.sh```
 
+  If notarization succeeds it will finish with a few lines of the form
+  ```
+  Current status: Accepted........Processing complete
+    id: bda82fa9-e0ab-4f86-aad8-3fee1d8c2369
+    status: Accepted
+  ```
+  and staple the package.
+
   If notarizaton fails, it is occasionally due to Apple
   changing the contracts and demanding that 
   "You must first sign the relevant contracts online. (1048)". In that
   case, go to [appstoreconnect.apple.com](https://appstoreconnect.apple.com)
-  to accept the legal docs.  For other notarization failures, one must consult
-  the LogFileURL which can be obtained with
+  to accept the legal docs an try again. For other notarization failures, one must consult the LogFileURL which can be obtained with [^3]
+  [^3]: altool has been replaced by notarytool for purposes of notarization. See
+  https://developer.apple.com/documentation/technotes/tn3147-migrating-to-the-latest-notarization-tool
+  and ```nrn/src/mac/nrn_notarize.sh```
+
+  ```
+  % xcrun notarytool log \
+    --apple-id "$apple_id" \
+    --team-id "$team_id" \
+    --password "$app_specific_password" \
+    "$id"
   ```
-  % xcrun altool  --notarization-info $RequestIdentifier \
-      --username "[email protected]" \
-      --password "`cat ~/.ssh/notarization-password`"
-  No errors getting notarization info.
-
-            Date: 2022-01-02 23:38:12 +0000
-            Hash: 7254157952d4f3573c2804879cf6da8d...
-     LogFileURL: https://osxapps-ssl.itunes.apple.com/itunes-assets...
-     RequestUUID: 152f0f0e-af58-4d22-b291-6a441825dd20
-          Status: invalid
-     Status Code: 2
-  Status Message: Package Invalid
+  where $id was printed by the notarization request. The apple_id, team_id, and
+  app_specfic_password are set by the script to the contents of the files
   ```
-  where RequestIdentifer (the RequestUUID) appears in the email sent
-  back in response to the notarization request.
+  $HOME/.ssh/apple-id
+  $HOME/.ssh/apple-team-id
+  $HOME/.ssh/apple-notarization-password
+  ```
+
 
 The script ends by printing:
 ```
-  Until we figure out how to automatically staple the notarization
-  the following two commands must be executed manually.
-  xcrun stapler staple $PACKAGE_FILE_NAME
   cp $PACKAGE_FILE_NAME $HOME/$PACKAGE_FULL_NAME
+  Manually upload $HOME/nrn-9.0a-88-gc6d8b9af6-macosx-10.15-universal2-py-39-310-311.pkg to github
 ```
-where the variables are filled out and can be copy/pasted to your
-terminal after Apple sends an email declaring that notarization was successful.
 
-The email from Apple usually takes just a few minutes but can be hours.
 I've been uploading the ```$PACKAGE_FULL_NAME``` as an artifact for a
-Release version from https://github.com/neuronsimulatior/nrn by choosing
+Release version from https://github.com/neuronsimulator/nrn by choosing
 Releases, choosing the proper Release tag (e.g. Release 8.0a), Edit release,
 and clicking in the "Attach binaries ..." near the bottom of the page.
 

src/mac/nrn_notarize.sh

@@ -1,64 +1,72 @@
 #!/usr/bin/env bash
 set -e
-# App specific password used to request notarization
-password_path="$HOME/.ssh/notarization-password"
+
+# See https://developer.apple.com/documentation/technotes/tn3147-migrating-to-the-latest-notarization-tool
+
+# App specific password and credentials used to request notarization
+password_path="$HOME/.ssh/apple-notarization-password"
+apple_id_path="$HOME/.ssh/apple-id"
+team_id_path="$HOME/.ssh/apple-team-id"
+
 if test -f "$password_path" ; then
   app_specific_password=`cat "$password_path"`
 else
   echo "\"$password_path\" does not exist"
   exit 1
 fi
+if test -f "$apple_id_path" ; then
+  apple_id=`cat "$apple_id_path"`
+else
+  echo "\"$apple_id_path\" does not exist"
+  exit 1
+fi
+if test -f "$team_id_path" ; then
+  team_id=`cat "$team_id_path"`
+else
+  echo "\"$team_id_path\" does not exist"
+  exit 1
+fi
 
 pkg="$1"
 pkgname="$2"
 
 echo "Notarize request"
-xcrun altool --notarize-app \
-  --primary-bundle-id "edu.yale.neuron.pkg.$pkgname" \
-  --username "[email protected]" \
+xcrun notarytool submit \
+  --wait \
+  --apple-id "$apple_id" \
+  --team-id "$team_id" \
   --password "$app_specific_password" \
-  --file "$pkg"
+  "$pkg"
 
-# you should get
-# 2021-01-31 17:43:58.537 altool[3021:3231297] CFURLRequestSetHTTPCookieStorageAcceptPolicy_block_invoke: no longer implemented and should not be called
-# No errors uploading '/Users/michaelhines/neuron/notarize/build/src/mac/build/NEURON.pkg'.
-# RequestUUID = dc9dd4dd-a942-475a-ab59-4996f938d606
-
-# and eventually get an email in just a few minutes saying that
-# Your Mac software has been notarized.
+# this should end with something like
+#Processing complete
+#  id: 87c2e5e9-f37f-4a95-9941-8c2205fc90bd
+#  status: Accepted
 
 # At this point the software can be installed. However it is recommended
-# xcrun stapler staple "$pkg"
+# to staple the pkg file
 # so that Gatekeeper will be able to find the whitelist in the file itself
 # without the need to perform an online check.
-# However it is unclear how long the wait will be if you do a wait loop over
-# xcrun altool --notarization-info $RequestUUID \
-#  --username "[email protected]" \
-#  --password "$app_specific_password"
-# until
-#   Status: in progress
-# changes to
-#   Status: success
-# at which point it is ok to run the stapler.
-
-# For now, echo a suggestion about waiting for an email and what stapler
-# command to execute manually.
+xcrun stapler staple "$pkg"
 
-echo "
-After getting an email from Apple stating that:
-  Your Mac software has been notarized.
-manually execute
+# Lastly, copy the pkg file to its proper name in $HOME
 
- xcrun stapler staple \"$pkg\"
-
-so that Gatekeeper will be able to find the whitelist in the file itself
-without the need to perform an online check.
-"
+# I've read that it is a good idea to check for warnings by
+# fetching the notary log (using the id generated by the above submit)
+# And if there are errors, then you certainly need to fetch the log,
+# address the issues it shows and try again.
+if false ; then
+  xcrun notarytool log \
+    --apple-id "$apple_id" \
+    --team-id "$team_id" \
+    --password "$app_specific_password" \
+    "$id"
+fi
 
-# If the notarization fails. E.g. the message from Apple is
-#    "Your Mac software was not notarized"
-# then review the notarization LogFileURL by obtaining the log url with
-# xcrun altool --notarization-info $RequestUUID \
-#  --username "[email protected]" \
-#  --password "$app_specific_password"
-# and address the issues it shows and try again.
+# To get a history of notarizations...
+if false ; then
+  xcrun notarytool history \
+    --apple-id "$apple_id" \
+    --team-id "$team_id" \
+    --password "$app_specific_password"
+fi