Introduced UserInteraction utility class (for html mapper internal pu…

…rposes)

webfx-kit/webfx-kit-gwt/src/main/java/dev/webfx/kit/launcher/spi/impl/gwt/GwtWebFxKitLauncherProvider.java

@@ -7,6 +7,7 @@
 import dev.webfx.kit.mapper.WebFxKitMapper;
 import dev.webfx.kit.mapper.peers.javafxgraphics.gwt.html.CanvasElementHelper;
 import dev.webfx.kit.mapper.peers.javafxgraphics.gwt.html.HtmlNodePeer;
+import dev.webfx.kit.mapper.peers.javafxgraphics.gwt.html.UserInteraction;
 import dev.webfx.kit.mapper.peers.javafxgraphics.gwt.util.DragboardDataTransferHolder;
 import dev.webfx.kit.mapper.peers.javafxgraphics.gwt.util.HtmlFonts;
 import dev.webfx.kit.mapper.peers.javafxgraphics.gwt.util.HtmlUtil;
@@ -42,6 +43,12 @@
  */
 public final class GwtWebFxKitLauncherProvider extends WebFxKitLauncherProviderBase {
 
+    private static final boolean IS_SAFARI;
+    static {
+        String userAgent = DomGlobal.navigator.userAgent.toLowerCase();
+        IS_SAFARI = userAgent.contains("safari") && !userAgent.contains("chrome") && !userAgent.contains("android");
+    }
+
     private Application application;
     private HostServices hostServices;
 
@@ -52,7 +59,20 @@ public GwtWebFxKitLauncherProvider() {
     @Override
     public HostServices getHostServices() {
         if (hostServices == null)
-            hostServices = uri -> DomGlobal.window.open(uri);
+            hostServices = uri -> {
+                // Note: Safari is blocking (on macOS) or ignoring (on iOS) window.open() when not called during a user
+                // interaction. If we are in that case, it's better to postpone the window opening to the next user
+                // interaction (which we hope will happen soon, such as a key or mouse release).
+                if (IS_SAFARI && !UserInteraction.isUserInteracting()) {
+                    UserInteraction.runOnNextUserInteraction(() -> {
+                        DomGlobal.window.open(uri, "_blank");
+                    }, true);
+                } else {
+                    // For other browsers, or with Safari but during a user interaction (ex: mouse click), it's ok to
+                    // open the browser window straightaway.
+                    DomGlobal.window.open(uri, "_blank");
+                }
+            };
         return hostServices;
     }
 

webfx-kit/webfx-kit-javafxgraphics-peers-gwt/src/main/java/dev/webfx/kit/mapper/peers/javafxgraphics/gwt/html/HtmlScenePeer.java

@@ -104,17 +104,20 @@ private void registerMouseListener(String type) {
     private void passHtmlMouseEventOnToFx(MouseEvent e, String type) {
         javafx.scene.input.MouseEvent fxMouseEvent = FxEvents.toFxMouseEvent(e, type);
         if (fxMouseEvent != null) {
+            boolean isMousePressed = fxMouseEvent.getEventType() == javafx.scene.input.MouseEvent.MOUSE_PRESSED;
+            boolean isMouseReleased = fxMouseEvent.getEventType() == javafx.scene.input.MouseEvent.MOUSE_RELEASED;
+            UserInteraction.setUserInteracting(isMousePressed || isMouseReleased);
             // We now need to call Scene.impl_processMouseEvent() to pass the event to the JavaFX stack
             Scene scene = getScene();
             // Also fixing a problem: mouse released and mouse pressed are sent very closely on mobiles and might be
             // treated in the same animation frame, which prevents the button pressed state (ex: a background bound to
             // the button pressedProperty) to appear before the action (which might be time-consuming) is fired, so the
             // user doesn't know if the button has been successfully pressed or not during the action execution.
-            if (fxMouseEvent.getEventType() == javafx.scene.input.MouseEvent.MOUSE_RELEASED && !atLeastOneAnimationFrameOccurredSinceLastMousePressed)
+            if (isMouseReleased && !atLeastOneAnimationFrameOccurredSinceLastMousePressed)
                 UiScheduler.scheduleInAnimationFrame(() -> scene.impl_processMouseEvent(fxMouseEvent), 1);
             else {
                 scene.impl_processMouseEvent(fxMouseEvent);
-                if (fxMouseEvent.getEventType() == javafx.scene.input.MouseEvent.MOUSE_PRESSED) {
+                if (isMousePressed) {
                     atLeastOneAnimationFrameOccurredSinceLastMousePressed = false;
                     UiScheduler.scheduleInAnimationFrame(() -> atLeastOneAnimationFrameOccurredSinceLastMousePressed = true, 1);
                     /* Try to uncomment this code if the focus hasn't been updated after clicking on a Node (not necessary so far)
@@ -129,6 +132,7 @@ private void passHtmlMouseEventOnToFx(MouseEvent e, String type) {
                     }*/
                 }
             }
+            UserInteraction.setUserInteracting(false);
             // Stopping propagation if the event has been consumed by JavaFX
             if (fxMouseEvent.isConsumed())
                 e.stopPropagation();
@@ -385,7 +389,9 @@ private static void registerKeyboardListener(String type, Scene scene) {
                 focusOwner = scene.getFocusOwner();
             }
             javafx.event.EventTarget fxTarget = focusOwner != null ? focusOwner : scene;
+            UserInteraction.setUserInteracting(true);
             boolean fxConsumed = passHtmlKeyEventOnToFx((KeyboardEvent) e, type, fxTarget);
+            UserInteraction.setUserInteracting(false);
             if (fxConsumed) {
                 e.stopPropagation();
                 e.preventDefault();

webfx-kit/webfx-kit-javafxgraphics-peers-gwt/src/main/java/dev/webfx/kit/mapper/peers/javafxgraphics/gwt/html/UserInteraction.java

@@ -0,0 +1,53 @@
+package dev.webfx.kit.mapper.peers.javafxgraphics.gwt.html;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * @author Bruno Salmon
+ */
+public final class UserInteraction {
+
+    private static int USER_INTERACTIONS_COUNT = 0;
+    private static boolean IS_USER_INTERACTING;
+    private static final List<Runnable> NEXT_USER_INTERACTION_RUNNABLES = new ArrayList<>();
+    private static boolean NEXT_USER_INTERACTION_RUNNABLE_REQUIRES_TOUCH_EVENT_DEFAULT;
+    public static void setUserInteracting(boolean on) {
+        IS_USER_INTERACTING = on;
+        if (on) {
+            USER_INTERACTIONS_COUNT++;
+            if (!NEXT_USER_INTERACTION_RUNNABLES.isEmpty()) {
+                for (Iterator<Runnable> it = NEXT_USER_INTERACTION_RUNNABLES.iterator(); it.hasNext(); ) {
+                    it.next().run();
+                    it.remove();
+                }
+                NEXT_USER_INTERACTION_RUNNABLE_REQUIRES_TOUCH_EVENT_DEFAULT = false;
+            }
+        }
+    }
+
+    public static boolean isUserInteracting() {
+        return IS_USER_INTERACTING;
+    }
+
+    public static boolean hasUserNotInteractedYet() {
+        return USER_INTERACTIONS_COUNT == 0;
+    }
+
+    public static boolean nextUserRunnableRequiresTouchEventDefault() {
+        return NEXT_USER_INTERACTION_RUNNABLE_REQUIRES_TOUCH_EVENT_DEFAULT;
+    }
+
+    public static void runOnNextUserInteraction(Runnable runnable) {
+        runOnNextUserInteraction(runnable, false);
+    }
+
+    public static void runOnNextUserInteraction(Runnable runnable, boolean requireTouchEventDefault) {
+        NEXT_USER_INTERACTION_RUNNABLES.add(runnable);
+        if (requireTouchEventDefault) {
+            NEXT_USER_INTERACTION_RUNNABLE_REQUIRES_TOUCH_EVENT_DEFAULT = true;
+        }
+    }
+
+}

webfx-kit/webfx-kit-javafxgraphics-peers-gwt/src/main/java/dev/webfx/kit/mapper/peers/javafxgraphics/gwt/shared/HtmlSvgNodePeer.java

@@ -12,6 +12,7 @@
 import dev.webfx.kit.mapper.peers.javafxgraphics.base.NodePeerImpl;
 import dev.webfx.kit.mapper.peers.javafxgraphics.base.NodePeerMixin;
 import dev.webfx.kit.mapper.peers.javafxgraphics.emul_coupling.LayoutMeasurable;
+import dev.webfx.kit.mapper.peers.javafxgraphics.gwt.html.UserInteraction;
 import dev.webfx.kit.mapper.peers.javafxgraphics.gwt.svg.SvgNodePeer;
 import dev.webfx.kit.mapper.peers.javafxgraphics.gwt.util.*;
 import dev.webfx.platform.uischeduler.UiScheduler;
@@ -407,7 +408,8 @@ private static void registerTouchListener(EventTarget htmlTarget, String type, j
             boolean fxConsumed = passHtmlTouchEventOnToFx((TouchEvent) e, type, fxTarget);
             if (fxConsumed) {
                 e.stopPropagation();
-                e.preventDefault();
+                if (!UserInteraction.nextUserRunnableRequiresTouchEventDefault())
+                    e.preventDefault();
             }
         }, passiveOption);
     }
@@ -429,11 +431,9 @@ protected static boolean passHtmlTouchEventOnToFx(TouchEvent e, String type, jav
             ((Scene) fxTarget).impl_processMouseEvent(mouseEvent);
             // We return true (even if not consumed) to always prevent browsers built-in touch scrolling, unless if the
             // target is a standard html tag that reacts to touch elements, such as:
-            if (e.target instanceof HTMLAnchorElement // <a> clickable link
-                    || e.target instanceof HTMLInputElement // <input> (ex: slider)
-                    || e.target instanceof HTMLLabelElement) // <label> that may embed an <input> such as WebFX Extras FilePicker button
-                return false;
-            return true;
+            consumed = !(e.target instanceof HTMLAnchorElement) // <a> clickable link
+                       && !(e.target instanceof HTMLInputElement) // <input> (ex: slider)
+                       && !(e.target instanceof HTMLLabelElement); // <label> that may embed an <input> such as WebFX Extras FilePicker button
         }
         return consumed; // should be normally: return consumed
     }

webfx-kit/webfx-kit-javafxmedia-peers-gwt/src/main/java/dev/webfx/kit/mapper/peers/javafxmedia/spi/gwt/GwtMediaModuleBooter.java

@@ -34,6 +34,7 @@
  *
  * @author Bruno Salmon
  */
+@Deprecated
 public class GwtMediaModuleBooter implements ApplicationModuleBooter {
 
     private static boolean AUDIO_REQUIRES_USER_INTERACTION_FIRST = true;

webfx-kit/webfx-kit-javafxmedia-peers-gwt/src/main/java/dev/webfx/kit/mapper/peers/javafxmedia/spi/gwt/GwtMediaPlayerPeer.java

@@ -3,6 +3,7 @@
 import com.google.gwt.core.client.JavaScriptObject;
 import com.google.gwt.core.client.JsonUtils;
 import com.google.gwt.storage.client.Storage;
+import dev.webfx.kit.mapper.peers.javafxgraphics.gwt.html.UserInteraction;
 import dev.webfx.kit.mapper.peers.javafxgraphics.gwt.util.HtmlUtil;
 import dev.webfx.kit.mapper.peers.javafxmedia.MediaPlayerPeer;
 import dev.webfx.platform.console.Console;
@@ -16,6 +17,7 @@
 import javafx.beans.property.ObjectProperty;
 import javafx.beans.property.SimpleObjectProperty;
 import javafx.scene.media.AudioSpectrumListener;
+import javafx.scene.media.Media;
 import javafx.scene.media.MediaPlayer;
 import javafx.util.Duration;
 
@@ -93,7 +95,7 @@ static AudioContext getAudioContext() {
     }*/
 
     private static boolean isAudioContextReady(boolean resumeIfSuspended) {
-        if (AUDIO_CONTEXT == null && GwtMediaModuleBooter.audioRequiresUserInteractionFirst())
+        if (AUDIO_CONTEXT == null && UserInteraction.hasUserNotInteractedYet())
             return false;
         if (getAudioContext().state.equalsIgnoreCase("suspended")) {
             if (!resumeIfSuspended)
@@ -208,8 +210,8 @@ private void fetchAudioBuffer(boolean resumeIfSuspended) {
                         return null;
                     });
             fetched = true;
-        } else if (GwtMediaModuleBooter.audioRequiresUserInteractionFirst())
-            GwtMediaModuleBooter.runOnFirstUserInteraction(() -> fetchAudioBuffer(true));
+        } else if (UserInteraction.hasUserNotInteractedYet())
+            UserInteraction.runOnNextUserInteraction(() -> fetchAudioBuffer(true));
     }
 
     private void onAudioBufferReady() {
@@ -265,10 +267,12 @@ private void playOnceCycle() {
         // it's better not to try (otherwise a warning will be logged in the console). If the media is an AudioClip
         // (ex: short game sound) we will just ignore it, but if it's a music (ex: background music), we postpone the
         // play to the first user interaction.
-        if (!mute && GwtMediaModuleBooter.audioRequiresUserInteractionFirst()) {
+        if (!mute && UserInteraction.hasUserNotInteractedYet()) {
             if (!audioClip && !playWhenReady) {
                 playWhenReady = true;
-                GwtMediaModuleBooter.runOnFirstUserInteraction(() -> {
+                // checking that the music hasn't been stopped or paused in the meantime
+                // Ok, we can now start playing the music
+                UserInteraction.runOnNextUserInteraction(() -> {
                     if (playWhenReady) // checking that the music hasn't been stopped or paused in the meantime
                         playOnceCycle(); // Ok, we can now start playing the music
                 });
@@ -700,4 +704,35 @@ private void memoriseWorkingCrossOrigin(HTMLMediaElement mediaElement) {
         }
     }
 
+    /**
+     *
+     * The purpose of this static initialiser is to ensure that the sound will play ok on iOS and iPadOS after the first
+     * user interaction.
+     * ==========================
+     * Description of the problem
+     * ==========================
+     * Other OS automatically unlock the sound on first user interaction, even if the application code doesn't play any
+     * sound at this time, it can play sound any time later after the first user interaction, even not necessarily during
+     *  a user interaction. On iOS and iPadOS however, this sound unlocking is not automatic. The unlocking happens only
+     * when the application plays a sound DURING the user interaction.
+     * Because of this difference, if the JavaFX application code tries to start playing sound using setOnMouseClicked(),
+     * this won't work (it will work however with setOnMousePressed() or setOnMouseReleased()). This is due to the way
+     * WebFX emulates the JavaFX click event, which is not based on the JavaScript "click" event as opposed to the other
+     * events, because JavaFX has its own way to fire it when detecting the mouse released, and WebFX postpones this process
+     * (see HtmlScenePeer.java, installMouseListeners() and passHtmlMouseEventOnToFx() methods).
+     * ===========================
+     * Description of the solution
+     * ===========================
+     * This static initializer will automatically detect the first (or next) user interaction and play a silent sound
+     * for a very short time during that interaction, causing the sound unlocking even on iOS and iPadOS. Then, if the
+     * JavaFX application requests playing sound using setOnMouseClicked(), it will work because the sound unlocking has
+     * previously been done.
+     */
+
+    static {
+        UserInteraction.runOnNextUserInteraction(() -> {
+            String tinySilentMp3Data = "data:audio/mpeg;base64,SUQzBAAAAAABEVRYWFgAAAAtAAADY29tbWVudABCaWdTb3VuZEJhbmsuY29tIC8gTGFTb25vdGhlcXVlLm9yZwBURU5DAAAAHQAAA1N3aXRjaCBQbHVzIMKpIE5DSCBTb2Z0d2FyZQBUSVQyAAAABgAAAzIyMzUAVFNTRQAAAA8AAANMYXZmNTcuODMuMTAwAAAAAAAAAAAAAAD/80DEAAAAA0gAAAAATEFNRTMuMTAwVVVVVVVVVVVVVUxBTUUzLjEwMFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVf/zQsRbAAADSAAAAABVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVf/zQMSkAAADSAAAAABVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV";
+            new GwtMediaPlayerPeer(new MediaPlayer(new Media(tinySilentMp3Data)), true).play(); // This will unlock the sound
+        });
+    }
 }

webfx-kit/webfx-kit-javafxmedia-peers-gwt/webfx.xml

@@ -6,7 +6,6 @@
     </exported-packages>
 
     <providers>
-        <provider interface="dev.webfx.platform.boot.spi.ApplicationModuleBooter">dev.webfx.kit.mapper.peers.javafxmedia.spi.gwt.GwtMediaModuleBooter</provider>
         <provider interface="dev.webfx.kit.mapper.peers.javafxmedia.spi.WebFxKitMediaMapperProvider">dev.webfx.kit.mapper.peers.javafxmedia.spi.gwt.GwtWebFxKitMediaMapperProvider</provider>
     </providers>