diff --git a/canvas/image.go b/canvas/image.go
index 074bb6574d..e0dd7e13ff 100644
--- a/canvas/image.go
+++ b/canvas/image.go
@@ -100,7 +100,9 @@ func (i *Image) Hide() {
 // MinSize returns the specified minimum size, if set, or {1, 1} otherwise.
 func (i *Image) MinSize() fyne.Size {
 	if i.Image == nil || i.aspect == 0 {
-		i.Refresh()
+		if i.File != "" || i.Resource != nil {
+			i.Refresh()
+		}
 	}
 	return i.baseObject.MinSize()
 }
@@ -134,6 +136,8 @@ func (i *Image) Refresh() {
 			return
 		}
 		rc = io.NopCloser(r)
+	} else {
+		return
 	}
 
 	if i.File != "" || i.Resource != nil {