diff --git a/pkg/work/spoke/controllers/timestampcontroller/timestamp_controller.go b/pkg/work/spoke/controllers/timestampcontroller/timestamp_controller.go
index 34ccabf57..4fda95d69 100644
--- a/pkg/work/spoke/controllers/timestampcontroller/timestamp_controller.go
+++ b/pkg/work/spoke/controllers/timestampcontroller/timestamp_controller.go
@@ -90,7 +90,12 @@ func (c *TimestampController) syncManifestWork(ctx context.Context, logger klog.
 		Generation: manifestWork.Generation,
 		StartTime:  manifestWork.CreationTimestamp.Time,
 	}
+	if oldGenerationTime != nil {
+		// keep the first generation applied time we observed unchanged
+		generationTime.FirstGenerationAppliedTime = oldGenerationTime.FirstGenerationAppliedTime
+	}
 
+	// set the generation start time
 	for _, field := range manifestWork.ManagedFields {
 		if field.Time == nil || field.FieldsV1 == nil {
 			continue
@@ -101,21 +106,27 @@ func (c *TimestampController) syncManifestWork(ctx context.Context, logger klog.
 				generationTime.StartTime = field.Time.Time
 			}
 		}
-
 	}
 
-	// set the applied time if it matches
+	// set the generation applied time if it matches
 	cond := meta.FindStatusCondition(manifestWork.Status.Conditions, workapiv1.WorkApplied)
-	if cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == generationTime.Generation {
-		generationTime.AppliedTime = cond.LastTransitionTime.Time
-
-		// if the generation changes, but the status of the manifestwork Applied condition does not change, the
-		// lastTransitionTime will not change, so it is possible that the lastTransitionTime is befor the generation
-		// start time
-		if generationTime.AppliedTime.Before(generationTime.StartTime) {
-			// TODO: review whether time.Now() makes sence here
-			generationTime.AppliedTime = time.Now()
+	if cond != nil && cond.Status == metav1.ConditionTrue {
+		// if the first generation applied time does not set, set the value we observed
+		if generationTime.FirstGenerationAppliedTime.IsZero() {
+			generationTime.FirstGenerationAppliedTime = cond.LastTransitionTime.Time
 		}
+		if cond.ObservedGeneration == generationTime.Generation {
+			generationTime.AppliedTime = cond.LastTransitionTime.Time
+
+			// if the generation changes, but the status of the manifestwork Applied condition does not change, the
+			// lastTransitionTime will not change, so it is possible that the lastTransitionTime is befor the generation
+			// start time
+			if generationTime.AppliedTime.Before(generationTime.StartTime) {
+				// TODO: review whether time.Now() makes sence here
+				generationTime.AppliedTime = time.Now()
+			}
+		}
+
 	}
 
 	// skip if the generation time annotation does not change.
@@ -158,7 +169,8 @@ func (c *TimestampController) getGenerationTime(mw *workapiv1.ManifestWork) *Gen
 }
 
 type GenerationTimestamp struct {
-	Generation  int64     `json:"generation"`
-	StartTime   time.Time `json:"startTime"`
-	AppliedTime time.Time `json:"appliedTime"`
+	Generation                 int64     `json:"generation"`
+	StartTime                  time.Time `json:"startTime"`
+	AppliedTime                time.Time `json:"appliedTime"`
+	FirstGenerationAppliedTime time.Time `json:"firstGenerationAppliedTime"`
 }