Skip to content

Commit

Permalink
feat: Add annotation refreshed-at to delay TTL (#97)
Browse files Browse the repository at this point in the history 
* 96: implementation of refreshed-at annotation with tests

* 96: add line in documentation

* 96: fix annotation name, typo

* 96: avoid err overriding

* 96: add kubectl annotate example

* 96: a bit clearer text

* Update README.md

---------

Co-authored-by: TwiN <[email protected]>
  • Loading branch information
@hiddenmarten @TwiN
hiddenmarten and TwiN authored Dec 8, 2024
1 parent 054dde7 commit 1b7a4af
Show file tree
Hide file tree
Showing 3 changed files with 76 additions and 6 deletions.
9 changes: 9 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ a value such as `30m`, `24h` and `7d`.
The resource is deleted after the current timestamp surpasses the sum of the resource's `metadata.creationTimestamp` and
the duration specified by the `k8s-ttl-controller.twin.sh/ttl` annotation.

If the resource is annotated with `k8s-ttl-controller.twin.sh/refreshed-at`, the TTL will be calculated from the value of
this annotation instead of the `metadata.creationTimestamp`.

## Usage
### Setting a TTL on a resource
Expand All @@ -23,6 +25,13 @@ kubectl annotate pod hello-world k8s-ttl-controller.twin.sh/ttl=1h
The pod `hello-world` would be deleted in approximately 40 minutes, because 20 minutes have already elapsed, leaving
40 minutes until the target TTL of 1h is reached.

In the same way, you can refresh the TTL by placing the annotation `k8s-ttl-controller.twin.sh/refreshed-at`, like this:
```console
kubectl annotate pod hello-world k8s-ttl-controller.twin.sh/refreshed-at=2024-12-08T20:48:11Z
# Alternatively:
kubectl annotate pod hello-world k8s-ttl-controller.twin.sh/refreshed-at=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
```

Alternatively, you can create resources with the annotation already present:
```yaml
apiVersion: v1
Expand Down
24 changes: 19 additions & 5 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ import (
)

const (
AnnotationTTL = "k8s-ttl-controller.twin.sh/ttl"
AnnotationTTL = "k8s-ttl-controller.twin.sh/ttl"
AnnotationRefreshedAt = "k8s-ttl-controller.twin.sh/refreshed-at"

MaximumFailedExecutionBeforePanic = 10 // Maximum number of allowed failed executions before panicking
ExecutionTimeout = 20 * time.Minute // Maximum time for each reconciliation before timing out
Expand Down Expand Up @@ -90,6 +91,18 @@ func Reconcile(kubernetesClient kubernetes.Interface, dynamicClient dynamic.Inte
}
}

func getStartTime(item unstructured.Unstructured) metav1.Time {
refreshedAt, exists := item.GetAnnotations()[AnnotationRefreshedAt]
if exists {
t, err := time.Parse(time.RFC3339, refreshedAt)
if err == nil {
return metav1.NewTime(t)
}
log.Printf("Failed to parse refreshed-at timestamp '%s' for %s/%s: %s", refreshedAt, item.GetKind(), item.GetName(), err)
}
return item.GetCreationTimestamp()
}

// DoReconcile goes over all API resources specified, retrieves all sub resources and deletes those who have expired
func DoReconcile(dynamicClient dynamic.Interface, eventManager *kevent.EventManager, resources []*metav1.APIResourceList) bool {
for _, resource := range resources {
Expand All @@ -116,6 +129,7 @@ func DoReconcile(dynamicClient dynamic.Interface, eventManager *kevent.EventMana
gvr.Resource = apiResource.Name
var list *unstructured.UnstructuredList
var continueToken string
var ttlInDuration time.Duration
var err error
for list == nil || continueToken != "" {
list, err = dynamicClient.Resource(gvr).List(context.TODO(), metav1.ListOptions{TimeoutSeconds: &listTimeoutSeconds, Continue: continueToken, Limit: ListLimit})
Expand All @@ -134,16 +148,16 @@ func DoReconcile(dynamicClient dynamic.Interface, eventManager *kevent.EventMana
if !exists {
continue
}
ttlInDuration, err := str2duration.ParseDuration(ttl)
ttlInDuration, err = str2duration.ParseDuration(ttl)
if err != nil {
log.Printf("[%s/%s] has an invalid TTL '%s': %s\n", apiResource.Name, item.GetName(), ttl, err)
continue
}
ttlExpired := time.Now().After(item.GetCreationTimestamp().Add(ttlInDuration))
ttlExpired := time.Now().After(getStartTime(item).Add(ttlInDuration))
if ttlExpired {
durationSinceExpired := time.Since(item.GetCreationTimestamp().Add(ttlInDuration)).Round(time.Second)
durationSinceExpired := time.Since(getStartTime(item).Add(ttlInDuration)).Round(time.Second)
log.Printf("[%s/%s] is configured with a TTL of %s, which means it has expired %s ago", apiResource.Name, item.GetName(), ttl, durationSinceExpired)
err := dynamicClient.Resource(gvr).Namespace(item.GetNamespace()).Delete(context.TODO(), item.GetName(), metav1.DeleteOptions{})
err = dynamicClient.Resource(gvr).Namespace(item.GetNamespace()).Delete(context.TODO(), item.GetName(), metav1.DeleteOptions{})
if err != nil {
log.Printf("[%s/%s] failed to delete: %s\n", apiResource.Name, item.GetName(), err)
eventManager.Create(item.GetNamespace(), item.GetKind(), item.GetName(), "FailedToDeleteExpiredTTL", "Unable to delete expired resource:"+err.Error(), true)
Expand Down
49 changes: 48 additions & 1 deletion main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,53 @@ func TestReconcile(t *testing.T) {
},
expectedResourcesLeftAfterReconciliation: 2,
},
{
name: "expired-pod-is-deleted-by-refreshed-at",
podsToCreate: []*unstructured.Unstructured{
newUnstructuredWithAnnotations("v1", "Pod", "default", "expired-pod-name", time.Now().Add(-time.Hour), map[string]interface{}{AnnotationTTL: "5m", AnnotationRefreshedAt: time.Now().Add(-10 * time.Minute).Format(time.RFC3339)}),
},
expectedResourcesLeftAfterReconciliation: 0,
},
{
name: "not-expired-pod-is-not-deleted-by-refreshed-at",
podsToCreate: []*unstructured.Unstructured{
newUnstructuredWithAnnotations("v1", "Pod", "default", "not-expired-pod-name", time.Now().Add(-time.Hour), map[string]interface{}{AnnotationTTL: "3d", AnnotationRefreshedAt: time.Now().Add(-10 * time.Minute).Format(time.RFC3339)}),
},
expectedResourcesLeftAfterReconciliation: 1,
},
{
name: "unannotated-pod-is-not-deleted-by-refreshed-at",
podsToCreate: []*unstructured.Unstructured{
newUnstructuredWithAnnotations("v1", "Pod", "default", "unannotated-pod-name", time.Now().Add(-time.Hour), map[string]interface{}{AnnotationRefreshedAt: time.Now().Add(-10 * time.Minute).Format(time.RFC3339)}),
},
expectedResourcesLeftAfterReconciliation: 1,
},
{
name: "one-out-of-two-pods-is-deleted-because-only-one-expired-by-refreshed-at",
podsToCreate: []*unstructured.Unstructured{
newUnstructuredWithAnnotations("v1", "Pod", "default", "not-expired-pod-name", time.Now().Add(-time.Hour), map[string]interface{}{AnnotationTTL: "3d", AnnotationRefreshedAt: time.Now().Add(-10 * time.Minute).Format(time.RFC3339)}),
newUnstructuredWithAnnotations("v1", "Pod", "default", "expired-pod-name", time.Now().Add(-time.Hour), map[string]interface{}{AnnotationTTL: "5m", AnnotationRefreshedAt: time.Now().Add(-10 * time.Minute).Format(time.RFC3339)}),
},
expectedResourcesLeftAfterReconciliation: 1,
},
{
name: "multiple-expired-pods-are-deleted-by-refreshed-at",
podsToCreate: []*unstructured.Unstructured{
newUnstructuredWithAnnotations("v1", "Pod", "default", "expired-pod-name-1", time.Now().Add(-time.Hour), map[string]interface{}{AnnotationTTL: "5m", AnnotationRefreshedAt: time.Now().Add(-10 * time.Minute).Format(time.RFC3339)}),
newUnstructuredWithAnnotations("v1", "Pod", "default", "expired-pod-name-2", time.Now().Add(-72*time.Hour), map[string]interface{}{AnnotationTTL: "2d", AnnotationRefreshedAt: time.Now().Add(-49 * time.Hour).Format(time.RFC3339)}),
},
expectedResourcesLeftAfterReconciliation: 0,
},
{
name: "only-expired-pods-are-deleted-by-refreshed-at",
podsToCreate: []*unstructured.Unstructured{
newUnstructuredWithAnnotations("v1", "Pod", "default", "expired-pod-name-1", time.Now().Add(-time.Hour), map[string]interface{}{AnnotationTTL: "5m", AnnotationRefreshedAt: time.Now().Add(-10 * time.Minute).Format(time.RFC3339)}),
newUnstructuredWithAnnotations("v1", "Pod", "default", "not-expired-pod-name", time.Now().Add(-time.Hour), map[string]interface{}{AnnotationTTL: "3d", AnnotationRefreshedAt: time.Now().Add(-10 * time.Minute).Format(time.RFC3339)}),
newUnstructuredWithAnnotations("v1", "Pod", "default", "expired-pod-name-2", time.Now().Add(-72*time.Hour), map[string]interface{}{AnnotationTTL: "2d", AnnotationRefreshedAt: time.Now().Add(-49 * time.Hour).Format(time.RFC3339)}),
newUnstructuredWithAnnotations("v1", "Pod", "default", "unannotated-pod-name", time.Now().Add(-time.Hour), map[string]interface{}{AnnotationRefreshedAt: time.Now().Add(-10 * time.Minute).Format(time.RFC3339)}),
},
expectedResourcesLeftAfterReconciliation: 2,
},
}

// Run scenarios
Expand Down Expand Up @@ -121,7 +168,7 @@ func TestReconcile(t *testing.T) {
t.Errorf("expected 3 resources, got %d", len(list.Items))
}
// Reconcile once
if err := Reconcile(kubernetesClient, dynamicClient, eventManager); err != nil {
if err = Reconcile(kubernetesClient, dynamicClient, eventManager); err != nil {
t.Errorf("unexpected error: %v", err)
}
// Make sure that the expired resources have been deleted
Expand Down

0 comments on commit 1b7a4af

Please sign in to comment.