diff --git a/doc/upgrade/cleanup_legacy_postgres.sh b/doc/upgrade/cleanup_legacy_postgres.sh new file mode 100755 index 000000000..1e4f6f8e1 --- /dev/null +++ b/doc/upgrade/cleanup_legacy_postgres.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Set the namespace, default to "multicluster-global-hub" if not provided +NAMESPACE="${1:-multicluster-global-hub}" + +echo ">> Starting cleanup of legacy Postgres resources in namespace: $NAMESPACE" + +# Remove the StatefulSet +echo ">> Deleting StatefulSet: multicluster-global-hub-postgres..." +kubectl delete sts multicluster-global-hub-postgres -n $NAMESPACE + +# Remove Services +echo ">> Deleting ConfigMaps..." +kubectl delete svc multicluster-global-hub-postgres -n $NAMESPACE + +# Remove ConfigMaps +echo ">> Deleting ConfigMaps..." +kubectl delete cm multicluster-global-hub-postgres-ca -n $NAMESPACE +kubectl delete cm multicluster-global-hub-postgres-config -n $NAMESPACE +kubectl delete cm multicluster-global-hub-postgres-init -n $NAMESPACE + +# Remove Secrets +echo ">> Deleting Secrets..." +kubectl delete secret postgres-credential-secret -n $NAMESPACE +kubectl delete secret multicluster-global-hub-postgres -n $NAMESPACE +kubectl delete secret multicluster-global-hub-postgres-certs -n $NAMESPACE + +echo ">> Cleanup completed for namespace: $NAMESPACE" diff --git a/doc/upgrade/restore_event_tables.sh b/doc/upgrade/restore_event_tables.sh index 2ce71b82b..50e190d00 100755 --- a/doc/upgrade/restore_event_tables.sh +++ b/doc/upgrade/restore_event_tables.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Tables: `history.local_compliance` and `history.local_compliance_job_log` +# Tables: event.local_policies , event.local_root_policies, and event.managed_clusters # Set namespace and pod names NAMESPACE="${1:-multicluster-global-hub}" @@ -8,21 +8,48 @@ SOURCE_POD="multicluster-global-hub-postgres-0" # Source pod SOURCE_CONTAINER="multicluster-global-hub-postgres" # Source container TARGET_POD="multicluster-global-hub-postgresql-0" # Target pod TARGET_CONTAINER="multicluster-global-hub-postgresql" # Target container -REMOTE_PATH="/tmp/history_tables_backup.sql" # Without leading `/` to suppress tar warnings +REMOTE_PATH="/tmp/event_tables_backup.sql" # Without leading `/` to suppress tar warnings DB_NAME="hoh" DB_USER="postgres" echo "Using namespace: $NAMESPACE" # Backup from source pod -echo ">> Backing up history tables from $SOURCE_POD ($SOURCE_CONTAINER)..." -kubectl exec -c $SOURCE_CONTAINER $SOURCE_POD -n $NAMESPACE -- pg_dump -U $DB_USER -d $DB_NAME -t 'history.*' --data-only -f $REMOTE_PATH 2>/dev/null -kubectl cp $NAMESPACE/$SOURCE_POD:$REMOTE_PATH ./history_tables_backup.sql -c $SOURCE_CONTAINER >/dev/null 2>&1 +echo ">> Backing up event tables from $SOURCE_POD ($SOURCE_CONTAINER)..." +kubectl exec -c $SOURCE_CONTAINER $SOURCE_POD -n $NAMESPACE -- pg_dump -U $DB_USER -d $DB_NAME -t 'event.*' --data-only -f $REMOTE_PATH 2>/dev/null +kubectl cp $NAMESPACE/$SOURCE_POD:$REMOTE_PATH ./event_tables_backup.sql -c $SOURCE_CONTAINER >/dev/null 2>&1 echo ">> Backup completed." +# List of tables to process +tables=("event.local_policies" "event.local_root_policies" "event.managed_clusters") + +echo ">> Removing duplicated event records from the target tables" + +for table in "${tables[@]}"; do + echo "> Processing table: $table" + + # Get the latest timestamp from the source table + latest_time=$(kubectl exec -it -c $SOURCE_CONTAINER $SOURCE_POD -n $NAMESPACE -- psql -U $DB_USER -d $DB_NAME -t -c "SELECT MAX(created_at) FROM $table;") + + latest_time=$(echo $latest_time | xargs) # Trim whitespace + + # Validate if latest_time is empty, NULL, or not a valid timestamp + if [ -z "$latest_time" ] || [ "$latest_time" == "NULL" ] || ! [[ "$latest_time" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then + echo " No records(or invalid) found in $table. Skipping..." + continue + fi + + echo "> Latest created_at time for $table: $latest_time" + # Delete records from the target table with a timestamp earlier than the latest time + echo "> Deleting records from $table with created_at <= $latest_time" + kubectl exec -it -c $TARGET_CONTAINER $TARGET_POD -n $NAMESPACE -- psql -U $DB_USER -d $DB_NAME -c "DELETE FROM $table WHERE created_at <= '$latest_time';" +done + +echo ">> Completed removing duplicated event records." + # Restore to target pod -echo ">> Restoring history tables to $TARGET_POD ($TARGET_CONTAINER)..." -kubectl cp ./history_tables_backup.sql $NAMESPACE/$TARGET_POD:$REMOTE_PATH -c $TARGET_CONTAINER +echo ">> Restoring event tables to $TARGET_POD ($TARGET_CONTAINER)..." +kubectl cp ./event_tables_backup.sql $NAMESPACE/$TARGET_POD:$REMOTE_PATH -c $TARGET_CONTAINER kubectl exec -c $TARGET_CONTAINER $TARGET_POD -n $NAMESPACE -- psql -U $DB_USER -d $DB_NAME -f $REMOTE_PATH echo ">> Restore completed." @@ -30,5 +57,5 @@ echo ">> Restore completed." echo ">> Cleaning up temporary files..." kubectl exec -c $SOURCE_CONTAINER $SOURCE_POD -n $NAMESPACE -- rm $REMOTE_PATH kubectl exec -c $TARGET_CONTAINER $TARGET_POD -n $NAMESPACE -- rm $REMOTE_PATH -rm ./history_tables_backup.sql +rm ./event_tables_backup.sql echo ">> Local and remote backup files removed. Done!" diff --git a/doc/upgrade/upgrade_to_globalhub_1.4.0.md b/doc/upgrade/upgrade_to_globalhub_1.4.0.md index b50229e91..0b17a32cc 100644 --- a/doc/upgrade/upgrade_to_globalhub_1.4.0.md +++ b/doc/upgrade/upgrade_to_globalhub_1.4.0.md @@ -1,36 +1,27 @@ -## Upgrade the Builtin Postgres from Global Hub Release-1.3.0 to 1.4.0 +# Upgrade the Built-in Postgres from Global Hub Release-1.3.0 to 1.4.0 -The builtin postgres version has been upgraed to 16 since Global Hub Release-1.4.0. And the statefulset instance is changed from `multicluster-global-hub-postgres`(version 13) to `multicluster-global-hub-postgresql`. By default, after the upgradation, the real time data like policy and cluster will be resynced to the new builtin postgres instances. +With the Global Hub Release 1.4.0, the built-in Postgres database has been upgraded to version 16, replacing the existing statefulset instance `multicluster-global-hub-postgres` (version 13) with a new instance, `multicluster-global-hub-postgresql`. By default, after the upgrade, real-time data such as **policies** and **clusters** will be automatically re-synced to the new Postgres instance. -But the history data like event and history tables will be be recovered. If you don't want ignore these history data. You can just skip the hitostry backup and restore session. +However, historical data (e.g., **event** and **history** tables) will not be recovered automatically. If retaining historical data is not a concern, you can skip the backup and restore steps below. Otherwise, follow the optional backup and restore instructions. -### History data backup and restore[Optional] +## Backup and Restore Data(history and event) [Optional] -1. Restore History Tables use the script `./doc/upgrade/restore_history_tables.sh`. +- Run the following script to restore history tables -2. Restore Event Tables use the script `./doc/upgrade/restore_event_tables.sh`. - -### Delete the legacy builtin postgres resources manually[Optional] - -Since the global hub will swith to the new builin postgres. So you can delete the resouces of the origin which The glolba hub operator will not delete them automatically! - -- Remove statefulset -``` -kubectl delete sts multicluster-global-hub-postgres -n multicluster-global-hub +```bash +./doc/upgrade/restore_history_tables.sh ``` -- Remove the configmap +- Run the following script to restore event tables -``` -kubectl delete cm multicluster-global-hub-postgres-ca -n multicluster-global-hub -kubectl delete cm multicluster-global-hub-postgres-config -n multicluster-global-hub -kubectl delete cm multicluster-global-hub-postgres-init -n multicluster-global-hub +```bash +./doc/upgrade/restore_event_tables.sh ``` -- Remove the secret +## Delete Legacy Built-in Postgres Resources Manually [Optional] -``` -kubectl delete secret postgres-credential-secret -n multicluster-global-hub -kubectl delete secret multicluster-global-hub-postgres -n multicluster-global-hub -kubectl delete secret multicluster-global-hub-postgres-certs -n multicluster-global-hub -``` +After the upgrade, the Global Hub switches to the new built-in Postgres instance. Resources associated with the legacy Postgres instance are not deleted automatically by the Global Hub operator. To clean up these resources, Use the following script to delete legacy Postgres resources. + + ```bash + ./doc/upgrade/restore_event_tables.sh + ``` \ No newline at end of file diff --git a/event_tables_backup.sql b/event_tables_backup.sql new file mode 100644 index 000000000..3bcf75179 --- /dev/null +++ b/event_tables_backup.sql @@ -0,0 +1,80 @@ +-- +-- PostgreSQL database dump +-- + +-- Dumped from database version 13.7 +-- Dumped by pg_dump version 13.7 + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Data for Name: data_retention_job_log; Type: TABLE DATA; Schema: event; Owner: postgres +-- + +COPY event.data_retention_job_log (table_name, start_at, end_at, min_partition, max_partition, min_deletion, error) FROM stdin; +\. + + +-- +-- Data for Name: local_policies_2024_11; Type: TABLE DATA; Schema: event; Owner: postgres +-- + +COPY event.local_policies_2024_11 (event_name, event_namespace, policy_id, cluster_id, cluster_name, leaf_hub_name, message, reason, count, source, created_at, compliance) FROM stdin; +\. + + +-- +-- Data for Name: local_policies_2024_12; Type: TABLE DATA; Schema: event; Owner: postgres +-- + +COPY event.local_policies_2024_12 (event_name, event_namespace, policy_id, cluster_id, cluster_name, leaf_hub_name, message, reason, count, source, created_at, compliance) FROM stdin; +PolicyViolation namespace-a 550e8400-e29b-41d4-a716-446655440000 660e8400-e29b-41d4-a716-446655440000 cluster-a leaf-hub-1 Policy violated on resource X. NonCompliant 1 {"type": "kubernetes", "component": "policy-engine"} 2024-12-18 03:14:21.795684 non_compliant +PolicyRemediation namespace-b 750e8400-e29b-41d4-a716-446655440001 860e8400-e29b-41d4-a716-446655440002 cluster-b leaf-hub-2 Remediation applied successfully. Compliant 1 {"type": "kubernetes", "component": "remediation-operator"} 2024-12-18 03:14:21.822416 compliant +\. + + +-- +-- Data for Name: local_root_policies_2024_11; Type: TABLE DATA; Schema: event; Owner: postgres +-- + +COPY event.local_root_policies_2024_11 (event_name, event_namespace, policy_id, leaf_hub_name, message, reason, count, source, created_at, compliance) FROM stdin; +\. + + +-- +-- Data for Name: local_root_policies_2024_12; Type: TABLE DATA; Schema: event; Owner: postgres +-- + +COPY event.local_root_policies_2024_12 (event_name, event_namespace, policy_id, leaf_hub_name, message, reason, count, source, created_at, compliance) FROM stdin; +\. + + +-- +-- Data for Name: managed_clusters_2024_11; Type: TABLE DATA; Schema: event; Owner: postgres +-- + +COPY event.managed_clusters_2024_11 (event_namespace, event_name, cluster_name, cluster_id, leaf_hub_name, message, reason, reporting_controller, reporting_instance, event_type, created_at) FROM stdin; +\. + + +-- +-- Data for Name: managed_clusters_2024_12; Type: TABLE DATA; Schema: event; Owner: postgres +-- + +COPY event.managed_clusters_2024_12 (event_namespace, event_name, cluster_name, cluster_id, leaf_hub_name, message, reason, reporting_controller, reporting_instance, event_type, created_at) FROM stdin; +\. + + +-- +-- PostgreSQL database dump complete +-- +