diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 2076d3cdf..4d0dec3db 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -1286,7 +1286,8 @@ def get_annotations(): } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-upgrade-test", pg_patch_version_higher_version) - self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync") + self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, "Operator does not get in sync", + retries=180) k8s.wait_for_pod_failover(master_nodes, 'spilo-role=replica,' + cluster_label) k8s.wait_for_pod_start('spilo-role=master,' + cluster_label) @@ -2412,6 +2413,19 @@ def test_taint_based_eviction(self): self.assertNotEqual(master_nodes, []) self.assertNotEqual(replica_nodes, []) + # Keep the operator running if it shares the master's node. + operator_pod = k8s.get_operator_pod() + operator_tolerations = [ + k8s.api.k8s_client.sanitize_for_serialization(toleration) + for toleration in (operator_pod.spec.tolerations or []) + ] + operator_tolerations.append({ + "key": "postgres", + "operator": "Exists", + "effect": "NoExecute" + }) + k8s.patch_pod({"spec": {"tolerations": operator_tolerations}}, operator_pod.metadata.name) + # taint node with postgres=:NoExecute to force failover body = { "spec": { @@ -2435,7 +2449,7 @@ def test_taint_based_eviction(self): try: k8s.update_config(patch_toleration_config, step="allow tainted nodes") self.eventuallyEqual(lambda: k8s.get_operator_state(), {"0": "idle"}, - "Operator does not get in sync") + "Operator does not get in sync", retries=180) self.eventuallyEqual(lambda: k8s.count_running_pods(), 2, "No 2 pods running") self.eventuallyEqual(lambda: len(k8s.get_patroni_running_members("acid-minimal-cluster-0")), 2, "Postgres status did not enter running") diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index d46dc6067..b55596d92 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -109,6 +109,27 @@ func (c *Cluster) pgConnectionString(dbname string) string { constants.PostgresConnectTimeout/time.Second) } +func logicalReplicationSlotLagQuery(threshold int64) string { + return fmt.Sprintf(`WITH slot_lag AS ( + SELECT slot_name, pg_wal_lsn_diff(pg_current_wal_lsn(), confirmed_flush_lsn) AS lag_bytes + FROM pg_replication_slots + WHERE slot_type = 'logical' + ) + SELECT slot_name, COALESCE(lag_bytes::text, 'unknown') + FROM slot_lag + WHERE lag_bytes IS NULL OR lag_bytes > %d`, threshold) +} + +func (c *Cluster) getLaggingLogicalReplicationSlots(podName *spec.NamespacedName, threshold int64) (string, error) { + query := logicalReplicationSlotLagQuery(threshold) + + output, err := c.ExecCommand(podName, "psql", "-X", "-A", "-t", "-v", "ON_ERROR_STOP=1", "-U", c.OpConfig.SuperUsername, "-d", "postgres", "-c", query) + if err != nil { + return "", fmt.Errorf("could not execute logical replication slot lag query: %w", err) + } + return strings.TrimSpace(output), nil +} + func (c *Cluster) databaseAccessDisabled() bool { if !c.OpConfig.EnableDBAccess { c.logger.Debug("database access is disabled") diff --git a/pkg/cluster/majorversionupgrade.go b/pkg/cluster/majorversionupgrade.go index dae5db2e8..1d8b4ba8f 100644 --- a/pkg/cluster/majorversionupgrade.go +++ b/pkg/cluster/majorversionupgrade.go @@ -16,8 +16,10 @@ import ( ) const ( - majorVersionUpgradeSuccessAnnotation = "last-major-upgrade-success" - majorVersionUpgradeFailureAnnotation = "last-major-upgrade-failure" + majorVersionUpgradeSuccessAnnotation = "last-major-upgrade-success" + majorVersionUpgradeFailureAnnotation = "last-major-upgrade-failure" + logicalReplicationSlotLagThreshold = 2 * 16 * 1024 * 1024 + logicalReplicationSlotLagCheckMinVersion = 17 * 10000 ) // IsBiggerPostgresVersion Compare two Postgres version numbers @@ -230,6 +232,18 @@ func (c *Cluster) majorVersionUpgrade() error { isUpgradeSuccess := true numberOfPods := len(pods) if allRunning { + podName := &spec.NamespacedName{Namespace: masterPod.Namespace, Name: masterPod.Name} + if c.currentMajorVersion >= logicalReplicationSlotLagCheckMinVersion { + laggingSlots, err := c.getLaggingLogicalReplicationSlots(podName, logicalReplicationSlotLagThreshold) + if err != nil { + return fmt.Errorf("could not check logical replication slot lag: %w", err) + } + if laggingSlots != "" { + c.logger.Errorf("skipping major version upgrade, logical replication slot lag exceeds %d bytes (slot_name|lag_bytes): %s", logicalReplicationSlotLagThreshold, laggingSlots) + return nil + } + } + c.logger.Infof("healthy cluster ready to upgrade, current: %d desired: %d", c.currentMajorVersion, desiredVersion) if c.currentMajorVersion < desiredVersion { defer func() error { @@ -243,7 +257,6 @@ func (c *Cluster) majorVersionUpgrade() error { return fmt.Errorf("failed to assign critical-operation label: %s", err) } - podName := &spec.NamespacedName{Namespace: masterPod.Namespace, Name: masterPod.Name} c.logger.Infof("triggering major version upgrade on pod %s of %d pods", masterPod.Name, numberOfPods) c.eventRecorder.Eventf(c.GetReference(), v1.EventTypeNormal, "Major Version Upgrade", "starting major version upgrade on pod %s of %d pods", masterPod.Name, numberOfPods) upgradeCommand := fmt.Sprintf("set -o pipefail && /usr/bin/python3 /scripts/inplace_upgrade.py %d 2>&1 | tee last_upgrade.log", numberOfPods)