Skip to content

Commit 99db2dc

Browse files
tmshortclaude
andcommitted
feat(experimental): run catalogd and operator-controller with 2 replicas
The experimental e2e suite uses a 2-node kind cluster, making it a natural fit to validate HA behaviour. Set replicas=2 for both components in helm/experimental.yaml so the experimental and experimental-e2e manifests exercise the multi-replica path end-to-end. This is safe for operator-controller (no leader-only HTTP servers) and for catalogd now that the catalog server starts on all pods via NeedLeaderElection=false, preventing the rolling-update deadlock that would arise if the server were leader-only. Also adds a @CatalogdHA experimental e2e scenario that force-deletes the catalogd leader pod and verifies that a new leader is elected and the catalog resumes serving. The scenario is gated on a 2-node cluster (detected in BeforeSuite and reflected in the featureGates map), so it is automatically skipped in the standard 1-node e2e suite. The experimental e2e timeout is bumped from 20m to 25m to accommodate leader re-election time (~163s worst case). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> Signed-off-by: Todd Short <tshort@redhat.com>
1 parent a375d74 commit 99db2dc

9 files changed

Lines changed: 115 additions & 14 deletions

File tree

Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -316,7 +316,7 @@ test-experimental-e2e: COVERAGE_NAME := experimental-e2e
316316
test-experimental-e2e: export MANIFEST := $(EXPERIMENTAL_RELEASE_MANIFEST)
317317
test-experimental-e2e: export INSTALL_DEFAULT_CATALOGS := false
318318
test-experimental-e2e: PROMETHEUS_VALUES := helm/prom_experimental.yaml
319-
test-experimental-e2e: E2E_TIMEOUT := 20m
319+
test-experimental-e2e: E2E_TIMEOUT := 25m
320320
test-experimental-e2e: run-internal prometheus e2e e2e-coverage kind-clean #HELP Run experimental e2e test suite on local kind cluster
321321

322322
.PHONY: prometheus

helm/experimental.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
# to pull in resources or additions
88
options:
99
operatorController:
10+
deployment:
11+
replicas: 2
1012
features:
1113
enabled:
1214
- SingleOwnNamespaceInstallSupport
@@ -20,6 +22,8 @@ options:
2022
# Use with {{- if has "FeatureGate" .Values.options.catalogd.features.enabled }}
2123
# to pull in resources or additions
2224
catalogd:
25+
deployment:
26+
replicas: 2
2327
features:
2428
enabled:
2529
- APIV1MetasHandler

internal/catalogd/serverutil/serverutil.go

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,10 @@ type CatalogServerConfig struct {
3030
}
3131

3232
// AddCatalogServerToManager adds the catalog HTTP server to the manager and registers
33-
// a readiness check that passes only when this pod is the leader and actively serving.
34-
// The listener is created lazily inside Start() so non-leader pods never bind the port,
35-
// which ensures the readiness check correctly excludes them from Service endpoints.
33+
// a readiness check that passes once the server has started serving. Because
34+
// NeedLeaderElection returns false, Start() is called on every pod immediately, so all
35+
// replicas bind the catalog port and become ready. Non-leader pods serve requests but
36+
// return 404 (empty local cache); callers are expected to retry.
3637
func AddCatalogServerToManager(mgr ctrl.Manager, cfg CatalogServerConfig, cw *certwatcher.CertWatcher) error {
3738
shutdownTimeout := 30 * time.Second
3839
r := &catalogServerRunnable{
@@ -52,11 +53,10 @@ func AddCatalogServerToManager(mgr ctrl.Manager, cfg CatalogServerConfig, cw *ce
5253
return fmt.Errorf("error adding catalog server to manager: %w", err)
5354
}
5455

55-
// Register a readiness check that passes only once Start() has been called (i.e.
56-
// this pod holds the leader lease and the catalog server is actively serving).
57-
// Non-leader pods never reach Start(), so they remain not-ready and are excluded
58-
// from Service endpoints — preventing catalog traffic from hitting a pod that
59-
// isn't serving the catalog port.
56+
// Register a readiness check that passes once Start() has been called and the
57+
// server is actively serving. All pods reach Start() (NeedLeaderElection=false),
58+
// so all replicas become ready and receive traffic; non-leaders return 404 until
59+
// they win the leader lease and populate their local cache.
6060
if err := mgr.AddReadyzCheck("catalog-server", r.readyzCheck()); err != nil {
6161
return fmt.Errorf("error adding catalog server readiness check: %w", err)
6262
}
@@ -112,7 +112,8 @@ func (r *catalogServerRunnable) Start(ctx context.Context) error {
112112
defer cancel()
113113
}
114114
if err := r.server.Shutdown(shutdownCtx); err != nil {
115-
// Shutdown errors are logged by the manager; nothing actionable here.
115+
// Shutdown errors (e.g. context deadline exceeded) are not actionable;
116+
// the process is terminating regardless.
116117
_ = err
117118
}
118119
}()

manifests/experimental-e2e.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2621,7 +2621,7 @@ metadata:
26212621
namespace: olmv1-system
26222622
spec:
26232623
minReadySeconds: 5
2624-
replicas: 1
2624+
replicas: 2
26252625
strategy:
26262626
type: RollingUpdate
26272627
rollingUpdate:
@@ -2772,7 +2772,7 @@ metadata:
27722772
name: operator-controller-controller-manager
27732773
namespace: olmv1-system
27742774
spec:
2775-
replicas: 1
2775+
replicas: 2
27762776
strategy:
27772777
type: RollingUpdate
27782778
rollingUpdate:

manifests/experimental.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2541,7 +2541,7 @@ metadata:
25412541
namespace: olmv1-system
25422542
spec:
25432543
minReadySeconds: 5
2544-
replicas: 1
2544+
replicas: 2
25452545
strategy:
25462546
type: RollingUpdate
25472547
rollingUpdate:
@@ -2679,7 +2679,7 @@ metadata:
26792679
name: operator-controller-controller-manager
26802680
namespace: olmv1-system
26812681
spec:
2682-
replicas: 1
2682+
replicas: 2
26832683
strategy:
26842684
type: RollingUpdate
26852685
rollingUpdate:

test/e2e/features/ha.feature

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
Feature: HA failover for catalogd
2+
3+
When catalogd is deployed with multiple replicas, the remaining pods must
4+
elect a new leader and resume serving catalogs if the leader pod is lost.
5+
6+
Background:
7+
Given OLM is available
8+
And an image registry is available
9+
10+
@CatalogdHA
11+
Scenario: Catalogd resumes serving catalogs after leader pod failure
12+
Given a catalog "test" with packages:
13+
| package | version | channel | replaces | contents |
14+
| test | 1.0.0 | stable | | CRD, Deployment, ConfigMap |
15+
And catalogd is ready to reconcile resources
16+
And catalog "test" is reconciled
17+
When the catalogd leader pod is force-deleted
18+
Then a new catalogd leader is elected
19+
And catalog "test" reports Serving as True with Reason Available

test/e2e/steps/ha_steps.go

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
package steps
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"strings"
7+
8+
"k8s.io/component-base/featuregate"
9+
)
10+
11+
// catalogdHAFeature gates scenarios that require a multi-node cluster.
12+
// It is set to true in BeforeSuite when the cluster has at least 2 nodes,
13+
// which is the case for the experimental e2e suite (kind-config-2node.yaml)
14+
// but not the standard suite.
15+
const catalogdHAFeature featuregate.Feature = "CatalogdHA"
16+
17+
// CatalogdLeaderPodIsForceDeleted force-deletes the catalogd leader pod to simulate leader loss.
18+
// The pod is identified from sc.leaderPods["catalogd"] (populated by a prior
19+
// "catalogd is ready to reconcile resources" step). Force-deletion is equivalent to
20+
// an abrupt process crash: the lease is no longer renewed and the surviving pod
21+
// acquires leadership after the lease expires.
22+
//
23+
// Note: stopping the kind node container is not used here because both nodes in the
24+
// experimental 2-node cluster are control-plane nodes that run etcd — stopping either
25+
// would break etcd quorum and make the API server unreachable for the rest of the test.
26+
func CatalogdLeaderPodIsForceDeleted(ctx context.Context) error {
27+
sc := scenarioCtx(ctx)
28+
leaderPod := sc.leaderPods["catalogd"]
29+
if leaderPod == "" {
30+
return fmt.Errorf("catalogd leader pod not found in scenario context; run 'catalogd is ready to reconcile resources' first")
31+
}
32+
33+
logger.Info("Force-deleting catalogd leader pod", "pod", leaderPod)
34+
if _, err := k8sClient("delete", "pod", leaderPod, "-n", olmNamespace,
35+
"--force", "--grace-period=0"); err != nil {
36+
return fmt.Errorf("failed to force-delete catalogd leader pod %q: %w", leaderPod, err)
37+
}
38+
return nil
39+
}
40+
41+
// NewCatalogdLeaderIsElected polls the catalogd leader election lease until the holder
42+
// identity changes to a pod other than the deleted leader. It updates
43+
// sc.leaderPods["catalogd"] with the new leader pod name.
44+
func NewCatalogdLeaderIsElected(ctx context.Context) error {
45+
sc := scenarioCtx(ctx)
46+
oldLeader := sc.leaderPods["catalogd"]
47+
48+
waitFor(ctx, func() bool {
49+
holder, err := k8sClient("get", "lease", leaseNames["catalogd"], "-n", olmNamespace,
50+
"-o", "jsonpath={.spec.holderIdentity}")
51+
if err != nil || holder == "" {
52+
return false
53+
}
54+
newPod := strings.Split(strings.TrimSpace(holder), "_")[0]
55+
if newPod == oldLeader {
56+
return false
57+
}
58+
sc.leaderPods["catalogd"] = newPod
59+
logger.Info("New catalogd leader elected", "pod", newPod)
60+
return true
61+
})
62+
return nil
63+
}

test/e2e/steps/hooks.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"os/exec"
99
"regexp"
1010
"strconv"
11+
"strings"
1112
"sync"
1213

1314
"github.com/cucumber/godog"
@@ -90,6 +91,7 @@ var (
9091
features.HelmChartSupport: false,
9192
features.BoxcutterRuntime: false,
9293
features.DeploymentConfig: false,
94+
catalogdHAFeature: false,
9395
}
9496
logger logr.Logger
9597
)
@@ -131,6 +133,14 @@ func BeforeSuite() {
131133
logger = textlogger.NewLogger(textlogger.NewConfig())
132134
}
133135

136+
// Enable HA scenarios when the cluster has at least 2 nodes. This runs
137+
// unconditionally so that upgrade scenarios (which install OLM in a Background
138+
// step and return early below) still get the gate set correctly.
139+
if out, err := k8sClient("get", "nodes", "--no-headers", "-o", "name"); err == nil &&
140+
len(strings.Fields(strings.TrimSpace(out))) >= 2 {
141+
featureGates[catalogdHAFeature] = true
142+
}
143+
134144
olm, err := detectOLMDeployment()
135145
if err != nil {
136146
logger.Info("OLM deployments not found; skipping feature gate detection (upgrade scenarios will install OLM in Background)")
@@ -152,6 +162,7 @@ func BeforeSuite() {
152162
}
153163
}
154164
}
165+
155166
logger.Info(fmt.Sprintf("Enabled feature gates: %v", featureGates))
156167
}
157168

test/e2e/steps/steps.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,9 @@ func RegisterSteps(sc *godog.ScenarioContext) {
194194
sc.Step(`^(?i)the "([^"]+)" component is configured with HTTPS_PROXY "([^"]+)"$`, ConfigureDeploymentWithHTTPSProxy)
195195
sc.Step(`^(?i)the "([^"]+)" component is configured with HTTPS_PROXY pointing to a recording proxy$`, StartRecordingProxyAndConfigureDeployment)
196196
sc.Step(`^(?i)the recording proxy received a CONNECT request for the catalogd service$`, RecordingProxyReceivedCONNECTForCatalogd)
197+
198+
sc.Step(`^(?i)the catalogd leader pod is force-deleted$`, CatalogdLeaderPodIsForceDeleted)
199+
sc.Step(`^(?i)a new catalogd leader is elected$`, NewCatalogdLeaderIsElected)
197200
}
198201

199202
func init() {

0 commit comments

Comments
 (0)