Skip to content

Commit 19ec661

Browse files
authored
Watch and reconcile operator-controller ConfigMaps (#2062)
* Watch and reconcile operator-controlled ConfigMaps If users apply configmap changes directly, the operator wasn't reverting those changes instantly; with this PR, it will watch applied ConfigMaps and apply `KnativeServing` and `KnativeEventing` configurations. Signed-off-by: Pierangelo Di Pilato <pierdipi@redhat.com> * Add e2e test Signed-off-by: Pierangelo Di Pilato <pierdipi@redhat.com> --------- Signed-off-by: Pierangelo Di Pilato <pierdipi@redhat.com>
1 parent 5bef76f commit 19ec661

13 files changed

Lines changed: 268 additions & 30 deletions

File tree

cmd/operator/main.go

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,18 @@ package main
1919
import (
2020
"knative.dev/operator/pkg/reconciler/knativeeventing"
2121
"knative.dev/operator/pkg/reconciler/knativeserving"
22+
kubefilteredfactory "knative.dev/pkg/client/injection/kube/informers/factory/filtered"
2223
"knative.dev/pkg/injection/sharedmain"
24+
"knative.dev/pkg/signals"
2325
)
2426

2527
func main() {
26-
sharedmain.Main("knative-operator",
28+
ctx := signals.NewContext()
29+
ctx = kubefilteredfactory.WithSelectors(ctx,
30+
knativeserving.Selector,
31+
knativeeventing.Selector,
32+
)
33+
sharedmain.MainWithContext(ctx, "knative-operator",
2734
knativeserving.NewController,
2835
knativeeventing.NewController,
2936
)

pkg/reconciler/common/transformers.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,3 +83,16 @@ func InjectNamespace(manifest *mf.Manifest, instance base.KComponent, extra ...m
8383
*manifest = m
8484
return nil
8585
}
86+
87+
// InjectLabel adds the given key and value as label.
88+
func InjectLabel(key, value string) mf.Transformer {
89+
return func(u *unstructured.Unstructured) error {
90+
curr := u.GetLabels()
91+
if curr == nil {
92+
curr = map[string]string{}
93+
}
94+
curr[key] = value
95+
u.SetLabels(curr)
96+
return nil
97+
}
98+
}

pkg/reconciler/knativeeventing/controller.go

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,23 @@ import (
2828
knereconciler "knative.dev/operator/pkg/client/injection/reconciler/operator/v1beta1/knativeeventing"
2929
"knative.dev/operator/pkg/reconciler/common"
3030
kubeclient "knative.dev/pkg/client/injection/kube/client"
31-
deploymentinformer "knative.dev/pkg/client/injection/kube/informers/apps/v1/deployment"
31+
deploymentinformer "knative.dev/pkg/client/injection/kube/informers/apps/v1/deployment/filtered"
32+
configmapinformer "knative.dev/pkg/client/injection/kube/informers/core/v1/configmap/filtered"
3233
"knative.dev/pkg/configmap"
3334
"knative.dev/pkg/controller"
3435
"knative.dev/pkg/injection"
3536
"knative.dev/pkg/logging"
3637
)
3738

39+
const (
40+
// SelectorKey is the key of the selector for the KnativeEventing resources.
41+
SelectorKey = "app.kubernetes.io/name"
42+
// SelectorValue is the value of the selector for the KnativeEventing resources.
43+
SelectorValue = "knative-eventing"
44+
// Selector is the selector for the KnativeEventing resources.
45+
Selector = SelectorKey + "=" + SelectorValue
46+
)
47+
3848
// NewController initializes the controller and is called by the generated code
3949
// Registers eventhandlers to enqueue events
4050
func NewController(ctx context.Context, cmw configmap.Watcher) *controller.Impl {
@@ -45,7 +55,8 @@ func NewController(ctx context.Context, cmw configmap.Watcher) *controller.Impl
4555
func NewExtendedController(generator common.ExtensionGenerator) injection.ControllerConstructor {
4656
return func(ctx context.Context, cmw configmap.Watcher) *controller.Impl {
4757
knativeEventingInformer := knativeEventinginformer.Get(ctx)
48-
deploymentInformer := deploymentinformer.Get(ctx)
58+
deploymentInformer := deploymentinformer.Get(ctx, Selector)
59+
configMapInformer := configmapinformer.Get(ctx, Selector)
4960
kubeClient := kubeclient.Get(ctx)
5061
logger := logging.FromContext(ctx)
5162

@@ -72,6 +83,10 @@ func NewExtendedController(generator common.ExtensionGenerator) injection.Contro
7283
FilterFunc: controller.FilterControllerGVK(v1beta1.SchemeGroupVersion.WithKind("KnativeEventing")),
7384
Handler: controller.HandleAll(impl.EnqueueControllerOf),
7485
})
86+
configMapInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{
87+
FilterFunc: controller.FilterControllerGVK(v1beta1.SchemeGroupVersion.WithKind("KnativeEventing")),
88+
Handler: controller.HandleAll(impl.EnqueueControllerOf),
89+
})
7590

7691
return impl
7792
}

pkg/reconciler/knativeeventing/knativeeventing.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,8 @@ func (r *Reconciler) transform(ctx context.Context, manifest *mf.Manifest, comp
160160
kec.DefaultBrokerConfigMapTransform(instance, logger),
161161
kec.SinkBindingSelectionModeTransform(instance, logger),
162162
kec.ReplicasEnvVarsTransform(manifest.Client),
163+
// Ensure all resources have the selector applied so that the controller re-queues applied resources when they change.
164+
common.InjectLabel(SelectorKey, SelectorValue),
163165
}
164166
extra = append(extra, r.extension.Transformers(instance)...)
165167
return common.Transform(ctx, manifest, instance, extra...)

pkg/reconciler/knativeserving/controller.go

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@ import (
2828
knsreconciler "knative.dev/operator/pkg/client/injection/reconciler/operator/v1beta1/knativeserving"
2929
"knative.dev/operator/pkg/reconciler/common"
3030
kubeclient "knative.dev/pkg/client/injection/kube/client"
31-
deploymentinformer "knative.dev/pkg/client/injection/kube/informers/apps/v1/deployment"
31+
deploymentinformer "knative.dev/pkg/client/injection/kube/informers/apps/v1/deployment/filtered"
32+
configmapinformer "knative.dev/pkg/client/injection/kube/informers/core/v1/configmap/filtered"
3233
"knative.dev/pkg/configmap"
3334
"knative.dev/pkg/controller"
3435
"knative.dev/pkg/injection"
@@ -41,11 +42,21 @@ func NewController(ctx context.Context, cmw configmap.Watcher) *controller.Impl
4142
return NewExtendedController(common.NoExtension)(ctx, cmw)
4243
}
4344

45+
const (
46+
// SelectorKey is the key of the selector for the KnativeServing resources.
47+
SelectorKey = "app.kubernetes.io/name"
48+
// SelectorValue is the value of the selector for the KnativeServing resources.
49+
SelectorValue = "knative-serving"
50+
// Selector is the selector for the KnativeServing resources.
51+
Selector = SelectorKey + "=" + SelectorValue
52+
)
53+
4454
// NewExtendedController returns a controller extended to a specific platform
4555
func NewExtendedController(generator common.ExtensionGenerator) injection.ControllerConstructor {
4656
return func(ctx context.Context, cmw configmap.Watcher) *controller.Impl {
4757
knativeServingInformer := knativeServinginformer.Get(ctx)
48-
deploymentInformer := deploymentinformer.Get(ctx)
58+
deploymentInformer := deploymentinformer.Get(ctx, Selector)
59+
configMapInformer := configmapinformer.Get(ctx, Selector)
4960
kubeClient := kubeclient.Get(ctx)
5061
logger := logging.FromContext(ctx)
5162

@@ -72,6 +83,10 @@ func NewExtendedController(generator common.ExtensionGenerator) injection.Contro
7283
FilterFunc: controller.FilterControllerGVK(v1beta1.SchemeGroupVersion.WithKind("KnativeServing")),
7384
Handler: controller.HandleAll(impl.EnqueueControllerOf),
7485
})
86+
configMapInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{
87+
FilterFunc: controller.FilterControllerGVK(v1beta1.SchemeGroupVersion.WithKind("KnativeServing")),
88+
Handler: controller.HandleAll(impl.EnqueueControllerOf),
89+
})
7590

7691
return impl
7792
}

pkg/reconciler/knativeserving/knativeserving.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,8 @@ func (r *Reconciler) transform(ctx context.Context, manifest *mf.Manifest, comp
142142
extra := []mf.Transformer{
143143
ksc.CustomCertsTransform(instance, logger),
144144
ksc.AggregationRuleTransform(manifest.Client),
145+
// Ensure all resources have the selector applied so that the controller re-queues applied resources when they change.
146+
common.InjectLabel(SelectorKey, SelectorValue),
145147
}
146148
extra = append(extra, r.extension.Transformers(instance)...)
147149
extra = append(extra, ingress.Transformers(ctx, instance)...)

test/e2e/knativeeventingdeployment_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ func TestKnativeEventingDeployment(t *testing.T) {
5353
t.Run("restore", func(t *testing.T) {
5454
resources.AssertKEOperatorCRReadyStatus(t, clients, names)
5555
resources.DeleteAndVerifyEventingDeployments(t, clients, names)
56+
resources.DeleteAndVerifyEventingConfigMaps(t, clients, names)
5657
})
5758

5859
// Delete the KnativeEventing to see if all resources will be removed

test/e2e/knativeservingdeployment_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ func TestKnativeServingDeployment(t *testing.T) {
5858
t.Run("restore", func(t *testing.T) {
5959
resources.AssertKSOperatorCRReadyStatus(t, clients, names)
6060
resources.DeleteAndVerifyDeployments(t, clients, names)
61+
resources.DeleteAndVerifyConfigMaps(t, clients, names)
6162
})
6263

6364
// Delete the KnativeServing to see if all resources will be removed

test/resources/verify.go

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,47 @@ func DeleteAndVerifyDeployments(t *testing.T, clients *test.Clients, names test.
211211
t.Logf("The deployment %s/%s reached the desired state.", deployment.Namespace, deployment.Name)
212212
}
213213

214+
// DeleteAndVerifyConfigMaps verify whether all the ConfigMaps for knative serving are able to recreate, when they are deleted.
215+
func DeleteAndVerifyConfigMaps(t *testing.T, clients *test.Clients, names test.ResourceNames) {
216+
cmList, err := clients.KubeClient.CoreV1().ConfigMaps(names.Namespace).List(context.TODO(), metav1.ListOptions{})
217+
if err != nil {
218+
t.Fatalf("Failed to get any ConfigMap under the namespace %q: %v",
219+
test.ServingOperatorNamespace, err)
220+
}
221+
if len(cmList.Items) == 0 {
222+
t.Fatalf("No ConfigMap under the namespace %q was found",
223+
test.ServingOperatorNamespace)
224+
}
225+
// Delete the first ConfigMap and verify the operator recreates it
226+
configMap := cmList.Items[0]
227+
if err := clients.KubeClient.CoreV1().ConfigMaps(configMap.Namespace).Delete(context.TODO(), configMap.Name, metav1.DeleteOptions{}); err != nil {
228+
t.Fatalf("Failed to delete ConfigMap %s/%s: %v", configMap.Namespace, configMap.Name, err)
229+
}
230+
231+
waitErr := wait.PollUntilContextTimeout(context.TODO(), Interval, Timeout, true, func(context.Context) (bool, error) {
232+
_, err := clients.KubeClient.CoreV1().ConfigMaps(configMap.Namespace).Get(context.TODO(), configMap.Name, metav1.GetOptions{})
233+
if err != nil {
234+
// If the ConfigMap is not found, we continue to wait for the availability.
235+
if apierrs.IsNotFound(err) {
236+
return false, nil
237+
}
238+
return false, err
239+
}
240+
// For ConfigMaps, we consider it recreated as soon as it reappears.
241+
return true, nil
242+
})
243+
244+
if waitErr != nil {
245+
t.Fatalf("The ConfigMap %s/%s failed to be recreated: %v", configMap.Namespace, configMap.Name, err)
246+
}
247+
248+
if _, err := WaitForKnativeServingState(clients.KnativeServing(), test.OperatorName,
249+
IsKnativeServingReady); err != nil {
250+
t.Fatalf("KnativeService %q failed to reach the desired state: %v", test.OperatorName, err)
251+
}
252+
t.Logf("The ConfigMap %s/%s was successfully recreated.", configMap.Namespace, configMap.Name)
253+
}
254+
214255
// KSOperatorCRDelete deletes tha KnativeServing to see if all resources will be deleted
215256
func KSOperatorCRDelete(t *testing.T, clients *test.Clients, names test.ResourceNames) {
216257
if err := clients.KnativeServing().Delete(context.TODO(), names.KnativeServing, metav1.DeleteOptions{}); err != nil {
@@ -399,6 +440,46 @@ func DeleteAndVerifyEventingDeployments(t *testing.T, clients *test.Clients, nam
399440
t.Logf("The deployment %s/%s reached the desired state.", deployment.Namespace, deployment.Name)
400441
}
401442

443+
// DeleteAndVerifyEventingConfigMaps verify whether all the ConfigMaps for knative eventing are able to recreate, when they are deleted.
444+
func DeleteAndVerifyEventingConfigMaps(t *testing.T, clients *test.Clients, names test.ResourceNames) {
445+
cmList, err := clients.KubeClient.CoreV1().ConfigMaps(names.Namespace).List(context.TODO(), metav1.ListOptions{})
446+
if err != nil {
447+
t.Fatalf("Failed to get any ConfigMap under the namespace %q: %v",
448+
test.EventingOperatorNamespace, err)
449+
}
450+
if len(cmList.Items) == 0 {
451+
t.Fatalf("No ConfigMap under the namespace %q was found",
452+
test.EventingOperatorNamespace)
453+
}
454+
// Delete the first ConfigMap and verify the operator recreates it
455+
configMap := cmList.Items[0]
456+
if err := clients.KubeClient.CoreV1().ConfigMaps(configMap.Namespace).Delete(context.TODO(), configMap.Name, metav1.DeleteOptions{}); err != nil {
457+
t.Fatalf("Failed to delete ConfigMap %s/%s: %v", configMap.Namespace, configMap.Name, err)
458+
}
459+
460+
waitErr := wait.PollUntilContextTimeout(context.TODO(), Interval, Timeout, true, func(context.Context) (bool, error) {
461+
_, err := clients.KubeClient.CoreV1().ConfigMaps(configMap.Namespace).Get(context.TODO(), configMap.Name, metav1.GetOptions{})
462+
if err != nil {
463+
// If the ConfigMap is not found, we continue to wait for the availability.
464+
if apierrs.IsNotFound(err) {
465+
return false, nil
466+
}
467+
return false, err
468+
}
469+
return true, nil
470+
})
471+
472+
if waitErr != nil {
473+
t.Fatalf("The ConfigMap %s/%s failed to reach the desired state: %v", configMap.Namespace, configMap.Name, err)
474+
}
475+
476+
if _, err := WaitForKnativeEventingState(clients.KnativeEventing(), test.OperatorName,
477+
IsKnativeEventingReady); err != nil {
478+
t.Fatalf("KnativeService %q failed to reach the desired state: %v", test.OperatorName, err)
479+
}
480+
t.Logf("The ConfigMap %s/%s reached the desired state.", configMap.Namespace, configMap.Name)
481+
}
482+
402483
// KEOperatorCRDelete deletes tha KnativeEventing to see if all resources will be deleted
403484
func KEOperatorCRDelete(t *testing.T, clients *test.Clients, names test.ResourceNames) {
404485
if err := clients.KnativeEventing().Delete(context.TODO(), names.KnativeEventing, metav1.DeleteOptions{}); err != nil {

vendor/knative.dev/pkg/client/injection/kube/informers/apps/v1/deployment/deployment.go renamed to vendor/knative.dev/pkg/client/injection/kube/informers/apps/v1/deployment/filtered/deployment.go

Lines changed: 25 additions & 12 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)