Skip to content

Bug 2086519: Introduce Pod Security Admission Label Synchronization controller #75

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jun 3, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@ go 1.18
require (
github.com/davecgh/go-spew v1.1.1
github.com/go-bindata/go-bindata v3.1.2+incompatible
github.com/google/go-cmp v0.5.5
github.com/openshift/api v0.0.0-20220525145417-ee5b62754c68
github.com/openshift/build-machinery-go v0.0.0-20211213093930-7e33a7eb4ce3
github.com/openshift/client-go v0.0.0-20220525160904-9e1acff93e4a
github.com/openshift/library-go v0.0.0-20220525173854-9b950a41acdc
github.com/prometheus/client_golang v1.12.1
github.com/spf13/cobra v1.4.0
github.com/stretchr/testify v1.7.0
k8s.io/api v0.24.0
k8s.io/apimachinery v0.24.0
k8s.io/apiserver v0.24.0
Expand All @@ -19,6 +21,7 @@ require (
k8s.io/controller-manager v0.24.0
k8s.io/klog/v2 v2.60.1
k8s.io/kubernetes v1.24.0
k8s.io/pod-security-admission v0.0.0
)

require (
Expand All @@ -45,7 +48,6 @@ require (
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/gnostic v0.5.7-v3refs // indirect
github.com/google/go-cmp v0.5.5 // indirect
github.com/google/gofuzz v1.1.0 // indirect
github.com/google/uuid v1.1.2 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
Expand All @@ -62,6 +64,7 @@ require (
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pkg/profile v1.3.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
Expand Down
1 change: 1 addition & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1219,6 +1219,7 @@ k8s.io/kubernetes v1.24.0/go.mod h1:8e8maMiZzBR2/8Po5Uulx+MXZUYJuN3vtKwD4Ct1Xi0=
k8s.io/legacy-cloud-providers v0.24.0/go.mod h1:j2gujMUYBEtbYfJaL8JUOgInzERm9fxJwEaOkZcnEUk=
k8s.io/metrics v0.24.0/go.mod h1:jrLlFGdKl3X+szubOXPG0Lf2aVxuV3QJcbsgVRAM6fI=
k8s.io/mount-utils v0.24.0/go.mod h1:XrSqB3a2e8sq+aU+rlbcBtQ3EgcuDk5RP9ZsGxjoDrI=
k8s.io/pod-security-admission v0.24.0 h1:nTZtZPdJ5ZusFyuxGZxfGxQ5piuhJyxuG5YmVUWG/Gs=
k8s.io/pod-security-admission v0.24.0/go.mod h1:YBS4mAdoba2qMvLPE3S7eMIxGlqUf4amHH26jUUqXX4=
k8s.io/sample-apiserver v0.24.0/go.mod h1:6YGSatoHMHIac/2dTtARwYH8PVWY5qq1L9ZYbxZ9lHY=
k8s.io/system-validators v1.7.0/go.mod h1:gP1Ky+R9wtrSiFbrpEPwWMeYz9yqyy1S/KOh0Vci7WI=
Expand Down
10 changes: 6 additions & 4 deletions pkg/cmd/controller/config.go
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
package controller

var ControllerInitializers = map[string]InitFunc{
"openshift.io/namespace-security-allocation": RunNamespaceSecurityAllocationController,
"openshift.io/resourcequota": RunResourceQuotaManager,
"openshift.io/cluster-quota-reconciliation": RunClusterQuotaReconciliationController,
"openshift.io/cluster-csr-approver": RunCSRApproverController,
"openshift.io/namespace-security-allocation": RunNamespaceSecurityAllocationController,
"openshift.io/resourcequota": RunResourceQuotaManager,
"openshift.io/cluster-quota-reconciliation": RunClusterQuotaReconciliationController,
"openshift.io/cluster-csr-approver": RunCSRApproverController,
"openshift.io/podsecurity-admission-label-syncer": runPodSecurityAdmissionLabelSynchronizationController,
}

const (
infraClusterQuotaReconciliationControllerServiceAccountName = "cluster-quota-reconciliation-controller"
infraClusterCSRApproverControllerServiceAccountName = "cluster-csr-approver-controller"
infraNamespaceSecurityAllocationControllerServiceAccountName = "namespace-security-allocation-controller"
podSecurityAdmissionLabelSyncerControllerServiceAccountName = "podsecurity-admission-label-syncer-controller"
defaultOpenShiftInfraNamespace = "openshift-infra"
)
60 changes: 26 additions & 34 deletions pkg/cmd/controller/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,11 @@ import (

"k8s.io/klog/v2"

"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
cacheddiscovery "k8s.io/client-go/discovery/cached"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/restmapper"
"k8s.io/controller-manager/app"
"k8s.io/controller-manager/pkg/clientbuilder"

Expand All @@ -34,7 +30,9 @@ import (
quotainformer "github.com/openshift/client-go/quota/informers/externalversions"
routeclient "github.com/openshift/client-go/route/clientset/versioned"
routeinformer "github.com/openshift/client-go/route/informers/externalversions"
securityclient "github.com/openshift/client-go/securityinternal/clientset/versioned"
securityclient "github.com/openshift/client-go/security/clientset/versioned"
securityinformer "github.com/openshift/client-go/security/informers/externalversions"
securityinternalclient "github.com/openshift/client-go/securityinternal/clientset/versioned"
templateclient "github.com/openshift/client-go/template/clientset/versioned"
templateinformer "github.com/openshift/client-go/template/informers/externalversions"
"github.com/openshift/library-go/pkg/controller/controllercmd"
Expand Down Expand Up @@ -66,11 +64,6 @@ func NewControllerContext(
clientConfig.Burst = clientConfig.Burst/10 + 1
}

discoveryClient := cacheddiscovery.NewMemCacheClient(kubeClient.Discovery())
dynamicRestMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient)
dynamicRestMapper.Reset()
go wait.Until(dynamicRestMapper.Reset, 30*time.Second, ctx.Done())

appsClient, err := appsclient.NewForConfig(clientConfig)
if err != nil {
return nil, err
Expand Down Expand Up @@ -103,6 +96,10 @@ func NewControllerContext(
if err != nil {
return nil, err
}
securityClient, err := securityclient.NewForConfig(clientConfig)
if err != nil {
return nil, err
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

general nit: fmt.Errorf("error creating security client: %w", err)

so we have good stacktraces if things goe wonky.

}

openshiftControllerContext := &EnhancedControllerContext{
ControllerContext: controllerContext,
Expand All @@ -111,19 +108,17 @@ func NewControllerContext(
ClientBuilder: OpenshiftControllerClientBuilder{
ControllerClientBuilder: clientbuilder.NewDynamicClientBuilder(rest.AnonymousClientConfig(clientConfig), kubeClient.CoreV1(), defaultOpenShiftInfraNamespace),
},
KubernetesInformers: informers.NewSharedInformerFactory(kubeClient, defaultInformerResyncPeriod),
OpenshiftConfigKubernetesInformers: informers.NewSharedInformerFactoryWithOptions(kubeClient, defaultInformerResyncPeriod, informers.WithNamespace("openshift-config")),
ControllerManagerKubeInformers: informers.NewSharedInformerFactoryWithOptions(kubeClient, defaultInformerResyncPeriod, informers.WithNamespace("openshift-controller-manager")),
AppsInformers: appsinformer.NewSharedInformerFactory(appsClient, defaultInformerResyncPeriod),
BuildInformers: buildinformer.NewSharedInformerFactory(buildClient, defaultInformerResyncPeriod),
ConfigInformers: configinformer.NewSharedInformerFactory(configClient, defaultInformerResyncPeriod),
ImageInformers: imageinformer.NewSharedInformerFactory(imageClient, defaultInformerResyncPeriod),
OperatorInformers: operatorinformer.NewSharedInformerFactory(operatorClient, defaultInformerResyncPeriod),
QuotaInformers: quotainformer.NewSharedInformerFactory(quotaClient, defaultInformerResyncPeriod),
RouteInformers: routeinformer.NewSharedInformerFactory(routerClient, defaultInformerResyncPeriod),
TemplateInformers: templateinformer.NewSharedInformerFactory(templateClient, defaultInformerResyncPeriod),
InformersStarted: make(chan struct{}),
RestMapper: dynamicRestMapper,
KubernetesInformers: informers.NewSharedInformerFactory(kubeClient, defaultInformerResyncPeriod),
AppsInformers: appsinformer.NewSharedInformerFactory(appsClient, defaultInformerResyncPeriod),
BuildInformers: buildinformer.NewSharedInformerFactory(buildClient, defaultInformerResyncPeriod),
ConfigInformers: configinformer.NewSharedInformerFactory(configClient, defaultInformerResyncPeriod),
ImageInformers: imageinformer.NewSharedInformerFactory(imageClient, defaultInformerResyncPeriod),
OperatorInformers: operatorinformer.NewSharedInformerFactory(operatorClient, defaultInformerResyncPeriod),
QuotaInformers: quotainformer.NewSharedInformerFactory(quotaClient, defaultInformerResyncPeriod),
RouteInformers: routeinformer.NewSharedInformerFactory(routerClient, defaultInformerResyncPeriod),
SecurityInformers: securityinformer.NewSharedInformerFactory(securityClient, defaultInformerResyncPeriod),
TemplateInformers: templateinformer.NewSharedInformerFactory(templateClient, defaultInformerResyncPeriod),
InformersStarted: make(chan struct{}),
}
openshiftControllerContext.GenericResourceInformer = openshiftControllerContext.ToGenericInformer()

Expand Down Expand Up @@ -165,9 +160,7 @@ type EnhancedControllerContext struct {
// ClientBuilder will provide a client for this controller to use
ClientBuilder ControllerClientBuilder

KubernetesInformers informers.SharedInformerFactory
OpenshiftConfigKubernetesInformers informers.SharedInformerFactory
ControllerManagerKubeInformers informers.SharedInformerFactory
KubernetesInformers informers.SharedInformerFactory

TemplateInformers templateinformer.SharedInformerFactory
QuotaInformers quotainformer.SharedInformerFactory
Expand All @@ -178,9 +171,9 @@ type EnhancedControllerContext struct {
ConfigInformers configinformer.SharedInformerFactory
ImageInformers imageinformer.SharedInformerFactory
OperatorInformers operatorinformer.SharedInformerFactory
SecurityInformers securityinformer.SharedInformerFactory

GenericResourceInformer genericinformers.GenericResourceInformer
RestMapper meta.RESTMapper

informersStartedLock sync.Mutex
informersStartedClosed bool
Expand All @@ -191,13 +184,12 @@ type EnhancedControllerContext struct {

func (c *EnhancedControllerContext) StartInformers(stopCh <-chan struct{}) {
c.KubernetesInformers.Start(stopCh)
c.OpenshiftConfigKubernetesInformers.Start(stopCh)
c.ControllerManagerKubeInformers.Start(stopCh)

c.AppsInformers.Start(stopCh)
c.BuildInformers.Start(stopCh)
c.ConfigInformers.Start(stopCh)
c.ImageInformers.Start(stopCh)
c.SecurityInformers.Start(stopCh)

c.TemplateInformers.Start(stopCh)
c.QuotaInformers.Start(stopCh)
Expand Down Expand Up @@ -228,8 +220,8 @@ type ControllerClientBuilder interface {
OpenshiftConfigClient(name string) (configclient.Interface, error)
OpenshiftConfigClientOrDie(name string) configclient.Interface

OpenshiftSecurityClient(name string) (securityclient.Interface, error)
OpenshiftSecurityClientOrDie(name string) securityclient.Interface
OpenshiftSecurityClient(name string) (securityinternalclient.Interface, error)
OpenshiftSecurityClientOrDie(name string) securityinternalclient.Interface

// OpenShift clients based on generated internal clientsets
OpenshiftTemplateClient(name string) (templateclient.Interface, error)
Expand Down Expand Up @@ -399,15 +391,15 @@ func (b OpenshiftControllerClientBuilder) OpenshiftQuotaClientOrDie(name string)
return client
}

func (b OpenshiftControllerClientBuilder) OpenshiftSecurityClient(name string) (securityclient.Interface, error) {
func (b OpenshiftControllerClientBuilder) OpenshiftSecurityClient(name string) (securityinternalclient.Interface, error) {
clientConfig, err := b.Config(name)
if err != nil {
return nil, err
}
return securityclient.NewForConfig(nonProtobufConfig(clientConfig))
return securityinternalclient.NewForConfig(nonProtobufConfig(clientConfig))
}

func (b OpenshiftControllerClientBuilder) OpenshiftSecurityClientOrDie(name string) securityclient.Interface {
func (b OpenshiftControllerClientBuilder) OpenshiftSecurityClientOrDie(name string) securityinternalclient.Interface {
client, err := b.OpenshiftSecurityClient(name)
if err != nil {
klog.Fatal(err)
Expand Down
31 changes: 31 additions & 0 deletions pkg/cmd/controller/psalabelsyncer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package controller

import (
"context"

"github.com/openshift/cluster-policy-controller/pkg/psalabelsyncer"
)

func runPodSecurityAdmissionLabelSynchronizationController(ctx context.Context, controllerCtx *EnhancedControllerContext) (bool, error) {

kubeClient, err := controllerCtx.ClientBuilder.Client(podSecurityAdmissionLabelSyncerControllerServiceAccountName)
if err != nil {
return true, err
}

controller, err := psalabelsyncer.NewPodSecurityAdmissionLabelSynchronizationController(
kubeClient.CoreV1().Namespaces(),
controllerCtx.KubernetesInformers.Core().V1().Namespaces(),
controllerCtx.KubernetesInformers.Rbac().V1(),
controllerCtx.KubernetesInformers.Core().V1().ServiceAccounts(),
controllerCtx.SecurityInformers.Security().V1().SecurityContextConstraints(),
controllerCtx.EventRecorder.ForComponent("podsecurity-admission-label-sync-controller"),
)

if err != nil {
return true, err
}

go controller.Run(ctx, 1)
return true, nil
}
Loading