Skip to content

Commit e425d69

Browse files
soulebsouleb
committed
Enable remote dependencies from OCI repositories
If implemented, the source controller will be able to resolve charts dependencies from OCI repositories. The remote builder has been refactored as part of this work. Signed-off-by: Soule BA <soule@weave.works>
1 parent a68eaf7 commit e425d69

File tree

10 files changed

+538
-137
lines changed

10 files changed

+538
-137
lines changed

‎controllers/helmchart_controller.go

+80-27
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import (
3535
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3636
"k8s.io/apimachinery/pkg/runtime"
3737
"k8s.io/apimachinery/pkg/types"
38+
kerrors "k8s.io/apimachinery/pkg/util/errors"
3839
"k8s.io/apimachinery/pkg/util/uuid"
3940
kuberecorder "k8s.io/client-go/tools/record"
4041
ctrl "sigs.k8s.io/controller-runtime"
@@ -460,9 +461,10 @@ func (r *HelmChartReconciler) buildFromHelmRepository(ctx context.Context, obj *
460461
loginOpts []helmreg.LoginOption
461462
)
462463

464+
normalizedURL := strings.TrimSuffix(repo.Spec.URL, "/")
463465
// Construct the Getter options from the HelmRepository data
464466
clientOpts := []helmgetter.Option{
465-
helmgetter.WithURL(repo.Spec.URL),
467+
helmgetter.WithURL(normalizedURL),
466468
helmgetter.WithTimeout(repo.Spec.Timeout.Duration),
467469
helmgetter.WithPassCredentialsAll(repo.Spec.PassCredentials),
468470
}
@@ -490,7 +492,7 @@ func (r *HelmChartReconciler) buildFromHelmRepository(ctx context.Context, obj *
490492
}
491493
clientOpts = append(clientOpts, opts...)
492494

493-
tlsConfig, err = getter.TLSClientConfigFromSecret(*secret, repo.Spec.URL)
495+
tlsConfig, err = getter.TLSClientConfigFromSecret(*secret, normalizedURL)
494496
if err != nil {
495497
e := &serror.Event{
496498
Err: fmt.Errorf("failed to create TLS client config with secret data: %w", err),
@@ -502,7 +504,7 @@ func (r *HelmChartReconciler) buildFromHelmRepository(ctx context.Context, obj *
502504
}
503505

504506
// Build registryClient options from secret
505-
loginOpt, err := registry.LoginOptionFromSecret(repo.Spec.URL, *secret)
507+
loginOpt, err := registry.LoginOptionFromSecret(normalizedURL, *secret)
506508
if err != nil {
507509
e := &serror.Event{
508510
Err: fmt.Errorf("failed to configure Helm client with secret data: %w", err),
@@ -517,11 +519,11 @@ func (r *HelmChartReconciler) buildFromHelmRepository(ctx context.Context, obj *
517519
}
518520

519521
// Initialize the chart repository
520-
var chartRepo chart.Repository
522+
var chartRepo chart.Downloader
521523
switch repo.Spec.Type {
522524
case sourcev1.HelmRepositoryTypeOCI:
523-
if !helmreg.IsOCI(repo.Spec.URL) {
524-
err := fmt.Errorf("invalid OCI registry URL: %s", repo.Spec.URL)
525+
if !helmreg.IsOCI(normalizedURL) {
526+
err := fmt.Errorf("invalid OCI registry URL: %s", normalizedURL)
525527
return chartRepoConfigErrorReturn(err, obj)
526528
}
527529

@@ -550,7 +552,7 @@ func (r *HelmChartReconciler) buildFromHelmRepository(ctx context.Context, obj *
550552

551553
// Tell the chart repository to use the OCI client with the configured getter
552554
clientOpts = append(clientOpts, helmgetter.WithRegistryClient(registryClient))
553-
ociChartRepo, err := repository.NewOCIChartRepository(repo.Spec.URL, repository.WithOCIGetter(r.Getters), repository.WithOCIGetterOptions(clientOpts), repository.WithOCIRegistryClient(registryClient))
555+
ociChartRepo, err := repository.NewOCIChartRepository(normalizedURL, repository.WithOCIGetter(r.Getters), repository.WithOCIGetterOptions(clientOpts), repository.WithOCIRegistryClient(registryClient))
554556
if err != nil {
555557
return chartRepoConfigErrorReturn(err, obj)
556558
}
@@ -570,7 +572,7 @@ func (r *HelmChartReconciler) buildFromHelmRepository(ctx context.Context, obj *
570572
}
571573
}
572574
default:
573-
httpChartRepo, err := repository.NewChartRepository(repo.Spec.URL, r.Storage.LocalPath(*repo.GetArtifact()), r.Getters, tlsConfig, clientOpts,
575+
httpChartRepo, err := repository.NewChartRepository(normalizedURL, r.Storage.LocalPath(*repo.GetArtifact()), r.Getters, tlsConfig, clientOpts,
574576
repository.WithMemoryCache(r.Storage.LocalPath(*repo.GetArtifact()), r.Cache, r.TTL, func(event string) {
575577
r.IncCacheEvents(event, obj.Name, obj.Namespace)
576578
}))
@@ -683,7 +685,7 @@ func (r *HelmChartReconciler) buildFromTarballArtifact(ctx context.Context, obj
683685

684686
// Setup dependency manager
685687
dm := chart.NewDependencyManager(
686-
chart.WithRepositoryCallback(r.namespacedChartRepositoryCallback(ctx, obj.GetName(), obj.GetNamespace())),
688+
chart.WithDownloaderCallback(r.namespacedChartRepositoryCallback(ctx, obj.GetName(), obj.GetNamespace())),
687689
)
688690
defer dm.Clear()
689691

@@ -912,12 +914,17 @@ func (r *HelmChartReconciler) garbageCollect(ctx context.Context, obj *sourcev1.
912914
return nil
913915
}
914916

915-
// namespacedChartRepositoryCallback returns a chart.GetChartRepositoryCallback scoped to the given namespace.
917+
// namespacedChartRepositoryCallback returns a chart.GetChartDownloaderCallback scoped to the given namespace.
916918
// The returned callback returns a repository.ChartRepository configured with the retrieved v1beta1.HelmRepository,
917919
// or a shim with defaults if no object could be found.
918-
func (r *HelmChartReconciler) namespacedChartRepositoryCallback(ctx context.Context, name, namespace string) chart.GetChartRepositoryCallback {
919-
return func(url string) (*repository.ChartRepository, error) {
920-
var tlsConfig *tls.Config
920+
// The callback returns an object with a state, so the caller has to do the necessary cleanup.
921+
func (r *HelmChartReconciler) namespacedChartRepositoryCallback(ctx context.Context, name, namespace string) chart.GetChartDownloaderCallback {
922+
return func(url string) (chart.CleanDownloader, error) {
923+
var (
924+
tlsConfig *tls.Config
925+
loginOpts []helmreg.LoginOption
926+
)
927+
normalizedURL := strings.TrimSuffix(url, "/")
921928
repo, err := r.resolveDependencyRepository(ctx, url, namespace)
922929
if err != nil {
923930
// Return Kubernetes client errors, but ignore others
@@ -932,7 +939,7 @@ func (r *HelmChartReconciler) namespacedChartRepositoryCallback(ctx context.Cont
932939
}
933940
}
934941
clientOpts := []helmgetter.Option{
935-
helmgetter.WithURL(repo.Spec.URL),
942+
helmgetter.WithURL(normalizedURL),
936943
helmgetter.WithTimeout(repo.Spec.Timeout.Duration),
937944
helmgetter.WithPassCredentialsAll(repo.Spec.PassCredentials),
938945
}
@@ -946,26 +953,72 @@ func (r *HelmChartReconciler) namespacedChartRepositoryCallback(ctx context.Cont
946953
}
947954
clientOpts = append(clientOpts, opts...)
948955

949-
tlsConfig, err = getter.TLSClientConfigFromSecret(*secret, repo.Spec.URL)
956+
tlsConfig, err = getter.TLSClientConfigFromSecret(*secret, normalizedURL)
950957
if err != nil {
951958
return nil, fmt.Errorf("failed to create TLS client config for HelmRepository '%s': %w", repo.Name, err)
952959
}
953-
}
954960

955-
chartRepo, err := repository.NewChartRepository(repo.Spec.URL, "", r.Getters, tlsConfig, clientOpts)
956-
if err != nil {
957-
return nil, err
961+
// Build registryClient options from secret
962+
loginOpt, err := registry.LoginOptionFromSecret(normalizedURL, *secret)
963+
if err != nil {
964+
return nil, fmt.Errorf("failed to create login options for HelmRepository '%s': %w", repo.Name, err)
965+
}
966+
967+
loginOpts = append([]helmreg.LoginOption{}, loginOpt)
958968
}
959969

960-
// Ensure that the cache key is the same as the artifact path
961-
// otherwise don't enable caching. We don't want to cache indexes
962-
// for repositories that are not reconciled by the source controller.
963-
if repo.Status.Artifact != nil {
964-
chartRepo.CachePath = r.Storage.LocalPath(*repo.GetArtifact())
965-
chartRepo.SetMemCache(r.Storage.LocalPath(*repo.GetArtifact()), r.Cache, r.TTL, func(event string) {
966-
r.IncCacheEvents(event, name, namespace)
967-
})
970+
var chartRepo chart.CleanDownloader
971+
if helmreg.IsOCI(normalizedURL) {
972+
registryClient, file, err := r.RegistryClientGenerator(loginOpts != nil)
973+
if err != nil {
974+
return nil, fmt.Errorf("failed to create registry client for HelmRepository '%s': %w", repo.Name, err)
975+
}
976+
977+
var errs []error
978+
// Tell the chart repository to use the OCI client with the configured getter
979+
clientOpts = append(clientOpts, helmgetter.WithRegistryClient(registryClient))
980+
ociChartRepo, err := repository.NewOCIChartRepository(normalizedURL, repository.WithOCIGetter(r.Getters),
981+
repository.WithOCIGetterOptions(clientOpts),
982+
repository.WithOCIRegistryClient(registryClient),
983+
repository.WithCredentialFile(file))
984+
if err != nil {
985+
// clean up the file
986+
if err := os.Remove(file); err != nil {
987+
errs = append(errs, err)
988+
}
989+
errs = append(errs, fmt.Errorf("failed to create OCI chart repository for HelmRepository '%s': %w", repo.Name, err))
990+
return nil, kerrors.NewAggregate(errs)
991+
}
992+
993+
// If login options are configured, use them to login to the registry
994+
// The OCIGetter will later retrieve the stored credentials to pull the chart
995+
if loginOpts != nil {
996+
err = ociChartRepo.Login(loginOpts...)
997+
if err != nil {
998+
return nil, fmt.Errorf("failed to login to OCI chart repository for HelmRepository '%s': %w", repo.Name, err)
999+
}
1000+
}
1001+
1002+
chartRepo = ociChartRepo
1003+
} else {
1004+
httpChartRepo, err := repository.NewChartRepository(normalizedURL, "", r.Getters, tlsConfig, clientOpts)
1005+
if err != nil {
1006+
return nil, err
1007+
}
1008+
1009+
// Ensure that the cache key is the same as the artifact path
1010+
// otherwise don't enable caching. We don't want to cache indexes
1011+
// for repositories that are not reconciled by the source controller.
1012+
if repo.Status.Artifact != nil {
1013+
httpChartRepo.CachePath = r.Storage.LocalPath(*repo.GetArtifact())
1014+
httpChartRepo.SetMemCache(r.Storage.LocalPath(*repo.GetArtifact()), r.Cache, r.TTL, func(event string) {
1015+
r.IncCacheEvents(event, name, namespace)
1016+
})
1017+
}
1018+
1019+
chartRepo = httpChartRepo
9681020
}
1021+
9691022
return chartRepo, nil
9701023
}
9711024
}

‎controllers/helmchart_controller_test.go

+6-5
Original file line numberDiff line numberDiff line change
@@ -1070,7 +1070,7 @@ func TestHelmChartReconciler_buildFromTarballArtifact(t *testing.T) {
10701070
assertFunc: func(g *WithT, build chart.Build) {
10711071
g.Expect(build.Name).To(Equal("helmchartwithdeps"))
10721072
g.Expect(build.Version).To(Equal("0.1.0"))
1073-
g.Expect(build.ResolvedDependencies).To(Equal(3))
1073+
g.Expect(build.ResolvedDependencies).To(Equal(4))
10741074
g.Expect(build.Path).To(BeARegularFile())
10751075
},
10761076
cleanFunc: func(g *WithT, build *chart.Build) {
@@ -1178,10 +1178,11 @@ func TestHelmChartReconciler_buildFromTarballArtifact(t *testing.T) {
11781178
g := NewWithT(t)
11791179

11801180
r := &HelmChartReconciler{
1181-
Client: fake.NewClientBuilder().Build(),
1182-
EventRecorder: record.NewFakeRecorder(32),
1183-
Storage: storage,
1184-
Getters: testGetters,
1181+
Client: fake.NewClientBuilder().Build(),
1182+
EventRecorder: record.NewFakeRecorder(32),
1183+
Storage: storage,
1184+
Getters: testGetters,
1185+
RegistryClientGenerator: registry.ClientGenerator,
11851186
}
11861187

11871188
obj := &sourcev1.HelmChart{

‎controllers/testdata/charts/helmchartwithdeps/Chart.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,6 @@ dependencies:
3131
- name: grafana
3232
version: ">=5.7.0"
3333
repository: "https://grafana.github.io/helm-charts"
34+
- name: podinfo
35+
version: ">=6.1.*"
36+
repository: "oci://ghcr.io/stefanprodan/charts"

‎internal/helm/chart/builder_local_test.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ func TestLocalBuilder_Build(t *testing.T) {
6767
reference Reference
6868
buildOpts BuildOptions
6969
valuesFiles []helmchart.File
70-
repositories map[string]*repository.ChartRepository
70+
repositories map[string]CleanDownloader
7171
dependentChartPaths []string
7272
wantValues chartutil.Values
7373
wantVersion string
@@ -146,7 +146,7 @@ fullnameOverride: "full-foo-name-override"`),
146146
{
147147
name: "chart with dependencies",
148148
reference: LocalReference{Path: "../testdata/charts/helmchartwithdeps"},
149-
repositories: map[string]*repository.ChartRepository{
149+
repositories: map[string]CleanDownloader{
150150
"https://grafana.github.io/helm-charts/": mockRepo(),
151151
},
152152
dependentChartPaths: []string{"./../testdata/charts/helmchart"},
@@ -165,7 +165,7 @@ fullnameOverride: "full-foo-name-override"`),
165165
{
166166
name: "v1 chart with dependencies",
167167
reference: LocalReference{Path: "../testdata/charts/helmchartwithdeps-v1"},
168-
repositories: map[string]*repository.ChartRepository{
168+
repositories: map[string]CleanDownloader{
169169
"https://grafana.github.io/helm-charts/": mockRepo(),
170170
},
171171
dependentChartPaths: []string{"../testdata/charts/helmchart-v1"},

‎internal/helm/chart/builder_remote.go

+9-9
Original file line numberDiff line numberDiff line change
@@ -36,22 +36,22 @@ import (
3636
"github.com/fluxcd/source-controller/internal/helm/chart/secureloader"
3737
)
3838

39-
// Repository is a repository.ChartRepository or a repository.OCIChartRepository.
40-
// It is used to download a chart from a remote Helm repository or OCI registry.
41-
type Repository interface {
42-
// GetChartVersion returns the repo.ChartVersion for the given name and version.
39+
// Downloader is used to download a chart from a remote Helm repository or OCI registry.
40+
type Downloader interface {
41+
// GetChartVersion returns the repo.ChartVersion for the given name and version
42+
// from the remote repository.ChartRepository.
4343
GetChartVersion(name, version string) (*repo.ChartVersion, error)
44-
// GetChartVersion returns a chart.ChartVersion from the remote repository.
44+
// DownloadChart downloads a chart from the remote Helm repository or OCI registry.
4545
DownloadChart(chart *repo.ChartVersion) (*bytes.Buffer, error)
4646
}
4747

4848
type remoteChartBuilder struct {
49-
remote Repository
49+
remote Downloader
5050
}
5151

5252
// NewRemoteBuilder returns a Builder capable of building a Helm
53-
// chart with a RemoteReference in the given repository.ChartRepository.
54-
func NewRemoteBuilder(repository Repository) Builder {
53+
// chart with a RemoteReference in the given Downloader.
54+
func NewRemoteBuilder(repository Downloader) Builder {
5555
return &remoteChartBuilder{
5656
remote: repository,
5757
}
@@ -132,7 +132,7 @@ func (b *remoteChartBuilder) Build(_ context.Context, ref Reference, p string, o
132132
return result, nil
133133
}
134134

135-
func (b *remoteChartBuilder) downloadFromRepository(remote Repository, remoteRef RemoteReference, opts BuildOptions) (*bytes.Buffer, *Build, error) {
135+
func (b *remoteChartBuilder) downloadFromRepository(remote Downloader, remoteRef RemoteReference, opts BuildOptions) (*bytes.Buffer, *Build, error) {
136136
// Get the current version for the RemoteReference
137137
cv, err := remote.GetChartVersion(remoteRef.Name, remoteRef.Version)
138138
if err != nil {

0 commit comments

Comments
 (0)