Skip to content

Commit e66175d

Browse files
authored
Merge pull request #9 from weaveworks/repeat-templates
Repeat templates
2 parents 88b7e20 + be59f54 commit e66175d

File tree

7 files changed

+247
-68
lines changed

7 files changed

+247
-68
lines changed

api/v1alpha1/gitopsset_types.go

+8-2
Original file line numberDiff line numberDiff line change
@@ -8,23 +8,29 @@ import (
88

99
// // ©itOpsSetTemplate describes a resource to create
1010
type GitOpsSetTemplate struct {
11+
// Repeat is a JSONPath string defining that the template content should be
12+
// repeated for each of the matching elements in the JSONPath expression.
13+
// https://kubernetes.io/docs/reference/kubectl/jsonpath/
14+
Repeat string `json:"repeat,omitempty"`
15+
// Content is the YAML to be templated and generated.
1116
Content runtime.RawExtension `json:"content"`
1217
}
1318

1419
// ListGenerator generates from a hard-coded list.
1520
type ListGenerator struct {
16-
Elements []apiextensionsv1.JSON `json:"elements"`
21+
Elements []apiextensionsv1.JSON `json:"elements,omitempty"`
1722
}
1823

1924
// GitRepositoryGeneratorFileItemm defines a path to a file to be parsed when generating.
2025
type GitRepositoryGeneratorFileItem struct {
26+
// Path is the name of a file to read and generate from can be JSON or YAML.
2127
Path string `json:"path"`
2228
}
2329

2430
// GitRepositoryGenerator generates from files in a Flux GitRepository resource.
2531
type GitRepositoryGenerator struct {
2632
// RepositoryRef is the name of a GitRepository resource to be generated from.
27-
RepositoryRef string `json:"repositoryRef"`
33+
RepositoryRef string `json:"repositoryRef,omitempty"`
2834

2935
// Files is a set of rules for identifying files to be parsed.
3036
Files []GitRepositoryGeneratorFileItem `json:"files,omitempty"`

config/crd/bases/templates.weave.works_gitopssets.yaml

+8-4
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@ spec:
6363
path to a file to be parsed when generating.
6464
properties:
6565
path:
66+
description: Path is the name of a file to read and
67+
generate from can be JSON or YAML.
6668
type: string
6769
required:
6870
- path
@@ -72,8 +74,6 @@ spec:
7274
description: RepositoryRef is the name of a GitRepository
7375
resource to be generated from.
7476
type: string
75-
required:
76-
- repositoryRef
7777
type: object
7878
list:
7979
description: ListGenerator generates from a hard-coded list.
@@ -82,8 +82,6 @@ spec:
8282
items:
8383
x-kubernetes-preserve-unknown-fields: true
8484
type: array
85-
required:
86-
- elements
8785
type: object
8886
type: object
8987
type: array
@@ -98,8 +96,14 @@ spec:
9896
description: // ©itOpsSetTemplate describes a resource to create
9997
properties:
10098
content:
99+
description: Content is the YAML to be templated and generated.
101100
type: object
102101
x-kubernetes-preserve-unknown-fields: true
102+
repeat:
103+
description: Repeat is a JSONPath string defining that the template
104+
content should be repeated for each of the matching elements
105+
in the JSONPath expression. https://kubernetes.io/docs/reference/kubectl/jsonpath/
106+
type: string
103107
required:
104108
- content
105109
type: object

controllers/templates/renderer.go

+78-27
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"k8s.io/apimachinery/pkg/runtime"
1515
yamlserializer "k8s.io/apimachinery/pkg/runtime/serializer/yaml"
1616
"k8s.io/apimachinery/pkg/util/yaml"
17+
"k8s.io/client-go/util/jsonpath"
1718

1819
templatesv1 "github.com/weaveworks/gitopssets-controller/api/v1alpha1"
1920
"github.com/weaveworks/gitopssets-controller/controllers/templates/generators"
@@ -49,56 +50,106 @@ func Render(ctx context.Context, r *templatesv1.GitOpsSet, configuredGenerators
4950
return rendered, nil
5051
}
5152

52-
func renderTemplateParams(tmpl templatesv1.GitOpsSetTemplate, params map[string]any, ns string) ([]*unstructured.Unstructured, error) {
53-
rendered, err := render(tmpl.Content.Raw, params)
53+
func repeat(tmpl templatesv1.GitOpsSetTemplate, params map[string]any) ([]any, error) {
54+
if tmpl.Repeat == "" {
55+
return []any{
56+
map[string]any{
57+
"element": params,
58+
},
59+
}, nil
60+
}
61+
62+
jp := jsonpath.New("repeat")
63+
err := jp.Parse(tmpl.Repeat)
5464
if err != nil {
55-
return nil, err
65+
return nil, fmt.Errorf("failed to parse repeat on template %q: %w", tmpl.Repeat, err)
5666
}
5767

58-
// Technically multiple objects could be in the YAML...
59-
var objects []*unstructured.Unstructured
60-
decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(rendered), 100)
61-
for {
62-
var rawObj runtime.RawExtension
63-
if err := decoder.Decode(&rawObj); err != nil {
64-
if err != io.EOF {
65-
return nil, fmt.Errorf("failed to parse rendered template: %w", err)
68+
results, err := jp.FindResults(params)
69+
if err != nil {
70+
return nil, fmt.Errorf("failed to find results from expression %q: %w", tmpl.Repeat, err)
71+
}
72+
73+
repeated := []any{}
74+
for _, result := range results {
75+
for _, v := range result {
76+
slice, ok := v.Interface().([]any)
77+
if ok {
78+
repeated = append(repeated, slice...)
79+
} else {
80+
repeated = append(repeated, v)
6681
}
67-
break
6882
}
83+
}
84+
85+
elements := []any{}
86+
for _, v := range repeated {
87+
elements = append(elements, map[string]any{
88+
"element": params,
89+
"repeat": v,
90+
})
91+
}
92+
93+
return elements, nil
94+
}
95+
96+
func renderTemplateParams(tmpl templatesv1.GitOpsSetTemplate, params map[string]any, ns string) ([]*unstructured.Unstructured, error) {
97+
var objects []*unstructured.Unstructured
6998

70-
m, _, err := yamlserializer.NewDecodingSerializer(unstructured.UnstructuredJSONScheme).Decode(rawObj.Raw, nil, nil)
99+
repeatedParams, err := repeat(tmpl, params)
100+
if err != nil {
101+
return nil, err
102+
}
103+
104+
for _, p := range repeatedParams {
105+
rendered, err := render(tmpl.Content.Raw, p)
71106
if err != nil {
72-
return nil, fmt.Errorf("failed to decode rendered template: %w", err)
107+
return nil, err
73108
}
74109

75-
unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(m)
76-
if err != nil {
77-
return nil, fmt.Errorf("failed convert parsed template: %w", err)
110+
// Technically multiple objects could be in the YAML...
111+
decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(rendered), 100)
112+
for {
113+
var rawObj runtime.RawExtension
114+
if err := decoder.Decode(&rawObj); err != nil {
115+
if err != io.EOF {
116+
return nil, fmt.Errorf("failed to parse rendered template: %w", err)
117+
}
118+
break
119+
}
120+
121+
m, _, err := yamlserializer.NewDecodingSerializer(unstructured.UnstructuredJSONScheme).Decode(rawObj.Raw, nil, nil)
122+
if err != nil {
123+
return nil, fmt.Errorf("failed to decode rendered template: %w", err)
124+
}
125+
126+
unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(m)
127+
if err != nil {
128+
return nil, fmt.Errorf("failed convert parsed template: %w", err)
129+
}
130+
delete(unstructuredMap, "status")
131+
uns := &unstructured.Unstructured{Object: unstructuredMap}
132+
133+
if uns.GetKind() != "Namespace" {
134+
uns.SetNamespace(ns)
135+
}
136+
objects = append(objects, uns)
78137
}
79-
delete(unstructuredMap, "status")
80-
uns := &unstructured.Unstructured{Object: unstructuredMap}
81-
uns.SetNamespace(ns)
82-
objects = append(objects, uns)
83138
}
84139

85140
return objects, nil
86141
}
87142

88143
// TODO: pass the `GitOpsSet` through to here so that we can fix the
89144
// `template.New` to include the name/namespace.
90-
func render(b []byte, params map[string]any) ([]byte, error) {
145+
func render(b []byte, params any) ([]byte, error) {
91146
t, err := template.New("gitopsset-template").Funcs(templateFuncs).Parse(string(b))
92147
if err != nil {
93148
return nil, fmt.Errorf("failed to parse template: %w", err)
94149
}
95150

96-
data := map[string]any{
97-
"element": params,
98-
}
99-
100151
var out bytes.Buffer
101-
if err := t.Execute(&out, data); err != nil {
152+
if err := t.Execute(&out, params); err != nil {
102153
return nil, fmt.Errorf("failed to render template: %w", err)
103154
}
104155

controllers/templates/renderer_test.go

+89-35
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,36 @@ func TestRender(t *testing.T) {
129129
addAnnotations(map[string]string{"app.kubernetes.io/instance": string("engineering-dev")}))),
130130
},
131131
},
132+
133+
{
134+
name: "repeat elements",
135+
elements: []apiextensionsv1.JSON{
136+
{Raw: []byte(`{"env": "engineering-dev","externalIP": "192.168.50.50","namespaces":["testing1","testing2"]}`)},
137+
},
138+
setOptions: []func(*templatesv1.GitOpsSet){
139+
func(s *templatesv1.GitOpsSet) {
140+
s.Spec.Templates = []templatesv1.GitOpsSetTemplate{
141+
{
142+
Content: runtime.RawExtension{
143+
Raw: mustMarshalJSON(t, makeTestService(types.NamespacedName{Name: "{{ .element.env}}-demo1"})),
144+
},
145+
},
146+
{
147+
Repeat: "{ $.namespaces }",
148+
Content: runtime.RawExtension{
149+
Raw: mustMarshalJSON(t, makeTestNamespace("{{ .repeat }}-{{ .element.env }}")),
150+
},
151+
},
152+
}
153+
},
154+
},
155+
want: []*unstructured.Unstructured{
156+
test.ToUnstructured(t, makeTestService(nsn("demo", "engineering-dev-demo1"), setClusterIP("192.168.50.50"),
157+
addAnnotations(map[string]string{"app.kubernetes.io/instance": string("engineering-dev")}))),
158+
test.ToUnstructured(t, makeTestNamespace("testing1-engineering-dev")),
159+
test.ToUnstructured(t, makeTestNamespace("testing2-engineering-dev")),
160+
},
161+
},
132162
}
133163

134164
for _, tt := range generatorTests {
@@ -144,43 +174,50 @@ func TestRender(t *testing.T) {
144174
}
145175
}
146176

147-
// TODO: Write tests for error cases?
148-
// func TestRender_errors(t *testing.T) {
149-
// templateTests := []struct {
150-
// name string
151-
// setOptions []func(*templatesv1.GitOpsSet)
152-
// wantErr string
153-
// }{
154-
// {
155-
// name: "bad template",
156-
// setOptions: []func(*templatesv1.GitOpsSet){
157-
// func(s *templatesv1.GitOpsSet) {
158-
// s.Spec.Templates = []templatesv1.GitOpsSetTemplate{
159-
// {
160-
// RawExtension: runtime.RawExtension{
161-
// Raw: mustMarshalJSON(t, makeTestService(types.NamespacedName{Name: "{{ .unknown}}-demo1"})),
162-
// },
163-
// },
164-
// }
165-
// },
166-
// },
167-
// wantErr: "template is empty",
168-
// },
169-
// }
170-
171-
// testGenerators := map[string]generators.Generator{
172-
// "List": list.NewGenerator(logr.Discard()),
173-
// }
177+
func TestRender_errors(t *testing.T) {
178+
templateTests := []struct {
179+
name string
180+
setOptions []func(*templatesv1.GitOpsSet)
181+
wantErr string
182+
}{
183+
{
184+
name: "bad template",
185+
setOptions: []func(*templatesv1.GitOpsSet){
186+
func(gs *templatesv1.GitOpsSet) {
187+
gs.Spec.Generators = []templatesv1.GitOpsSetGenerator{
188+
{
189+
List: &templatesv1.ListGenerator{
190+
Elements: []apiextensionsv1.JSON{
191+
{Raw: []byte(`{"env": "engineering-dev","externalIP": "192.168.50.50"}`)},
192+
},
193+
},
194+
},
195+
}
196+
gs.Spec.Templates = []templatesv1.GitOpsSetTemplate{
197+
{
198+
Content: runtime.RawExtension{
199+
Raw: []byte("{{ .test | tested }}"),
200+
},
201+
},
202+
}
203+
},
204+
},
205+
wantErr: `failed to parse template: template: gitopsset-template:1: function "tested" not defined`,
206+
},
207+
}
208+
testGenerators := map[string]generators.Generator{
209+
"List": list.NewGenerator(logr.Discard()),
210+
}
174211

175-
// for _, tt := range templateTests {
176-
// t.Run(tt.name, func(t *testing.T) {
177-
// gset := makeTestGitOpsSet(t, tt.setOptions...)
178-
// _, err := Render(context.TODO(), gset, testGenerators)
212+
for _, tt := range templateTests {
213+
t.Run(tt.name, func(t *testing.T) {
214+
gset := makeTestGitOpsSet(t, tt.setOptions...)
215+
_, err := Render(context.TODO(), gset, testGenerators)
179216

180-
// test.AssertErrorMatch(t, tt.wantErr, err)
181-
// })
182-
// }
183-
// }
217+
test.AssertErrorMatch(t, tt.wantErr, err)
218+
})
219+
}
220+
}
184221

185222
func listElements(el []apiextensionsv1.JSON) func(*templatesv1.GitOpsSet) {
186223
return func(gs *templatesv1.GitOpsSet) {
@@ -250,6 +287,23 @@ func makeTestService(name types.NamespacedName, opts ...func(*corev1.Service)) *
250287
return &s
251288
}
252289

290+
func makeTestNamespace(name string, opts ...func(*corev1.Namespace)) *corev1.Namespace {
291+
n := corev1.Namespace{
292+
TypeMeta: metav1.TypeMeta{
293+
Kind: "Namespace",
294+
APIVersion: "v1",
295+
},
296+
ObjectMeta: metav1.ObjectMeta{
297+
Name: name,
298+
},
299+
}
300+
for _, o := range opts {
301+
o(&n)
302+
}
303+
304+
return &n
305+
}
306+
253307
func setClusterIP(ip string) func(s *corev1.Service) {
254308
return func(s *corev1.Service) {
255309
s.Spec.ClusterIP = ip
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
resources:
2+
- role.yaml
3+
- repeated-list-generator.yaml
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
apiVersion: templates.weave.works/v1alpha1
2+
kind: GitOpsSet
3+
metadata:
4+
name: repeated-gitopsset-sample
5+
spec:
6+
generators:
7+
- list:
8+
elements:
9+
- env: dev
10+
team: dev-team
11+
teams:
12+
- name: "team1"
13+
- name: "team2"
14+
- name: "team3"
15+
- env: staging
16+
team: staging-team
17+
teams:
18+
- name: "team4"
19+
- name: "team5"
20+
- name: "team6"
21+
templates:
22+
- repeat: "{ .teams }"
23+
content:
24+
kind: ConfigMap
25+
apiVersion: v1
26+
metadata:
27+
name: "{{ .repeat.name }}-demo"
28+
data:
29+
name: "{{ .repeat.name }}-demo"
30+
team: "{{ .element.team }}"

0 commit comments

Comments
 (0)