Skip to content

Commit ef75f61

Browse files
iamcodingcatyounghun-jo-levit-com
iamcodingcat
and
younghun-jo-levit-com
authored
fix: Reopen for integration test about add s3 storage-based registry store in Go feature server (#5352)
* fix: upgrade protobuf version, make `protos` directory beforehand Signed-off-by: iamcodingcat <joyh951021@gmail.com> * feat: add aws s3 storage based registry store Signed-off-by: iamcodingcat <joyh951021@gmail.com> * chore: add aws s3 api related pkgs Signed-off-by: iamcodingcat <joyh951021@gmail.com> * style: remove my custom comment Signed-off-by: iamcodingcat <joyh951021@gmail.com> * refact: separate s3 registry file from `local.go` Signed-off-by: iamcodingcat <joyh951021@gmail.com> * feat: add if-statement in Makefile on linux arm64 os-platform Signed-off-by: iamcodingcat <joyh951021@gmail.com> * feat: add test-code for s3 registry store Signed-off-by: iamcodingcat <joyh951021@gmail.com> --------- Signed-off-by: iamcodingcat <joyh951021@gmail.com> Co-authored-by: 조영훈 <younghun.jo@ilevit.com>
1 parent 886a69d commit ef75f61

File tree

7 files changed

+265
-2
lines changed

7 files changed

+265
-2
lines changed

‎Makefile

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -655,11 +655,14 @@ format-ui:
655655

656656
# Go SDK & embedded
657657
PB_REL = https://github.com/protocolbuffers/protobuf/releases
658-
PB_VERSION = 3.11.2
658+
PB_VERSION = 30.2
659659
PB_ARCH := $(shell uname -m)
660660
ifeq ($(PB_ARCH), arm64)
661661
PB_ARCH=aarch_64
662662
endif
663+
ifeq ($(PB_ARCH), aarch64)
664+
PB_ARCH=aarch_64
665+
endif
663666
PB_PROTO_FOLDERS=core registry serving types storage
664667

665668
$(TOOL_DIR)/protoc-$(PB_VERSION)-$(OS)-$(PB_ARCH).zip: $(TOOL_DIR)
@@ -674,6 +677,7 @@ install-go-proto-dependencies: $(TOOL_DIR)/protoc-$(PB_VERSION)-$(OS)-$(PB_ARCH)
674677

675678
.PHONY: compile-protos-go
676679
compile-protos-go: install-go-proto-dependencies
680+
mkdir -p $(ROOT_DIR)/go/protos
677681
$(foreach folder,$(PB_PROTO_FOLDERS), \
678682
protoc --proto_path=$(ROOT_DIR)/protos \
679683
--go_out=$(ROOT_DIR)/go/protos \

‎go.mod

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,24 @@ require (
2323
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
2424
github.com/andybalholm/brotli v1.1.0 // indirect
2525
github.com/apache/thrift v0.21.0 // indirect
26+
github.com/aws/aws-sdk-go-v2 v1.36.3 // indirect
27+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
28+
github.com/aws/aws-sdk-go-v2/config v1.29.14 // indirect
29+
github.com/aws/aws-sdk-go-v2/credentials v1.17.67 // indirect
30+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect
31+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
32+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
33+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
34+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect
35+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect
36+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1 // indirect
37+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect
38+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect
39+
github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 // indirect
40+
github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect
41+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect
42+
github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect
43+
github.com/aws/smithy-go v1.22.2 // indirect
2644
github.com/cespare/xxhash/v2 v2.3.0 // indirect
2745
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
2846
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect

‎go.sum

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,42 @@ github.com/apache/arrow/go/v17 v17.0.0 h1:RRR2bdqKcdbss9Gxy2NS/hK8i4LDMh23L6BbkN
66
github.com/apache/arrow/go/v17 v17.0.0/go.mod h1:jR7QHkODl15PfYyjM2nU+yTLScZ/qfj7OSUZmJ8putc=
77
github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE=
88
github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw=
9+
github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM=
10+
github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
11+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs=
12+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14=
13+
github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM=
14+
github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g=
15+
github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM=
16+
github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ=
17+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw=
18+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M=
19+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q=
20+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=
21+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0=
22+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
23+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
24+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
25+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM=
26+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs=
27+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE=
28+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA=
29+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1 h1:4nm2G6A4pV9rdlWzGMPv4BNtQp22v1hg3yrtkYpeLl8=
30+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0=
31+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM=
32+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=
33+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg=
34+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA=
35+
github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 h1:BRXS0U76Z8wfF+bnkilA2QwpIch6URlm++yPUt9QPmQ=
36+
github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3/go.mod h1:bNXKFFyaiVvWuR6O16h/I1724+aXe/tAkA9/QS01t5k=
37+
github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8=
38+
github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI=
39+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako=
40+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs=
41+
github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/XvaX32evhproijJEZY=
42+
github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=
43+
github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ=
44+
github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
945
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
1046
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
1147
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=

‎go/internal/feast/registry/local.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
package registry
22

33
import (
4+
"github.com/google/uuid"
45
"io/ioutil"
56
"os"
67
"path/filepath"
78

8-
"github.com/google/uuid"
99
"google.golang.org/protobuf/proto"
1010
"google.golang.org/protobuf/types/known/timestamppb"
1111

‎go/internal/feast/registry/registry.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -364,6 +364,8 @@ func getRegistryStoreFromType(registryStoreType string, registryConfig *Registry
364364
switch registryStoreType {
365365
case "FileRegistryStore":
366366
return NewFileRegistryStore(registryConfig, repoPath), nil
367+
case "S3RegistryStore":
368+
return NewS3RegistryStore(registryConfig, repoPath), nil
367369
}
368370
return nil, errors.New("only FileRegistryStore as a RegistryStore is supported at this moment")
369371
}

‎go/internal/feast/registry/registry_test.go

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
package registry
2+
3+
import (
4+
"context"
5+
"errors"
6+
"io/ioutil"
7+
"net/url"
8+
"strings"
9+
"testing"
10+
"time"
11+
12+
"github.com/aws/aws-sdk-go-v2/service/s3"
13+
)
14+
15+
func TestGetOnlineFeaturesS3Registry(t *testing.T) {
16+
mockS3Client := &MockS3Client{
17+
GetObjectFn: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
18+
return &s3.GetObjectOutput{
19+
Body: ioutil.NopCloser(strings.NewReader("mock data")),
20+
}, nil
21+
},
22+
DeleteObjectFn: func(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) {
23+
return &s3.DeleteObjectOutput{}, nil
24+
},
25+
}
26+
27+
tests := []struct {
28+
name string
29+
config *RepoConfig
30+
}{
31+
{
32+
name: "redis with simple features",
33+
config: &RepoConfig{
34+
Project: "feature_repo",
35+
Registry: map[string]interface{}{
36+
"path": "s3://test-bucket/path/to/registry.db",
37+
},
38+
Provider: "aws",
39+
},
40+
},
41+
}
42+
for _, test := range tests {
43+
registryConfig, err := test.config.GetRegistryConfig()
44+
if err != nil {
45+
t.Errorf("Error getting registry config. msg: %s", err.Error())
46+
}
47+
r := &Registry{
48+
project: test.config.Project,
49+
cachedRegistryProtoTtl: time.Duration(registryConfig.CacheTtlSeconds) * time.Second,
50+
}
51+
_ = registryConfig.RegistryStoreType
52+
registryPath := registryConfig.Path
53+
uri, err := url.Parse(registryPath)
54+
if err != nil {
55+
t.Errorf("Error parsing registry path. msg: %s", err.Error())
56+
}
57+
if registryStoreType, ok := REGISTRY_STORE_CLASS_FOR_SCHEME[uri.Scheme]; ok {
58+
switch registryStoreType {
59+
case "S3RegistryStore":
60+
registryStore := &S3RegistryStore{
61+
filePath: registryConfig.Path,
62+
s3Client: mockS3Client,
63+
}
64+
r.registryStore = registryStore
65+
err := r.InitializeRegistry()
66+
if err != nil {
67+
t.Errorf("Error initializing registry. msg: %s. registry path=%q", err.Error(), registryPath)
68+
}
69+
default:
70+
t.Errorf("Only S3RegistryStore is supported on this testing. got=%s", registryStoreType)
71+
}
72+
}
73+
}
74+
}
75+
76+
// MockS3Client is mock client for testing s3 registry store
77+
type MockS3Client struct {
78+
GetObjectFn func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
79+
DeleteObjectFn func(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
80+
}
81+
82+
func (m *MockS3Client) GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
83+
if m.GetObjectFn != nil {
84+
return m.GetObjectFn(ctx, params)
85+
}
86+
return nil, errors.New("not implemented")
87+
}
88+
89+
func (m *MockS3Client) DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) {
90+
if m.DeleteObjectFn != nil {
91+
return m.DeleteObjectFn(ctx, params)
92+
}
93+
return nil, errors.New("not implemented")
94+
}

‎go/internal/feast/registry/s3.go

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
package registry
2+
3+
import (
4+
"context"
5+
"errors"
6+
"io/ioutil"
7+
"strings"
8+
"time"
9+
10+
"github.com/aws/aws-sdk-go-v2/aws"
11+
awsConfig "github.com/aws/aws-sdk-go-v2/config"
12+
"github.com/aws/aws-sdk-go-v2/service/s3"
13+
"github.com/feast-dev/feast/go/protos/feast/core"
14+
15+
"google.golang.org/protobuf/proto"
16+
)
17+
18+
// S3ClientInterface define interface of s3.Client for making mocking s3 client and testing it
19+
type S3ClientInterface interface {
20+
GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
21+
DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
22+
}
23+
24+
// A S3RegistryStore is a S3 object storage-based implementation of the RegistryStore interface
25+
type S3RegistryStore struct {
26+
filePath string
27+
s3Client S3ClientInterface
28+
}
29+
30+
// NewS3RegistryStore creates a S3RegistryStore with the given configuration
31+
func NewS3RegistryStore(config *RegistryConfig, repoPath string) *S3RegistryStore {
32+
var lr S3RegistryStore
33+
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
34+
defer cancel()
35+
36+
cfg, err := awsConfig.LoadDefaultConfig(ctx)
37+
if err != nil {
38+
lr = S3RegistryStore{
39+
filePath: config.Path,
40+
}
41+
} else {
42+
lr = S3RegistryStore{
43+
filePath: config.Path,
44+
s3Client: s3.NewFromConfig(cfg),
45+
}
46+
}
47+
return &lr
48+
}
49+
50+
func (r *S3RegistryStore) GetRegistryProto() (*core.Registry, error) {
51+
bucket, key, err := r.parseS3Path()
52+
if err != nil {
53+
return nil, err
54+
}
55+
56+
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
57+
defer cancel()
58+
output, err := r.s3Client.GetObject(ctx,
59+
&s3.GetObjectInput{
60+
Bucket: aws.String(bucket),
61+
Key: aws.String(key),
62+
})
63+
if err != nil {
64+
panic(err)
65+
}
66+
defer output.Body.Close()
67+
68+
data, err := ioutil.ReadAll(output.Body)
69+
if err != nil {
70+
return nil, err
71+
}
72+
73+
registry := &core.Registry{}
74+
if err := proto.Unmarshal(data, registry); err != nil {
75+
return nil, err
76+
}
77+
return registry, nil
78+
}
79+
80+
func (r *S3RegistryStore) UpdateRegistryProto(rp *core.Registry) error {
81+
return errors.New("not implemented in S3RegistryStore")
82+
}
83+
84+
func (r *S3RegistryStore) Teardown() error {
85+
bucket, key, err := r.parseS3Path()
86+
if err != nil {
87+
return err
88+
}
89+
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
90+
defer cancel()
91+
_, err = r.s3Client.DeleteObject(ctx,
92+
&s3.DeleteObjectInput{
93+
Bucket: aws.String(bucket),
94+
Key: aws.String(key),
95+
})
96+
if err != nil {
97+
panic(err)
98+
}
99+
return nil
100+
}
101+
102+
func (r *S3RegistryStore) parseS3Path() (string, string, error) {
103+
path := strings.TrimPrefix(r.filePath, "s3://")
104+
parts := strings.SplitN(path, "/", 2)
105+
if len(parts) != 2 {
106+
return "", "", errors.New("invalid S3 file path format")
107+
}
108+
return parts[0], parts[1], nil
109+
}

0 commit comments

Comments
 (0)