-
Notifications
You must be signed in to change notification settings - Fork 1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
chore: Operator / Feast should have matching Data Source types (#5041)
data store types Signed-off-by: Tommy Hughes <tohughes@redhat.com>
- Loading branch information
1 parent
b24d531
commit d937dcb
Showing
11 changed files
with
145 additions
and
19 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -16,6 +16,9 @@ issues: | |
| linters: | ||
| - dupl | ||
| - lll | ||
| - path: "test/*" | ||
| linters: | ||
| - lll | ||
| linters: | ||
| disable-all: true | ||
| enable: | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
18 changes: 18 additions & 0 deletions
18
infra/feast-operator/test/data-source-types/data-source-types.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| import os | ||
| from feast.repo_config import REGISTRY_CLASS_FOR_TYPE, OFFLINE_STORE_CLASS_FOR_TYPE, ONLINE_STORE_CLASS_FOR_TYPE, LEGACY_ONLINE_STORE_CLASS_FOR_TYPE | ||
|
|
||
| def save_in_script_directory(filename: str, typedict: dict[str, str]): | ||
| script_dir = os.path.dirname(os.path.abspath(__file__)) | ||
| file_path = os.path.join(script_dir, filename) | ||
|
|
||
| with open(file_path, 'w') as file: | ||
| for k in typedict.keys(): | ||
| file.write(k+"\n") | ||
|
|
||
| for legacyType in LEGACY_ONLINE_STORE_CLASS_FOR_TYPE.keys(): | ||
| if legacyType in ONLINE_STORE_CLASS_FOR_TYPE: | ||
| del ONLINE_STORE_CLASS_FOR_TYPE[legacyType] | ||
|
|
||
| save_in_script_directory("registry.out", REGISTRY_CLASS_FOR_TYPE) | ||
| save_in_script_directory("online-store.out", ONLINE_STORE_CLASS_FOR_TYPE) | ||
| save_in_script_directory("offline-store.out", OFFLINE_STORE_CLASS_FOR_TYPE) |
88 changes: 88 additions & 0 deletions
88
infra/feast-operator/test/data-source-types/data_source_types_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| package datasources | ||
|
|
||
| import ( | ||
| "bufio" | ||
| "os" | ||
| "slices" | ||
| "testing" | ||
|
|
||
| . "github.com/onsi/ginkgo/v2" | ||
| . "github.com/onsi/gomega" | ||
|
|
||
| feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" | ||
| "github.com/feast-dev/feast/infra/feast-operator/internal/controller/services" | ||
| ) | ||
|
|
||
| func TestDataSourceTypes(t *testing.T) { | ||
| RegisterFailHandler(Fail) | ||
|
|
||
| RunSpecs(t, "Data Source Suite") | ||
| } | ||
|
|
||
| var _ = Describe("FeatureStore Data Source Types", func() { | ||
| Context("When checking against the python code in feast.repo_config", func() { | ||
| It("should match defined registry persistence types in the operator", func() { | ||
| registryFilePersistenceTypes := []string{string(services.RegistryFileConfigType)} | ||
| registryPersistenceTypes := append(feastdevv1alpha1.ValidRegistryDBStorePersistenceTypes, registryFilePersistenceTypes...) | ||
| checkPythonPersistenceTypes("registry.out", registryPersistenceTypes) | ||
| }) | ||
| It("should match defined onlineStore persistence types in the operator", func() { | ||
| onlineFilePersistenceTypes := []string{string(services.OnlineSqliteConfigType)} | ||
| onlinePersistenceTypes := append(feastdevv1alpha1.ValidOnlineStoreDBStorePersistenceTypes, onlineFilePersistenceTypes...) | ||
| checkPythonPersistenceTypes("online-store.out", onlinePersistenceTypes) | ||
| }) | ||
| It("should match defined offlineStore persistence types in the operator", func() { | ||
| offlinePersistenceTypes := append(feastdevv1alpha1.ValidOfflineStoreDBStorePersistenceTypes, feastdevv1alpha1.ValidOfflineStoreFilePersistenceTypes...) | ||
| checkPythonPersistenceTypes("offline-store.out", offlinePersistenceTypes) | ||
| }) | ||
| }) | ||
| }) | ||
|
|
||
| func checkPythonPersistenceTypes(fileName string, operatorDsTypes []string) { | ||
| feastDsTypes, err := readFileLines(fileName) | ||
| Expect(err).NotTo(HaveOccurred()) | ||
|
|
||
| // Add remote type to slice, as its not a file or db type and we want to limit its use to registry service when deploying with the operator | ||
| operatorDsTypes = append(operatorDsTypes, "remote") | ||
| missingFeastTypes := []string{} | ||
| for _, ods := range operatorDsTypes { | ||
| if len(ods) > 0 { | ||
| if !slices.Contains(feastDsTypes, ods) { | ||
| missingFeastTypes = append(missingFeastTypes, ods) | ||
| } | ||
| } | ||
| } | ||
| Expect(missingFeastTypes).To(BeEmpty()) | ||
|
|
||
| missingOperatorTypes := []string{} | ||
| for _, fds := range feastDsTypes { | ||
| if len(fds) > 0 { | ||
| if !slices.Contains(operatorDsTypes, fds) { | ||
| missingOperatorTypes = append(missingOperatorTypes, fds) | ||
| } | ||
| } | ||
| } | ||
| Expect(missingOperatorTypes).To(BeEmpty()) | ||
| } | ||
|
|
||
| func readFileLines(filePath string) ([]string, error) { | ||
| file, err := os.Open(filePath) | ||
| Expect(err).NotTo(HaveOccurred()) | ||
| defer closeFile(file) | ||
|
|
||
| var lines []string | ||
| scanner := bufio.NewScanner(file) | ||
| for scanner.Scan() { | ||
| lines = append(lines, scanner.Text()) | ||
| } | ||
|
|
||
| err = scanner.Err() | ||
| Expect(err).NotTo(HaveOccurred()) | ||
|
|
||
| return lines, nil | ||
| } | ||
|
|
||
| func closeFile(file *os.File) { | ||
| err := file.Close() | ||
| Expect(err).NotTo(HaveOccurred()) | ||
| } |