Merge branch 'feast-dev:master' into feature/upgrade-3.11-remove-bytewax

Makefile

@@ -38,7 +38,8 @@ build: protos build-java build-docker
 
 install-python-ci-dependencies:
 	python -m piptools sync sdk/python/requirements/py$(PYTHON)-ci-requirements.txt
-	COMPILE_GO=true python setup.py develop
+	pip install --no-deps -e .
+	python setup.py build_python_protos --inplace
 
 lock-python-ci-dependencies:
 	python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py$(PYTHON)-ci-requirements.txt

docs/reference/offline-stores/spark.md

@@ -30,6 +30,8 @@ offline_store:
         spark.sql.catalogImplementation: "hive"
         spark.sql.parser.quotedRegexColumnNames: "true"
         spark.sql.session.timeZone: "UTC"
+        spark.sql.execution.arrow.fallback.enabled: "true"
+        spark.sql.execution.arrow.pyspark.enabled: "true"
 online_store:
     path: data/online_store.db
 ```

protos/feast/core/OnDemandFeatureView.proto

@@ -27,6 +27,7 @@ import "feast/core/FeatureView.proto";
 import "feast/core/FeatureViewProjection.proto";
 import "feast/core/Feature.proto";
 import "feast/core/DataSource.proto";
+import "feast/core/Transformation.proto";
 
 message OnDemandFeatureView {
     // User-specified specifications of this feature view.
@@ -48,10 +49,10 @@ message OnDemandFeatureViewSpec {
     // Map of sources for this feature view.
     map<string, OnDemandSource> sources = 4;
 
-    oneof transformation {
-        UserDefinedFunction user_defined_function = 5;
-        OnDemandSubstraitTransformation on_demand_substrait_transformation = 9;
-    }
+    UserDefinedFunction user_defined_function = 5 [deprecated = true];
+
+    // Oneof with {user_defined_function, on_demand_substrait_transformation}
+    FeatureTransformationV2 feature_transformation = 10;
 
     // Description of the on demand feature view.
     string description = 6;
@@ -61,6 +62,7 @@ message OnDemandFeatureViewSpec {
 
     // Owner of the on demand feature view.
     string owner = 8;
+    string mode = 11;
 }
 
 message OnDemandFeatureViewMeta {
@@ -81,6 +83,8 @@ message OnDemandSource {
 
 // Serialized representation of python function.
 message UserDefinedFunction {
+    option deprecated = true;
+
     // The function name
     string name = 1;
 
@@ -90,7 +94,3 @@ message UserDefinedFunction {
     // The string representation of the udf
     string body_text = 3;
 }
-
-message OnDemandSubstraitTransformation {
-    bytes substrait_plan = 1;
-}

protos/feast/core/Registry.proto

@@ -27,7 +27,6 @@ import "feast/core/FeatureTable.proto";
 import "feast/core/FeatureView.proto";
 import "feast/core/InfraObject.proto";
 import "feast/core/OnDemandFeatureView.proto";
-import "feast/core/RequestFeatureView.proto";
 import "feast/core/StreamFeatureView.proto";
 import "feast/core/DataSource.proto";
 import "feast/core/SavedDataset.proto";
@@ -41,7 +40,6 @@ message Registry {
     repeated FeatureView feature_views = 6;
     repeated DataSource data_sources = 12;
     repeated OnDemandFeatureView on_demand_feature_views = 8;
-    repeated RequestFeatureView request_feature_views = 9;
     repeated StreamFeatureView stream_feature_views = 14;
     repeated FeatureService feature_services = 7;
     repeated SavedDataset saved_datasets = 11;

protos/feast/core/StreamFeatureView.proto

@@ -29,6 +29,7 @@ import "feast/core/FeatureView.proto";
 import "feast/core/Feature.proto";
 import "feast/core/DataSource.proto";
 import "feast/core/Aggregation.proto";
+import "feast/core/Transformation.proto";
 
 message StreamFeatureView {
     // User-specified specifications of this feature view.
@@ -77,7 +78,8 @@ message StreamFeatureViewSpec {
     bool online = 12;
 
     // Serialized function that is encoded in the streamfeatureview
-    UserDefinedFunction user_defined_function = 13;
+    UserDefinedFunction user_defined_function = 13 [deprecated = true];
+
 
     // Mode of execution
     string mode = 14;
@@ -87,5 +89,8 @@ message StreamFeatureViewSpec {
 
     // Timestamp field for aggregation
     string timestamp_field = 16;
+
+    // Oneof with {user_defined_function, on_demand_substrait_transformation}
+    FeatureTransformationV2 feature_transformation = 17;
 }
 

protos/feast/core/Transformation.proto

@@ -0,0 +1,32 @@
+syntax = "proto3";
+package feast.core;
+
+option go_package = "github.com/feast-dev/feast/go/protos/feast/core";
+option java_outer_classname = "FeatureTransformationProto";
+option java_package = "feast.proto.core";
+
+import "google/protobuf/duration.proto";
+
+// Serialized representation of python function.
+message UserDefinedFunctionV2 {
+    // The function name
+    string name = 1;
+
+    // The python-syntax function body (serialized by dill)
+    bytes body = 2;
+
+    // The string representation of the udf
+    string body_text = 3;
+}
+
+// A feature transformation executed as a user-defined function
+message FeatureTransformationV2 {
+  oneof transformation {
+    UserDefinedFunctionV2 user_defined_function = 1;
+    SubstraitTransformationV2 substrait_transformation = 2;
+  }
+}
+
+message SubstraitTransformationV2 {
+  bytes substrait_plan = 1;
+}

protos/feast/registry/RegistryServer.proto

@@ -7,7 +7,6 @@ import "feast/core/Registry.proto";
 import "feast/core/Entity.proto";
 import "feast/core/DataSource.proto";
 import "feast/core/FeatureView.proto";
-import "feast/core/RequestFeatureView.proto";
 import "feast/core/StreamFeatureView.proto";
 import "feast/core/OnDemandFeatureView.proto";
 import "feast/core/FeatureService.proto";
@@ -28,10 +27,6 @@ service RegistryServer{
   rpc GetFeatureView (GetFeatureViewRequest) returns (feast.core.FeatureView) {}
   rpc ListFeatureViews (ListFeatureViewsRequest) returns (ListFeatureViewsResponse) {}
 
-  // RequestFeatureView RPCs
-  rpc GetRequestFeatureView (GetRequestFeatureViewRequest) returns (feast.core.RequestFeatureView) {}
-  rpc ListRequestFeatureViews (ListRequestFeatureViewsRequest) returns (ListRequestFeatureViewsResponse) {}
-
   // StreamFeatureView RPCs
   rpc GetStreamFeatureView (GetStreamFeatureViewRequest) returns (feast.core.StreamFeatureView) {}
   rpc ListStreamFeatureViews (ListStreamFeatureViewsRequest) returns (ListStreamFeatureViewsResponse) {}
@@ -126,23 +121,6 @@ message ListFeatureViewsResponse {
     repeated feast.core.FeatureView feature_views = 1;
 }
 
-// RequestFeatureView
-
-message GetRequestFeatureViewRequest {
-    string name = 1;
-    string project = 2;
-    bool allow_cache = 3;
-}
-
-message ListRequestFeatureViewsRequest {
-    string project = 1;
-    bool allow_cache = 2;
-}
-
-message ListRequestFeatureViewsResponse {
-    repeated feast.core.RequestFeatureView request_feature_views = 1;
-}
-
 // StreamFeatureView
 
 message GetStreamFeatureViewRequest {

sdk/python/docs/source/feast.protos.feast.core.rst

@@ -228,22 +228,6 @@ feast.protos.feast.core.Registry\_pb2\_grpc module
    :undoc-members:
    :show-inheritance:
 
-feast.protos.feast.core.RequestFeatureView\_pb2 module
-------------------------------------------------------
-
-.. automodule:: feast.protos.feast.core.RequestFeatureView_pb2
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-feast.protos.feast.core.RequestFeatureView\_pb2\_grpc module
-------------------------------------------------------------
-
-.. automodule:: feast.protos.feast.core.RequestFeatureView_pb2_grpc
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
 feast.protos.feast.core.SavedDataset\_pb2 module
 ------------------------------------------------
 

sdk/python/docs/source/feast.rst

@@ -273,14 +273,6 @@ feast.repo\_upgrade module
    :undoc-members:
    :show-inheritance:
 
-feast.request\_feature\_view module
------------------------------------
-
-.. automodule:: feast.request_feature_view
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
 feast.saved\_dataset module
 ---------------------------
 

sdk/python/feast/__init__.py

@@ -22,7 +22,6 @@
 from .field import Field
 from .on_demand_feature_view import OnDemandFeatureView
 from .repo_config import RepoConfig
-from .request_feature_view import RequestFeatureView
 from .stream_feature_view import StreamFeatureView
 from .value_type import ValueType
 
@@ -49,7 +48,6 @@
     "BigQuerySource",
     "FileSource",
     "RedshiftSource",
-    "RequestFeatureView",
     "SnowflakeSource",
     "PushSource",
     "RequestSource",

sdk/python/feast/cli.py

@@ -381,7 +381,6 @@ def feature_view_list(ctx: click.Context):
     table = []
     for feature_view in [
         *store.list_feature_views(),
-        *store.list_request_feature_views(),
         *store.list_on_demand_feature_views(),
     ]:
         entities = set()

sdk/python/feast/diff/registry_diff.py

@@ -20,9 +20,6 @@
     OnDemandFeatureView as OnDemandFeatureViewProto,
 )
 from feast.protos.feast.core.OnDemandFeatureView_pb2 import OnDemandFeatureViewSpec
-from feast.protos.feast.core.RequestFeatureView_pb2 import (
-    RequestFeatureView as RequestFeatureViewProto,
-)
 from feast.protos.feast.core.StreamFeatureView_pb2 import (
     StreamFeatureView as StreamFeatureViewProto,
 )
@@ -110,7 +107,6 @@ def tag_objects_for_keep_delete_update_add(
     FeatureViewProto,
     FeatureServiceProto,
     OnDemandFeatureViewProto,
-    RequestFeatureViewProto,
     StreamFeatureViewProto,
     ValidationReferenceProto,
 )
@@ -144,11 +140,26 @@ def diff_registry_objects(
             if _field.name in FIELDS_TO_IGNORE:
                 continue
             elif getattr(current_spec, _field.name) != getattr(new_spec, _field.name):
-                if _field.name == "user_defined_function":
+                if _field.name == "feature_transformation":
                     current_spec = cast(OnDemandFeatureViewSpec, current_spec)
                     new_spec = cast(OnDemandFeatureViewSpec, new_spec)
-                    current_udf = current_spec.user_defined_function
-                    new_udf = new_spec.user_defined_function
+                    # Check if the old proto is populated and use that if it is
+                    feature_transformation_udf = (
+                        current_spec.feature_transformation.user_defined_function
+                    )
+                    if (
+                        current_spec.HasField("user_defined_function")
+                        and not feature_transformation_udf
+                    ):
+                        deprecated_udf = current_spec.user_defined_function
+                    else:
+                        deprecated_udf = None
+                    current_udf = (
+                        deprecated_udf
+                        if deprecated_udf is not None
+                        else feature_transformation_udf
+                    )
+                    new_udf = new_spec.feature_transformation.user_defined_function
                     for _udf_field in current_udf.DESCRIPTOR.fields:
                         if _udf_field.name == "body":
                             continue
@@ -324,7 +335,6 @@ def apply_diff_to_registry(
             elif feast_object_diff.feast_object_type in [
                 FeastObjectType.FEATURE_VIEW,
                 FeastObjectType.ON_DEMAND_FEATURE_VIEW,
-                FeastObjectType.REQUEST_FEATURE_VIEW,
                 FeastObjectType.STREAM_FEATURE_VIEW,
             ]:
                 feature_view_obj = cast(
@@ -368,7 +378,6 @@ def apply_diff_to_registry(
             elif feast_object_diff.feast_object_type in [
                 FeastObjectType.FEATURE_VIEW,
                 FeastObjectType.ON_DEMAND_FEATURE_VIEW,
-                FeastObjectType.REQUEST_FEATURE_VIEW,
                 FeastObjectType.STREAM_FEATURE_VIEW,
             ]:
                 registry.apply_feature_view(

sdk/python/feast/feast_object.py

@@ -11,20 +11,17 @@
 from .protos.feast.core.FeatureService_pb2 import FeatureServiceSpec
 from .protos.feast.core.FeatureView_pb2 import FeatureViewSpec
 from .protos.feast.core.OnDemandFeatureView_pb2 import OnDemandFeatureViewSpec
-from .protos.feast.core.RequestFeatureView_pb2 import RequestFeatureViewSpec
 from .protos.feast.core.StreamFeatureView_pb2 import StreamFeatureViewSpec
 from .protos.feast.core.ValidationProfile_pb2 import (
     ValidationReference as ValidationReferenceProto,
 )
-from .request_feature_view import RequestFeatureView
 from .saved_dataset import ValidationReference
 from .stream_feature_view import StreamFeatureView
 
 # Convenience type representing all Feast objects
 FeastObject = Union[
     FeatureView,
     OnDemandFeatureView,
-    RequestFeatureView,
     BatchFeatureView,
     StreamFeatureView,
     Entity,
@@ -36,7 +33,6 @@
 FeastObjectSpecProto = Union[
     FeatureViewSpec,
     OnDemandFeatureViewSpec,
-    RequestFeatureViewSpec,
     StreamFeatureViewSpec,
     EntitySpecV2,
     FeatureServiceSpec,