diff --git a/batch_inference_v2/batch_inference_v2.py b/batch_inference_v2/batch_inference_v2.py index dbe6e6867..dfabf16eb 100644 --- a/batch_inference_v2/batch_inference_v2.py +++ b/batch_inference_v2/batch_inference_v2.py @@ -11,10 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# - -from typing import Any, Dict, List, Union +from inspect import signature +from typing import Any, Dict, List, Optional, Union import mlrun @@ -79,6 +78,28 @@ def _prepare_result_set( ) +def _parse_record_results_kwarg( + last_in_batch_set: Optional[bool], +) -> dict[str, bool]: + """ + Check if `last_in_batch_set` is provided and expected as a parameter. + Return it as a dictionary. + """ + kwarg = "last_in_batch_set" + if last_in_batch_set is None: + return {} + if ( + signature(mlrun.model_monitoring.api.record_results).parameters.get(kwarg) + is None + ): + raise mlrun.errors.MLRunInvalidArgumentError( + f"Unexpected parameter `{kwarg}` for function: " + "`mlrun.model_monitoring.api.record_results`. " + "Please make sure that you are using `mlrun>=1.6.0` version." + ) + return {kwarg: last_in_batch_set} + + def infer( context: mlrun.MLClientCtx, dataset: Union[mlrun.DataItem, list, dict, pd.DataFrame, pd.Series, np.ndarray], @@ -104,6 +125,7 @@ def infer( model_endpoint_sample_set: Union[ mlrun.DataItem, list, dict, pd.DataFrame, pd.Series, np.ndarray ] = None, + last_in_batch_set: Optional[bool] = None, **predict_kwargs: Dict[str, Any], ): """ @@ -163,8 +185,17 @@ def infer( :param model_endpoint_sample_set: A sample dataset to give to compare the inputs in the drift analysis. Can be provided as an input (DataItem) or as a parameter (e.g. string, list, DataFrame). The default chosen sample set will always be the one who is set in the model artifact itself. - - raises MLRunInvalidArgumentError: if both `model_path` and `endpoint_id` are not provided + :param last_in_batch_set: Relevant only when `perform_drift_analysis` is `True`. + This flag can (and should only) be used when the model endpoint does not have + model-monitoring set. + If set to `True` (the default), this flag marks the current monitoring window + (on this monitoring endpoint) as completed - the data inferred so far is assumed + to be the complete data for this monitoring window. + You may want to set this flag to `False` if you want to record multiple results in + close time proximity ("batch set"). In this case, set this flag to `False` on all + but the last batch in the set. + raises MLRunInvalidArgumentError: if both `model_path` and `endpoint_id` are not provided, or if `last_in_batch_set` is + provided for an unsupported `mlrun` version. """ # Loading the model: @@ -241,4 +272,5 @@ def infer( artifacts_tag=artifacts_tag, trigger_monitoring_job=trigger_monitoring_job, default_batch_image=batch_image_job, + **_parse_record_results_kwarg(last_in_batch_set=last_in_batch_set), ) diff --git a/batch_inference_v2/function.yaml b/batch_inference_v2/function.yaml index 7367f0282..0f81e3c65 100644 --- a/batch_inference_v2/function.yaml +++ b/batch_inference_v2/function.yaml @@ -2,7 +2,7 @@ kind: job metadata: name: batch-inference-v2 tag: '' - hash: 3206a38ca42ac3ec02b052ef9f5b218156ab65b0 + hash: 1bb938adfe2b006dad2c70d5fec32e88b22bde4e project: '' labels: author: eyald @@ -15,7 +15,7 @@ spec: args: [] image: mlrun/mlrun build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKCgpmcm9tIHR5cGluZyBpbXBvcnQgQW55LCBEaWN0LCBMaXN0LCBVbmlvbgoKaW1wb3J0IG1scnVuCgp0cnk6CiAgICBpbXBvcnQgbWxydW4ubW9kZWxfbW9uaXRvcmluZy5hcGkKZXhjZXB0IE1vZHVsZU5vdEZvdW5kRXJyb3I6CiAgICByYWlzZSBtbHJ1bi5lcnJvcnMuTUxSdW5Ob3RGb3VuZEVycm9yKAogICAgICAgIGYiUGxlYXNlIHVwZGF0ZSB5b3VyIGBtbHJ1bmAgdmVyc2lvbiB0byA+PTEuNS4wIG9yIHVzZSBhbiAiCiAgICAgICAgZiJvbGRlciB2ZXJzaW9uIG9mIHRoZSBiYXRjaCBpbmZlcmVuY2UgZnVuY3Rpb24uIgogICAgKQoKCmltcG9ydCBudW1weSBhcyBucAppbXBvcnQgcGFuZGFzIGFzIHBkCmZyb20gbWxydW4uZnJhbWV3b3Jrcy5hdXRvX21scnVuIGltcG9ydCBBdXRvTUxSdW4KCgpkZWYgX3ByZXBhcmVfcmVzdWx0X3NldCgKICAgIHg6IHBkLkRhdGFGcmFtZSwgbGFiZWxfY29sdW1uczogTGlzdFtzdHJdLCB5X3ByZWQ6IG5wLm5kYXJyYXkKKSAtPiBwZC5EYXRhRnJhbWU6CiAgICAiIiIKICAgIFNldCBkZWZhdWx0IGxhYmVsIGNvbHVtbiBuYW1lcyBhbmQgdmFsaWRhdGUgZ2l2ZW4gbmFtZXMgdG8gcHJlcGFyZSB0aGUgcmVzdWx0IHNldCAtIGEgY29uY2F0ZW5hdGlvbiBvZiB0aGUgaW5wdXRzCiAgICAoeCkgYW5kIHRoZSBtb2RlbCBwcmVkaWN0aW9ucyAoeV9wcmVkKS4KCiAgICA6cGFyYW0geDogICAgICAgICAgICAgVGhlIGlucHV0cy4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiBBIGxpc3Qgb2Ygc3RyaW5ncyByZXByZXNlbnRpbmcgdGhlIHRhcmdldCBjb2x1bW4gbmFtZXMgdG8gYWRkIHRvIHRoZSBwcmVkaWN0aW9ucy4gRGVmYXVsdCBuYW1lCiAgICAgICAgICAgICAgICAgICAgICAgICAgd2lsbCBiZSB1c2VkIGluIGNhc2UgdGhlIGxpc3QgaXMgZW1wdHkgKHByZWRpY3RlZF9sYWJlbF97aX0pLgogICAgOnBhcmFtIHlfcHJlZDogICAgICAgIFRoZSBtb2RlbCBwcmVkaWN0aW9ucyBvbiB0aGUgaW5wdXRzLgoKICAgIDpyZXR1cm5zOiBUaGUgcmVzdWx0IHNldC4KCiAgICByYWlzZXMgTUxSdW5JbnZhbGlkQXJndW1lbnRFcnJvcjogSWYgdGhlIGxhYmVscyBjb2x1bW5zIGFtb3VudCBkbyBub3QgbWF0Y2ggdGhlIG91dHB1dHMgb3IgaWYgb25lIG9mIHRoZSBsYWJlbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBjb2x1bW4gYWxyZWFkeSBleGlzdHMgaW4gdGhlIGRhdGFzZXQuCiAgICAiIiIKICAgICMgUHJlcGFyZSBkZWZhdWx0IHRhcmdldCBjb2x1bW5zIG5hbWVzIGlmIG5vdCBwcm92aWRlZDoKICAgIHByZWRpY3Rpb25fY29sdW1uc19hbW91bnQgPSAxIGlmIGxlbih5X3ByZWQuc2hhcGUpID09IDEgZWxzZSB5X3ByZWQuc2hhcGVbMV0KICAgIGlmIGxlbihsYWJlbF9jb2x1bW5zKSA9PSAwOgogICAgICAgICMgQWRkIGRlZmF1bHQgbGFiZWwgY29sdW1uIG5hbWVzOgogICAgICAgIGlmIHByZWRpY3Rpb25fY29sdW1uc19hbW91bnQgPT0gMToKICAgICAgICAgICAgbGFiZWxfY29sdW1ucyA9IFsicHJlZGljdGVkX2xhYmVsIl0KICAgICAgICBlbHNlOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zID0gWwogICAgICAgICAgICAgICAgZiJwcmVkaWN0ZWRfbGFiZWxfe2l9IiBmb3IgaSBpbiByYW5nZShwcmVkaWN0aW9uX2NvbHVtbnNfYW1vdW50KQogICAgICAgICAgICBdCgogICAgIyBWYWxpZGF0ZSB0aGUgbGFiZWwgY29sdW1uczoKICAgIGlmIHByZWRpY3Rpb25fY29sdW1uc19hbW91bnQgIT0gbGVuKGxhYmVsX2NvbHVtbnMpOgogICAgICAgICMgTm8gZXF1YWxpdHkgYmV0d2VlbiBwcm92aWRlZCBsYWJlbCBjb2x1bW4gbmFtZXMgYW5kIG91dHB1dHMgYW1vdW50OgogICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICBmIlRoZSBudW1iZXIgb2YgcHJlZGljdGVkIGxhYmVsczoge3ByZWRpY3Rpb25fY29sdW1uc19hbW91bnR9ICIKICAgICAgICAgICAgZiJpcyBub3QgZXF1YWwgdG8gdGhlIGdpdmVuIGxhYmVsIGNvbHVtbnM6IHtsZW4obGFiZWxfY29sdW1ucyl9IgogICAgICAgICkKICAgIGNvbW1vbl9sYWJlbHMgPSBzZXQobGFiZWxfY29sdW1ucykgJiBzZXQoeC5jb2x1bW5zLnRvbGlzdCgpKQogICAgaWYgY29tbW9uX2xhYmVsczoKICAgICAgICAjIExhYmVsIGNvbHVtbiBleGlzdCBpbiB0aGUgb3JpZ2luYWwgaW5wdXRzOgogICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICBmIlRoZSBsYWJlbHM6IHtjb21tb25fbGFiZWxzfSBhcmUgYWxyZWFkeSBleGlzdGVkIGluIHRoZSBnaXZlbiBkYXRhc2V0LiIKICAgICAgICApCgogICAgcmV0dXJuIHBkLmNvbmNhdCgKICAgICAgICBbeCwgcGQuRGF0YUZyYW1lKHlfcHJlZCwgY29sdW1ucz1sYWJlbF9jb2x1bW5zLCBpbmRleD14LmluZGV4KV0sIGF4aXM9MQogICAgKQoKCmRlZiBpbmZlcigKICAgIGNvbnRleHQ6IG1scnVuLk1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogVW5pb25bbWxydW4uRGF0YUl0ZW0sIGxpc3QsIGRpY3QsIHBkLkRhdGFGcmFtZSwgcGQuU2VyaWVzLCBucC5uZGFycmF5XSwKICAgIG1vZGVsX3BhdGg6IFVuaW9uW3N0ciwgbWxydW4uRGF0YUl0ZW1dLAogICAgZHJvcF9jb2x1bW5zOiBVbmlvbltzdHIsIExpc3Rbc3RyXSwgaW50LCBMaXN0W2ludF1dID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IFVuaW9uW3N0ciwgTGlzdFtzdHJdXSA9IE5vbmUsCiAgICBmZWF0dXJlX2NvbHVtbnM6IFVuaW9uW3N0ciwgTGlzdFtzdHJdXSA9IE5vbmUsCiAgICBsb2dfcmVzdWx0X3NldDogYm9vbCA9IFRydWUsCiAgICByZXN1bHRfc2V0X25hbWU6IHN0ciA9ICJwcmVkaWN0aW9uIiwKICAgIGJhdGNoX2lkOiBzdHIgPSBOb25lLAogICAgYXJ0aWZhY3RzX3RhZzogc3RyID0gIiIsCiAgICAjIERyaWZ0IGFuYWx5c2lzIHBhcmFtZXRlcnMKICAgIHBlcmZvcm1fZHJpZnRfYW5hbHlzaXM6IGJvb2wgPSBOb25lLAogICAgdHJpZ2dlcl9tb25pdG9yaW5nX2pvYjogYm9vbCA9IEZhbHNlLAogICAgYmF0Y2hfaW1hZ2Vfam9iOiBzdHIgPSAibWxydW4vbWxydW4iLAogICAgZW5kcG9pbnRfaWQ6IHN0ciA9ICIiLAogICAgIyBUaGUgZm9sbG93aW5nIG1vZGVsIGVuZHBvaW50IHBhcmFtZXRlcnMgYXJlIHJlbGV2YW50IG9ubHkgaWY6CiAgICAjIHBlcmZvcm0gZHJpZnQgYW5hbHlzaXMgaXMgbm90IGRpc2FibGVkCiAgICAjIGEgbmV3IG1vZGVsIGVuZHBvaW50IHJlY29yZCBpcyBnb2luZyB0byBiZSBnZW5lcmF0ZWQKICAgIG1vZGVsX2VuZHBvaW50X25hbWU6IHN0ciA9ICJiYXRjaC1pbmZlciIsCiAgICBtb2RlbF9lbmRwb2ludF9kcmlmdF90aHJlc2hvbGQ6IGZsb2F0ID0gMC43LAogICAgbW9kZWxfZW5kcG9pbnRfcG9zc2libGVfZHJpZnRfdGhyZXNob2xkOiBmbG9hdCA9IDAuNSwKICAgIG1vZGVsX2VuZHBvaW50X3NhbXBsZV9zZXQ6IFVuaW9uWwogICAgICAgIG1scnVuLkRhdGFJdGVtLCBsaXN0LCBkaWN0LCBwZC5EYXRhRnJhbWUsIHBkLlNlcmllcywgbnAubmRhcnJheQogICAgXSA9IE5vbmUsCiAgICAqKnByZWRpY3Rfa3dhcmdzOiBEaWN0W3N0ciwgQW55XSwKKToKICAgICIiIgogICAgUGVyZm9ybSBhIHByZWRpY3Rpb24gb24gYSBnaXZlbiBkYXRhc2V0IHdpdGggdGhlIGdpdmVuIG1vZGVsLiBQbGVhc2UgbWFrZSBzdXJlIHRoYXQgeW91IGhhdmUgYWxyZWFkeSBsb2dnZWQgdGhlIG1vZGVsCiAgICB1bmRlciB0aGUgY3VycmVudCBwcm9qZWN0LgogICAgQ2FuIHBlcmZvcm0gZHJpZnQgYW5hbHlzaXMgYmV0d2VlbiB0aGUgc2FtcGxlIHNldCBzdGF0aXN0aWNzIHN0b3JlZCBpbiB0aGUgbW9kZWwgdG8gdGhlIGN1cnJlbnQgaW5wdXQgZGF0YS4gVGhlCiAgICBkcmlmdCBydWxlIGlzIHRoZSB2YWx1ZSBwZXItZmVhdHVyZSBtZWFuIG9mIHRoZSBUVkQgYW5kIEhlbGxpbmdlciBzY29yZXMgYWNjb3JkaW5nIHRvIHRoZSB0aHJlc2hvbGRzIGNvbmZpZ3VyZXMKICAgIGhlcmUuIFdoZW4gcGVyZm9ybWluZyBkcmlmdCBhbmFseXNpcywgdGhpcyBmdW5jdGlvbiBlaXRoZXIgdXNlcyBhbiBleGlzdGluZyBtb2RlbCBlbmRwb2ludCByZWNvcmQgb3IgY3JlYXRlcwogICAgYSBuZXcgb25lLgogICAgQXQgdGhlIG1vbWVudCwgdGhpcyBmdW5jdGlvbiBpcyBzdXBwb3J0ZWQgZm9yIGBtbHJ1bj49MS41LjBgIHZlcnNpb25zLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBUaGUgZGF0YXNldCB0byBpbmZlciB0aHJvdWdoIHRoZSBtb2RlbC4gUHJvdmlkZWQgYXMgYW4gaW5wdXQgKERhdGFJdGVtKQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhhdCByZXByZXNlbnRzIERhdGFzZXQgYXJ0aWZhY3QgLyBGZWF0dXJlIHZlY3RvciBVUkkuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiB1c2luZyBNTFJ1biBTREssIGBkYXRhc2V0YCBjYW4gYWxzbyBiZSBwcm92aWRlZCBhcyBhIGxpc3QsIGRpY3Rpb25hcnkgb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG51bXB5IGFycmF5LgogICAgOnBhcmFtIG1vZGVsX3BhdGg6ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgTW9kZWwgc3RvcmUgdXJpIChzaG91bGQgc3RhcnQgd2l0aCBzdG9yZTovLykuIFByb3ZpZGVkIGFzIGFuIGlucHV0IChEYXRhSXRlbSkuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiB1c2luZyBNTFJ1biBTREssIGBtb2RlbF9wYXRoYCBjYW4gYWxzbyBiZSBwcm92aWRlZCBhcyBhIHBhcmFtZXRlciAoc3RyaW5nKS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFRvIGdlbmVyYXRlIGEgdmFsaWQgbW9kZWwgc3RvcmUgVVJJLCBwbGVhc2UgbG9nIHRoZSBtb2RlbCBiZWZvcmUgcnVubmluZyB0aGlzIGZ1bmN0aW9uLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgYGVuZHBvaW50X2lkYCBvZiBleGlzdGluZyBtb2RlbCBlbmRwb2ludCBpcyBwcm92aWRlZCwgbWFrZSBzdXJlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0aGF0IGl0IGhhcyBhIHNpbWlsYXIgbW9kZWwgc3RvcmUgcGF0aCwgb3RoZXJ3aXNlIHRoZSBkcmlmdCBhbmFseXNpcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgd29uJ3QgYmUgdHJpZ2dlcmVkLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICAgICAgICAgICAgICAgICAgQSBzdHJpbmcgLyBpbnRlZ2VyIG9yIGEgbGlzdCBvZiBzdHJpbmdzIC8gaW50ZWdlcnMgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbiBuYW1lcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLyBpbmRpY2VzIHRvIGRyb3AuIFdoZW4gdGhlIGRhdGFzZXQgaXMgYSBsaXN0IG9yIGEgbnVtcHkgYXJyYXkgdGhpcyBwYXJhbWV0ZXIgbXVzdAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYmUgcmVwcmVzZW50ZWQgYnkgaW50ZWdlcnMuCiAgICA6cGFyYW0gbGFiZWxfY29sdW1uczogICAgICAgICAgICAgICAgICAgICAgICAgICBUaGUgdGFyZ2V0IGxhYmVsKHMpIG9mIHRoZSBjb2x1bW4ocykgaW4gdGhlIGRhdGFzZXQgZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLiBUaGUgbGFiZWwgY29sdW1uIGNhbiBiZSBhY2Nlc3NlZCBmcm9tIHRoZSBtb2RlbCBvYmplY3QsIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0aGUgZmVhdHVyZSB2ZWN0b3IgcHJvdmlkZWQgaWYgYXZhaWxhYmxlLgogICAgOnBhcmFtIGZlYXR1cmVfY29sdW1uczogICAgICAgICAgICAgICAgICAgICAgICAgTGlzdCBvZiBmZWF0dXJlIGNvbHVtbnMgdGhhdCB3aWxsIGJlIHVzZWQgdG8gYnVpbGQgdGhlIGRhdGFmcmFtZSB3aGVuIGRhdGFzZXQgaXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGZyb20gdHlwZSBsaXN0IG9yIG51bXB5IGFycmF5LgogICAgOnBhcmFtIGxvZ19yZXN1bHRfc2V0OiAgICAgICAgICAgICAgICAgICAgICAgICAgV2hldGhlciB0byBsb2cgdGhlIHJlc3VsdCBzZXQgLSBhIERhdGFGcmFtZSBvZiB0aGUgZ2l2ZW4gaW5wdXRzIGNvbmNhdGVuYXRlZCB3aXRoCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0aGUgcHJlZGljdGlvbnMuIERlZmF1bHRlZCB0byBUcnVlLgogICAgOnBhcmFtIHJlc3VsdF9zZXRfbmFtZTogICAgICAgICAgICAgICAgICAgICAgICAgVGhlIGRiIGtleSB0byBzZXQgbmFtZSBvZiB0aGUgcHJlZGljdGlvbiByZXN1bHQgYW5kIHRoZSBmaWxlbmFtZS4gRGVmYXVsdGVkIHRvCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAncHJlZGljdGlvbicuCiAgICA6cGFyYW0gYmF0Y2hfaWQ6ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBUaGUgSUQgb2YgdGhlIGdpdmVuIGJhdGNoIChpbmZlcmVuY2UgZGF0YXNldCkuIElmIGBOb25lYCwgaXQgd2lsbCBiZSBnZW5lcmF0ZWQuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXaWxsIGJlIGxvZ2dlZCBhcyBhIHJlc3VsdCBvZiB0aGUgcnVuLgogICAgOnBhcmFtIGFydGlmYWN0c190YWc6ICAgICAgICAgICAgICAgICAgICAgICAgICAgVGFnIHRvIHVzZSBmb3IgYWxsIHRoZSBhcnRpZmFjdHMgcmVzdWx0ZWQgZnJvbSB0aGUgZnVuY3Rpb24gKHJlc3VsdCBzZXQgYW5kCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBtb2RlbCBtb25pdG9yaW5nIGFydGlmYWN0cykKICAgIDpwYXJhbSBwZXJmb3JtX2RyaWZ0X2FuYWx5c2lzOiAgICAgICAgICAgICAgICAgIFdoZXRoZXIgdG8gcGVyZm9ybSBkcmlmdCBhbmFseXNpcyBiZXR3ZWVuIHRoZSBzYW1wbGUgc2V0IG9mIHRoZSBtb2RlbCBvYmplY3QgdG8gdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBkYXRhc2V0IGdpdmVuLiBCeSBkZWZhdWx0LCBOb25lLCB3aGljaCBtZWFucyBpdCB3aWxsIHBlcmZvcm0gZHJpZnQgYW5hbHlzaXMgaWYgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBtb2RlbCBhbHJlYWR5IGhhcyBmZWF0dXJlIHN0YXRzIHRoYXQgYXJlIGNvbnNpZGVyZWQgYXMgYSByZWZlcmVuY2Ugc2FtcGxlIHNldC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFBlcmZvcm1pbmcgZHJpZnQgYW5hbHlzaXMgb24gYSBuZXcgZW5kcG9pbnQgaWQgd2lsbCBnZW5lcmF0ZSBhIG5ldyBtb2RlbCBlbmRwb2ludAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcmVjb3JkLiBQbGVhc2Ugbm90ZSB0aGF0IGluIG9yZGVyIHRvIHRyaWdnZXIgdGhlIGRyaWZ0IGFuYWx5c2lzIGpvYiwgeW91IG5lZWQgdG8KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNldCBgdHJpZ2dlcl9tb25pdG9yaW5nX2pvYj1UcnVlYC4gT3RoZXJ3aXNlLCB0aGUgZHJpZnQgYW5hbHlzaXMgd2lsbCBiZSB0cmlnZ2VyZWQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9ubHkgYXMgcGFydCB0aGUgc2NoZWR1bGVkIG1vbml0b3Jpbmcgam9iIChpZiBleGlzdCBpbiB0aGUgY3VycmVudCBwcm9qZWN0KSBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaWYgdHJpZ2dlcmVkIG1hbnVhbGx5IGJ5IHRoZSB1c2VyLgogICAgOnBhcmFtIHRyaWdnZXJfbW9uaXRvcmluZ19qb2I6ICAgICAgICAgICAgICAgICAgV2hldGhlciB0byB0cmlnZ2VyIHRoZSBiYXRjaCBkcmlmdCBhbmFseXNpcyBhZnRlciB0aGUgaW5mZXIgam9iLgogICAgOnBhcmFtIGJhdGNoX2ltYWdlX2pvYjogICAgICAgICAgICAgICAgICAgICAgICAgVGhlIGltYWdlIHRoYXQgd2lsbCBiZSB1c2VkIHRvIHJlZ2lzdGVyIHRoZSBtb25pdG9yaW5nIGJhdGNoIGpvYiBpZiBub3QgZXhpc3QuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBCeSBkZWZhdWx0LCB0aGUgaW1hZ2UgaXMgbWxydW4vbWxydW4uCiAgICA6cGFyYW0gZW5kcG9pbnRfaWQ6ICAgICAgICAgICAgICAgICAgICAgICAgICAgICBNb2RlbCBlbmRwb2ludCB1bmlxdWUgSUQuIElmIGBwZXJmb3JtX2RyaWZ0X2FuYWx5c2lzYCB3YXMgc2V0LCB0aGUgZW5kcG9pbnRfaWQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHdpbGwgYmUgdXNlZCBlaXRoZXIgdG8gcGVyZm9ybSB0aGUgYW5hbHlzaXMgb24gZXhpc3RpbmcgbW9kZWwgZW5kcG9pbnQgb3IgdG8KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGdlbmVyYXRlIGEgbmV3IG1vZGVsIGVuZHBvaW50IHJlY29yZC4KICAgIDpwYXJhbSBtb2RlbF9lbmRwb2ludF9uYW1lOiAgICAgICAgICAgICAgICAgICAgIElmIGEgbmV3IG1vZGVsIGVuZHBvaW50IGlzIGdlbmVyYXRlZCwgdGhlIG1vZGVsIG5hbWUgd2lsbCBiZSBwcmVzZW50ZWQgdW5kZXIgdGhpcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZW5kcG9pbnQuCiAgICA6cGFyYW0gbW9kZWxfZW5kcG9pbnRfZHJpZnRfdGhyZXNob2xkOiAgICAgICAgICBUaGUgdGhyZXNob2xkIG9mIHdoaWNoIHRvIG1hcmsgZHJpZnRzLiBEZWZhdWx0ZWQgdG8gMC43LgogICAgOnBhcmFtIG1vZGVsX2VuZHBvaW50X3Bvc3NpYmxlX2RyaWZ0X3RocmVzaG9sZDogVGhlIHRocmVzaG9sZCBvZiB3aGljaCB0byBtYXJrIHBvc3NpYmxlIGRyaWZ0cy4gRGVmYXVsdGVkIHRvIDAuNS4KICAgIDpwYXJhbSBtb2RlbF9lbmRwb2ludF9zYW1wbGVfc2V0OiAgICAgICAgICAgICAgIEEgc2FtcGxlIGRhdGFzZXQgdG8gZ2l2ZSB0byBjb21wYXJlIHRoZSBpbnB1dHMgaW4gdGhlIGRyaWZ0IGFuYWx5c2lzLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQ2FuIGJlIHByb3ZpZGVkIGFzIGFuIGlucHV0IChEYXRhSXRlbSkgb3IgYXMgYSBwYXJhbWV0ZXIgKGUuZy4gc3RyaW5nLCBsaXN0LCBEYXRhRnJhbWUpLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgVGhlIGRlZmF1bHQgY2hvc2VuIHNhbXBsZSBzZXQgd2lsbCBhbHdheXMgYmUgdGhlIG9uZSB3aG8gaXMgc2V0IGluIHRoZSBtb2RlbCBhcnRpZmFjdCBpdHNlbGYuCgogICAgcmFpc2VzIE1MUnVuSW52YWxpZEFyZ3VtZW50RXJyb3I6IGlmIGJvdGggYG1vZGVsX3BhdGhgIGFuZCBgZW5kcG9pbnRfaWRgIGFyZSBub3QgcHJvdmlkZWQKICAgICIiIgoKICAgICMgTG9hZGluZyB0aGUgbW9kZWw6CiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiTG9hZGluZyBtb2RlbC4uLiIpCiAgICBpZiBpc2luc3RhbmNlKG1vZGVsX3BhdGgsIG1scnVuLkRhdGFJdGVtKToKICAgICAgICBtb2RlbF9wYXRoID0gbW9kZWxfcGF0aC5hcnRpZmFjdF91cmwKICAgIGlmIG5vdCBtbHJ1bi5kYXRhc3RvcmUuaXNfc3RvcmVfdXJpKG1vZGVsX3BhdGgpOgogICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICBmIlRoZSBwcm92aWRlZCBtb2RlbCBwYXRoICh7bW9kZWxfcGF0aH0pIGlzIGludmFsaWQgLSBzaG91bGQgc3RhcnQgd2l0aCBgc3RvcmU6Ly9gLiAiCiAgICAgICAgICAgIGYiUGxlYXNlIG1ha2Ugc3VyZSB0aGF0IHlvdSBoYXZlIGxvZ2dlZCB0aGUgbW9kZWwgdXNpbmcgYHByb2plY3QubG9nX21vZGVsKClgICIKICAgICAgICAgICAgZiJ3aGljaCBnZW5lcmF0ZXMgYSB1bmlxdWUgc3RvcmUgdXJpIGZvciB0aGUgbG9nZ2VkIG1vZGVsLiIKICAgICAgICApCiAgICBtb2RlbF9oYW5kbGVyID0gQXV0b01MUnVuLmxvYWRfbW9kZWwobW9kZWxfcGF0aD1tb2RlbF9wYXRoLCBjb250ZXh0PWNvbnRleHQpCgogICAgaWYgbGFiZWxfY29sdW1ucyBpcyBOb25lOgogICAgICAgIGxhYmVsX2NvbHVtbnMgPSBbCiAgICAgICAgICAgIG91dHB1dC5uYW1lIGZvciBvdXRwdXQgaW4gbW9kZWxfaGFuZGxlci5fbW9kZWxfYXJ0aWZhY3Quc3BlYy5vdXRwdXRzCiAgICAgICAgXQoKICAgIGlmIGZlYXR1cmVfY29sdW1ucyBpcyBOb25lOgogICAgICAgIGZlYXR1cmVfY29sdW1ucyA9IFsKICAgICAgICAgICAgaW5wdXQubmFtZSBmb3IgaW5wdXQgaW4gbW9kZWxfaGFuZGxlci5fbW9kZWxfYXJ0aWZhY3Quc3BlYy5pbnB1dHMKICAgICAgICBdCgogICAgIyBHZXQgZGF0YXNldCBieSBvYmplY3QsIFVSTCBvciBieSBGZWF0dXJlVmVjdG9yOgogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmIkxvYWRpbmcgZGF0YS4uLiIpCiAgICB4LCBsYWJlbF9jb2x1bW5zID0gbWxydW4ubW9kZWxfbW9uaXRvcmluZy5hcGkucmVhZF9kYXRhc2V0X2FzX2RhdGFmcmFtZSgKICAgICAgICBkYXRhc2V0PWRhdGFzZXQsCiAgICAgICAgZmVhdHVyZV9jb2x1bW5zPWZlYXR1cmVfY29sdW1ucywKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIFByZWRpY3Q6CiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiQ2FsY3VsYXRpbmcgcHJlZGljdGlvbi4uLiIpCiAgICB5X3ByZWQgPSBtb2RlbF9oYW5kbGVyLm1vZGVsLnByZWRpY3QoeCwgKipwcmVkaWN0X2t3YXJncykKCiAgICAjIFByZXBhcmUgdGhlIHJlc3VsdCBzZXQ6CiAgICByZXN1bHRfc2V0ID0gX3ByZXBhcmVfcmVzdWx0X3NldCh4PXgsIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywgeV9wcmVkPXlfcHJlZCkKCiAgICAjIENoZWNrIGZvciBsb2dnaW5nIHRoZSByZXN1bHQgc2V0OgogICAgaWYgbG9nX3Jlc3VsdF9zZXQ6CiAgICAgICAgbWxydW4ubW9kZWxfbW9uaXRvcmluZy5hcGkubG9nX3Jlc3VsdCgKICAgICAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgICAgICByZXN1bHRfc2V0X25hbWU9cmVzdWx0X3NldF9uYW1lLAogICAgICAgICAgICByZXN1bHRfc2V0PXJlc3VsdF9zZXQsCiAgICAgICAgICAgIGFydGlmYWN0c190YWc9YXJ0aWZhY3RzX3RhZywKICAgICAgICAgICAgYmF0Y2hfaWQ9YmF0Y2hfaWQsCiAgICAgICAgKQoKICAgICMgQ2hlY2sgZm9yIHBlcmZvcm1pbmcgZHJpZnQgYW5hbHlzaXMKICAgIGlmICgKICAgICAgICBwZXJmb3JtX2RyaWZ0X2FuYWx5c2lzIGlzIE5vbmUKICAgICAgICBhbmQgbW9kZWxfaGFuZGxlci5fbW9kZWxfYXJ0aWZhY3Quc3BlYy5mZWF0dXJlX3N0YXRzIGlzIG5vdCBOb25lCiAgICApOgogICAgICAgIHBlcmZvcm1fZHJpZnRfYW5hbHlzaXMgPSBUcnVlCiAgICBpZiBwZXJmb3JtX2RyaWZ0X2FuYWx5c2lzOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oIlBlcmZvcm1pbmcgZHJpZnQgYW5hbHlzaXMuLi4iKQogICAgICAgICMgR2V0IHRoZSBzYW1wbGUgc2V0IHN0YXRpc3RpY3MgKGVpdGhlciBmcm9tIHRoZSBzYW1wbGUgc2V0IG9yIGZyb20gdGhlIHN0YXRpc3RpY3MgbG9nZ2VkIHdpdGggdGhlIG1vZGVsKQogICAgICAgIHNhbXBsZV9zZXRfc3RhdGlzdGljcyA9IG1scnVuLm1vZGVsX21vbml0b3JpbmcuYXBpLmdldF9zYW1wbGVfc2V0X3N0YXRpc3RpY3MoCiAgICAgICAgICAgIHNhbXBsZV9zZXQ9bW9kZWxfZW5kcG9pbnRfc2FtcGxlX3NldCwKICAgICAgICAgICAgbW9kZWxfYXJ0aWZhY3RfZmVhdHVyZV9zdGF0cz1tb2RlbF9oYW5kbGVyLl9tb2RlbF9hcnRpZmFjdC5zcGVjLmZlYXR1cmVfc3RhdHMsCiAgICAgICAgKQogICAgICAgIG1scnVuLm1vZGVsX21vbml0b3JpbmcuYXBpLnJlY29yZF9yZXN1bHRzKAogICAgICAgICAgICBwcm9qZWN0PWNvbnRleHQucHJvamVjdCwKICAgICAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgICAgICBlbmRwb2ludF9pZD1lbmRwb2ludF9pZCwKICAgICAgICAgICAgbW9kZWxfcGF0aD1tb2RlbF9wYXRoLAogICAgICAgICAgICBtb2RlbF9lbmRwb2ludF9uYW1lPW1vZGVsX2VuZHBvaW50X25hbWUsCiAgICAgICAgICAgIGluZmVyX3Jlc3VsdHNfZGY9cmVzdWx0X3NldC5jb3B5KCksCiAgICAgICAgICAgIHNhbXBsZV9zZXRfc3RhdGlzdGljcz1zYW1wbGVfc2V0X3N0YXRpc3RpY3MsCiAgICAgICAgICAgIGRyaWZ0X3RocmVzaG9sZD1tb2RlbF9lbmRwb2ludF9kcmlmdF90aHJlc2hvbGQsCiAgICAgICAgICAgIHBvc3NpYmxlX2RyaWZ0X3RocmVzaG9sZD1tb2RlbF9lbmRwb2ludF9wb3NzaWJsZV9kcmlmdF90aHJlc2hvbGQsCiAgICAgICAgICAgIGFydGlmYWN0c190YWc9YXJ0aWZhY3RzX3RhZywKICAgICAgICAgICAgdHJpZ2dlcl9tb25pdG9yaW5nX2pvYj10cmlnZ2VyX21vbml0b3Jpbmdfam9iLAogICAgICAgICAgICBkZWZhdWx0X2JhdGNoX2ltYWdlPWJhdGNoX2ltYWdlX2pvYiwKICAgICAgICApCg== + functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpmcm9tIGluc3BlY3QgaW1wb3J0IHNpZ25hdHVyZQpmcm9tIHR5cGluZyBpbXBvcnQgQW55LCBEaWN0LCBMaXN0LCBPcHRpb25hbCwgVW5pb24KCmltcG9ydCBtbHJ1bgoKdHJ5OgogICAgaW1wb3J0IG1scnVuLm1vZGVsX21vbml0b3JpbmcuYXBpCmV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yOgogICAgcmFpc2UgbWxydW4uZXJyb3JzLk1MUnVuTm90Rm91bmRFcnJvcigKICAgICAgICBmIlBsZWFzZSB1cGRhdGUgeW91ciBgbWxydW5gIHZlcnNpb24gdG8gPj0xLjUuMCBvciB1c2UgYW4gIgogICAgICAgIGYib2xkZXIgdmVyc2lvbiBvZiB0aGUgYmF0Y2ggaW5mZXJlbmNlIGZ1bmN0aW9uLiIKICAgICkKCgppbXBvcnQgbnVtcHkgYXMgbnAKaW1wb3J0IHBhbmRhcyBhcyBwZApmcm9tIG1scnVuLmZyYW1ld29ya3MuYXV0b19tbHJ1biBpbXBvcnQgQXV0b01MUnVuCgoKZGVmIF9wcmVwYXJlX3Jlc3VsdF9zZXQoCiAgICB4OiBwZC5EYXRhRnJhbWUsIGxhYmVsX2NvbHVtbnM6IExpc3Rbc3RyXSwgeV9wcmVkOiBucC5uZGFycmF5CikgLT4gcGQuRGF0YUZyYW1lOgogICAgIiIiCiAgICBTZXQgZGVmYXVsdCBsYWJlbCBjb2x1bW4gbmFtZXMgYW5kIHZhbGlkYXRlIGdpdmVuIG5hbWVzIHRvIHByZXBhcmUgdGhlIHJlc3VsdCBzZXQgLSBhIGNvbmNhdGVuYXRpb24gb2YgdGhlIGlucHV0cwogICAgKHgpIGFuZCB0aGUgbW9kZWwgcHJlZGljdGlvbnMgKHlfcHJlZCkuCgogICAgOnBhcmFtIHg6ICAgICAgICAgICAgIFRoZSBpbnB1dHMuCiAgICA6cGFyYW0gbGFiZWxfY29sdW1uczogQSBsaXN0IG9mIHN0cmluZ3MgcmVwcmVzZW50aW5nIHRoZSB0YXJnZXQgY29sdW1uIG5hbWVzIHRvIGFkZCB0byB0aGUgcHJlZGljdGlvbnMuIERlZmF1bHQgbmFtZQogICAgICAgICAgICAgICAgICAgICAgICAgIHdpbGwgYmUgdXNlZCBpbiBjYXNlIHRoZSBsaXN0IGlzIGVtcHR5IChwcmVkaWN0ZWRfbGFiZWxfe2l9KS4KICAgIDpwYXJhbSB5X3ByZWQ6ICAgICAgICBUaGUgbW9kZWwgcHJlZGljdGlvbnMgb24gdGhlIGlucHV0cy4KCiAgICA6cmV0dXJuczogVGhlIHJlc3VsdCBzZXQuCgogICAgcmFpc2VzIE1MUnVuSW52YWxpZEFyZ3VtZW50RXJyb3I6IElmIHRoZSBsYWJlbHMgY29sdW1ucyBhbW91bnQgZG8gbm90IG1hdGNoIHRoZSBvdXRwdXRzIG9yIGlmIG9uZSBvZiB0aGUgbGFiZWwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgY29sdW1uIGFscmVhZHkgZXhpc3RzIGluIHRoZSBkYXRhc2V0LgogICAgIiIiCiAgICAjIFByZXBhcmUgZGVmYXVsdCB0YXJnZXQgY29sdW1ucyBuYW1lcyBpZiBub3QgcHJvdmlkZWQ6CiAgICBwcmVkaWN0aW9uX2NvbHVtbnNfYW1vdW50ID0gMSBpZiBsZW4oeV9wcmVkLnNoYXBlKSA9PSAxIGVsc2UgeV9wcmVkLnNoYXBlWzFdCiAgICBpZiBsZW4obGFiZWxfY29sdW1ucykgPT0gMDoKICAgICAgICAjIEFkZCBkZWZhdWx0IGxhYmVsIGNvbHVtbiBuYW1lczoKICAgICAgICBpZiBwcmVkaWN0aW9uX2NvbHVtbnNfYW1vdW50ID09IDE6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBbInByZWRpY3RlZF9sYWJlbCJdCiAgICAgICAgZWxzZToKICAgICAgICAgICAgbGFiZWxfY29sdW1ucyA9IFsKICAgICAgICAgICAgICAgIGYicHJlZGljdGVkX2xhYmVsX3tpfSIgZm9yIGkgaW4gcmFuZ2UocHJlZGljdGlvbl9jb2x1bW5zX2Ftb3VudCkKICAgICAgICAgICAgXQoKICAgICMgVmFsaWRhdGUgdGhlIGxhYmVsIGNvbHVtbnM6CiAgICBpZiBwcmVkaWN0aW9uX2NvbHVtbnNfYW1vdW50ICE9IGxlbihsYWJlbF9jb2x1bW5zKToKICAgICAgICAjIE5vIGVxdWFsaXR5IGJldHdlZW4gcHJvdmlkZWQgbGFiZWwgY29sdW1uIG5hbWVzIGFuZCBvdXRwdXRzIGFtb3VudDoKICAgICAgICByYWlzZSBtbHJ1bi5lcnJvcnMuTUxSdW5JbnZhbGlkQXJndW1lbnRFcnJvcigKICAgICAgICAgICAgZiJUaGUgbnVtYmVyIG9mIHByZWRpY3RlZCBsYWJlbHM6IHtwcmVkaWN0aW9uX2NvbHVtbnNfYW1vdW50fSAiCiAgICAgICAgICAgIGYiaXMgbm90IGVxdWFsIHRvIHRoZSBnaXZlbiBsYWJlbCBjb2x1bW5zOiB7bGVuKGxhYmVsX2NvbHVtbnMpfSIKICAgICAgICApCiAgICBjb21tb25fbGFiZWxzID0gc2V0KGxhYmVsX2NvbHVtbnMpICYgc2V0KHguY29sdW1ucy50b2xpc3QoKSkKICAgIGlmIGNvbW1vbl9sYWJlbHM6CiAgICAgICAgIyBMYWJlbCBjb2x1bW4gZXhpc3QgaW4gdGhlIG9yaWdpbmFsIGlucHV0czoKICAgICAgICByYWlzZSBtbHJ1bi5lcnJvcnMuTUxSdW5JbnZhbGlkQXJndW1lbnRFcnJvcigKICAgICAgICAgICAgZiJUaGUgbGFiZWxzOiB7Y29tbW9uX2xhYmVsc30gYXJlIGFscmVhZHkgZXhpc3RlZCBpbiB0aGUgZ2l2ZW4gZGF0YXNldC4iCiAgICAgICAgKQoKICAgIHJldHVybiBwZC5jb25jYXQoCiAgICAgICAgW3gsIHBkLkRhdGFGcmFtZSh5X3ByZWQsIGNvbHVtbnM9bGFiZWxfY29sdW1ucywgaW5kZXg9eC5pbmRleCldLCBheGlzPTEKICAgICkKCgpkZWYgX3BhcnNlX3JlY29yZF9yZXN1bHRzX2t3YXJnKAogICAgbGFzdF9pbl9iYXRjaF9zZXQ6IE9wdGlvbmFsW2Jvb2xdLAopIC0+IGRpY3Rbc3RyLCBib29sXToKICAgICIiIgogICAgQ2hlY2sgaWYgYGxhc3RfaW5fYmF0Y2hfc2V0YCBpcyBwcm92aWRlZCBhbmQgZXhwZWN0ZWQgYXMgYSBwYXJhbWV0ZXIuCiAgICBSZXR1cm4gaXQgYXMgYSBkaWN0aW9uYXJ5LgogICAgIiIiCiAgICBrd2FyZyA9ICJsYXN0X2luX2JhdGNoX3NldCIKICAgIGlmIGxhc3RfaW5fYmF0Y2hfc2V0IGlzIE5vbmU6CiAgICAgICAgcmV0dXJuIHt9CiAgICBpZiAoCiAgICAgICAgc2lnbmF0dXJlKG1scnVuLm1vZGVsX21vbml0b3JpbmcuYXBpLnJlY29yZF9yZXN1bHRzKS5wYXJhbWV0ZXJzLmdldChrd2FyZykKICAgICAgICBpcyBOb25lCiAgICApOgogICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICBmIlVuZXhwZWN0ZWQgcGFyYW1ldGVyIGB7a3dhcmd9YCBmb3IgZnVuY3Rpb246ICIKICAgICAgICAgICAgImBtbHJ1bi5tb2RlbF9tb25pdG9yaW5nLmFwaS5yZWNvcmRfcmVzdWx0c2AuICIKICAgICAgICAgICAgIlBsZWFzZSBtYWtlIHN1cmUgdGhhdCB5b3UgYXJlIHVzaW5nIGBtbHJ1bj49MS42LjBgIHZlcnNpb24uIgogICAgICAgICkKICAgIHJldHVybiB7a3dhcmc6IGxhc3RfaW5fYmF0Y2hfc2V0fQoKCmRlZiBpbmZlcigKICAgIGNvbnRleHQ6IG1scnVuLk1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogVW5pb25bbWxydW4uRGF0YUl0ZW0sIGxpc3QsIGRpY3QsIHBkLkRhdGFGcmFtZSwgcGQuU2VyaWVzLCBucC5uZGFycmF5XSwKICAgIG1vZGVsX3BhdGg6IFVuaW9uW3N0ciwgbWxydW4uRGF0YUl0ZW1dLAogICAgZHJvcF9jb2x1bW5zOiBVbmlvbltzdHIsIExpc3Rbc3RyXSwgaW50LCBMaXN0W2ludF1dID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IFVuaW9uW3N0ciwgTGlzdFtzdHJdXSA9IE5vbmUsCiAgICBmZWF0dXJlX2NvbHVtbnM6IFVuaW9uW3N0ciwgTGlzdFtzdHJdXSA9IE5vbmUsCiAgICBsb2dfcmVzdWx0X3NldDogYm9vbCA9IFRydWUsCiAgICByZXN1bHRfc2V0X25hbWU6IHN0ciA9ICJwcmVkaWN0aW9uIiwKICAgIGJhdGNoX2lkOiBzdHIgPSBOb25lLAogICAgYXJ0aWZhY3RzX3RhZzogc3RyID0gIiIsCiAgICAjIERyaWZ0IGFuYWx5c2lzIHBhcmFtZXRlcnMKICAgIHBlcmZvcm1fZHJpZnRfYW5hbHlzaXM6IGJvb2wgPSBOb25lLAogICAgdHJpZ2dlcl9tb25pdG9yaW5nX2pvYjogYm9vbCA9IEZhbHNlLAogICAgYmF0Y2hfaW1hZ2Vfam9iOiBzdHIgPSAibWxydW4vbWxydW4iLAogICAgZW5kcG9pbnRfaWQ6IHN0ciA9ICIiLAogICAgIyBUaGUgZm9sbG93aW5nIG1vZGVsIGVuZHBvaW50IHBhcmFtZXRlcnMgYXJlIHJlbGV2YW50IG9ubHkgaWY6CiAgICAjIHBlcmZvcm0gZHJpZnQgYW5hbHlzaXMgaXMgbm90IGRpc2FibGVkCiAgICAjIGEgbmV3IG1vZGVsIGVuZHBvaW50IHJlY29yZCBpcyBnb2luZyB0byBiZSBnZW5lcmF0ZWQKICAgIG1vZGVsX2VuZHBvaW50X25hbWU6IHN0ciA9ICJiYXRjaC1pbmZlciIsCiAgICBtb2RlbF9lbmRwb2ludF9kcmlmdF90aHJlc2hvbGQ6IGZsb2F0ID0gMC43LAogICAgbW9kZWxfZW5kcG9pbnRfcG9zc2libGVfZHJpZnRfdGhyZXNob2xkOiBmbG9hdCA9IDAuNSwKICAgIG1vZGVsX2VuZHBvaW50X3NhbXBsZV9zZXQ6IFVuaW9uWwogICAgICAgIG1scnVuLkRhdGFJdGVtLCBsaXN0LCBkaWN0LCBwZC5EYXRhRnJhbWUsIHBkLlNlcmllcywgbnAubmRhcnJheQogICAgXSA9IE5vbmUsCiAgICBsYXN0X2luX2JhdGNoX3NldDogT3B0aW9uYWxbYm9vbF0gPSBOb25lLAogICAgKipwcmVkaWN0X2t3YXJnczogRGljdFtzdHIsIEFueV0sCik6CiAgICAiIiIKICAgIFBlcmZvcm0gYSBwcmVkaWN0aW9uIG9uIGEgZ2l2ZW4gZGF0YXNldCB3aXRoIHRoZSBnaXZlbiBtb2RlbC4gUGxlYXNlIG1ha2Ugc3VyZSB0aGF0IHlvdSBoYXZlIGFscmVhZHkgbG9nZ2VkIHRoZSBtb2RlbAogICAgdW5kZXIgdGhlIGN1cnJlbnQgcHJvamVjdC4KICAgIENhbiBwZXJmb3JtIGRyaWZ0IGFuYWx5c2lzIGJldHdlZW4gdGhlIHNhbXBsZSBzZXQgc3RhdGlzdGljcyBzdG9yZWQgaW4gdGhlIG1vZGVsIHRvIHRoZSBjdXJyZW50IGlucHV0IGRhdGEuIFRoZQogICAgZHJpZnQgcnVsZSBpcyB0aGUgdmFsdWUgcGVyLWZlYXR1cmUgbWVhbiBvZiB0aGUgVFZEIGFuZCBIZWxsaW5nZXIgc2NvcmVzIGFjY29yZGluZyB0byB0aGUgdGhyZXNob2xkcyBjb25maWd1cmVzCiAgICBoZXJlLiBXaGVuIHBlcmZvcm1pbmcgZHJpZnQgYW5hbHlzaXMsIHRoaXMgZnVuY3Rpb24gZWl0aGVyIHVzZXMgYW4gZXhpc3RpbmcgbW9kZWwgZW5kcG9pbnQgcmVjb3JkIG9yIGNyZWF0ZXMKICAgIGEgbmV3IG9uZS4KICAgIEF0IHRoZSBtb21lbnQsIHRoaXMgZnVuY3Rpb24gaXMgc3VwcG9ydGVkIGZvciBgbWxydW4+PTEuNS4wYCB2ZXJzaW9ucy4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBNTFJ1biBjb250ZXh0LgogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgVGhlIGRhdGFzZXQgdG8gaW5mZXIgdGhyb3VnaCB0aGUgbW9kZWwuIFByb3ZpZGVkIGFzIGFuIGlucHV0IChEYXRhSXRlbSkKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoYXQgcmVwcmVzZW50cyBEYXRhc2V0IGFydGlmYWN0IC8gRmVhdHVyZSB2ZWN0b3IgVVJJLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgdXNpbmcgTUxSdW4gU0RLLCBgZGF0YXNldGAgY2FuIGFsc28gYmUgcHJvdmlkZWQgYXMgYSBsaXN0LCBkaWN0aW9uYXJ5IG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBudW1weSBhcnJheS4KICAgIDpwYXJhbSBtb2RlbF9wYXRoOiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIE1vZGVsIHN0b3JlIHVyaSAoc2hvdWxkIHN0YXJ0IHdpdGggc3RvcmU6Ly8pLiBQcm92aWRlZCBhcyBhbiBpbnB1dCAoRGF0YUl0ZW0pLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgdXNpbmcgTUxSdW4gU0RLLCBgbW9kZWxfcGF0aGAgY2FuIGFsc28gYmUgcHJvdmlkZWQgYXMgYSBwYXJhbWV0ZXIgKHN0cmluZykuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBUbyBnZW5lcmF0ZSBhIHZhbGlkIG1vZGVsIHN0b3JlIFVSSSwgcGxlYXNlIGxvZyB0aGUgbW9kZWwgYmVmb3JlIHJ1bm5pbmcgdGhpcyBmdW5jdGlvbi4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIElmIGBlbmRwb2ludF9pZGAgb2YgZXhpc3RpbmcgbW9kZWwgZW5kcG9pbnQgaXMgcHJvdmlkZWQsIG1ha2Ugc3VyZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhhdCBpdCBoYXMgYSBzaW1pbGFyIG1vZGVsIHN0b3JlIHBhdGgsIG90aGVyd2lzZSB0aGUgZHJpZnQgYW5hbHlzaXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHdvbid0IGJlIHRyaWdnZXJlZC4KICAgIDpwYXJhbSBkcm9wX2NvbHVtbnM6ICAgICAgICAgICAgICAgICAgICAgICAgICAgIEEgc3RyaW5nIC8gaW50ZWdlciBvciBhIGxpc3Qgb2Ygc3RyaW5ncyAvIGludGVnZXJzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC8gaW5kaWNlcyB0byBkcm9wLiBXaGVuIHRoZSBkYXRhc2V0IGlzIGEgbGlzdCBvciBhIG51bXB5IGFycmF5IHRoaXMgcGFyYW1ldGVyIG11c3QKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGJlIHJlcHJlc2VudGVkIGJ5IGludGVnZXJzLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgICAgICAgICAgICAgICAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0IGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gVGhlIGxhYmVsIGNvbHVtbiBjYW4gYmUgYWNjZXNzZWQgZnJvbSB0aGUgbW9kZWwgb2JqZWN0LCBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhlIGZlYXR1cmUgdmVjdG9yIHByb3ZpZGVkIGlmIGF2YWlsYWJsZS4KICAgIDpwYXJhbSBmZWF0dXJlX2NvbHVtbnM6ICAgICAgICAgICAgICAgICAgICAgICAgIExpc3Qgb2YgZmVhdHVyZSBjb2x1bW5zIHRoYXQgd2lsbCBiZSB1c2VkIHRvIGJ1aWxkIHRoZSBkYXRhZnJhbWUgd2hlbiBkYXRhc2V0IGlzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmcm9tIHR5cGUgbGlzdCBvciBudW1weSBhcnJheS4KICAgIDpwYXJhbSBsb2dfcmVzdWx0X3NldDogICAgICAgICAgICAgICAgICAgICAgICAgIFdoZXRoZXIgdG8gbG9nIHRoZSByZXN1bHQgc2V0IC0gYSBEYXRhRnJhbWUgb2YgdGhlIGdpdmVuIGlucHV0cyBjb25jYXRlbmF0ZWQgd2l0aAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhlIHByZWRpY3Rpb25zLiBEZWZhdWx0ZWQgdG8gVHJ1ZS4KICAgIDpwYXJhbSByZXN1bHRfc2V0X25hbWU6ICAgICAgICAgICAgICAgICAgICAgICAgIFRoZSBkYiBrZXkgdG8gc2V0IG5hbWUgb2YgdGhlIHByZWRpY3Rpb24gcmVzdWx0IGFuZCB0aGUgZmlsZW5hbWUuIERlZmF1bHRlZCB0bwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ3ByZWRpY3Rpb24nLgogICAgOnBhcmFtIGJhdGNoX2lkOiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgVGhlIElEIG9mIHRoZSBnaXZlbiBiYXRjaCAoaW5mZXJlbmNlIGRhdGFzZXQpLiBJZiBgTm9uZWAsIGl0IHdpbGwgYmUgZ2VuZXJhdGVkLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgV2lsbCBiZSBsb2dnZWQgYXMgYSByZXN1bHQgb2YgdGhlIHJ1bi4KICAgIDpwYXJhbSBhcnRpZmFjdHNfdGFnOiAgICAgICAgICAgICAgICAgICAgICAgICAgIFRhZyB0byB1c2UgZm9yIGFsbCB0aGUgYXJ0aWZhY3RzIHJlc3VsdGVkIGZyb20gdGhlIGZ1bmN0aW9uIChyZXN1bHQgc2V0IGFuZAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9kZWwgbW9uaXRvcmluZyBhcnRpZmFjdHMpCiAgICA6cGFyYW0gcGVyZm9ybV9kcmlmdF9hbmFseXNpczogICAgICAgICAgICAgICAgICBXaGV0aGVyIHRvIHBlcmZvcm0gZHJpZnQgYW5hbHlzaXMgYmV0d2VlbiB0aGUgc2FtcGxlIHNldCBvZiB0aGUgbW9kZWwgb2JqZWN0IHRvIHRoZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZGF0YXNldCBnaXZlbi4gQnkgZGVmYXVsdCwgTm9uZSwgd2hpY2ggbWVhbnMgaXQgd2lsbCBwZXJmb3JtIGRyaWZ0IGFuYWx5c2lzIGlmIHRoZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9kZWwgYWxyZWFkeSBoYXMgZmVhdHVyZSBzdGF0cyB0aGF0IGFyZSBjb25zaWRlcmVkIGFzIGEgcmVmZXJlbmNlIHNhbXBsZSBzZXQuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBQZXJmb3JtaW5nIGRyaWZ0IGFuYWx5c2lzIG9uIGEgbmV3IGVuZHBvaW50IGlkIHdpbGwgZ2VuZXJhdGUgYSBuZXcgbW9kZWwgZW5kcG9pbnQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHJlY29yZC4gUGxlYXNlIG5vdGUgdGhhdCBpbiBvcmRlciB0byB0cmlnZ2VyIHRoZSBkcmlmdCBhbmFseXNpcyBqb2IsIHlvdSBuZWVkIHRvCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzZXQgYHRyaWdnZXJfbW9uaXRvcmluZ19qb2I9VHJ1ZWAuIE90aGVyd2lzZSwgdGhlIGRyaWZ0IGFuYWx5c2lzIHdpbGwgYmUgdHJpZ2dlcmVkCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBvbmx5IGFzIHBhcnQgdGhlIHNjaGVkdWxlZCBtb25pdG9yaW5nIGpvYiAoaWYgZXhpc3QgaW4gdGhlIGN1cnJlbnQgcHJvamVjdCkgb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGlmIHRyaWdnZXJlZCBtYW51YWxseSBieSB0aGUgdXNlci4KICAgIDpwYXJhbSB0cmlnZ2VyX21vbml0b3Jpbmdfam9iOiAgICAgICAgICAgICAgICAgIFdoZXRoZXIgdG8gdHJpZ2dlciB0aGUgYmF0Y2ggZHJpZnQgYW5hbHlzaXMgYWZ0ZXIgdGhlIGluZmVyIGpvYi4KICAgIDpwYXJhbSBiYXRjaF9pbWFnZV9qb2I6ICAgICAgICAgICAgICAgICAgICAgICAgIFRoZSBpbWFnZSB0aGF0IHdpbGwgYmUgdXNlZCB0byByZWdpc3RlciB0aGUgbW9uaXRvcmluZyBiYXRjaCBqb2IgaWYgbm90IGV4aXN0LgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQnkgZGVmYXVsdCwgdGhlIGltYWdlIGlzIG1scnVuL21scnVuLgogICAgOnBhcmFtIGVuZHBvaW50X2lkOiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgTW9kZWwgZW5kcG9pbnQgdW5pcXVlIElELiBJZiBgcGVyZm9ybV9kcmlmdF9hbmFseXNpc2Agd2FzIHNldCwgdGhlIGVuZHBvaW50X2lkCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB3aWxsIGJlIHVzZWQgZWl0aGVyIHRvIHBlcmZvcm0gdGhlIGFuYWx5c2lzIG9uIGV4aXN0aW5nIG1vZGVsIGVuZHBvaW50IG9yIHRvCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBnZW5lcmF0ZSBhIG5ldyBtb2RlbCBlbmRwb2ludCByZWNvcmQuCiAgICA6cGFyYW0gbW9kZWxfZW5kcG9pbnRfbmFtZTogICAgICAgICAgICAgICAgICAgICBJZiBhIG5ldyBtb2RlbCBlbmRwb2ludCBpcyBnZW5lcmF0ZWQsIHRoZSBtb2RlbCBuYW1lIHdpbGwgYmUgcHJlc2VudGVkIHVuZGVyIHRoaXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGVuZHBvaW50LgogICAgOnBhcmFtIG1vZGVsX2VuZHBvaW50X2RyaWZ0X3RocmVzaG9sZDogICAgICAgICAgVGhlIHRocmVzaG9sZCBvZiB3aGljaCB0byBtYXJrIGRyaWZ0cy4gRGVmYXVsdGVkIHRvIDAuNy4KICAgIDpwYXJhbSBtb2RlbF9lbmRwb2ludF9wb3NzaWJsZV9kcmlmdF90aHJlc2hvbGQ6IFRoZSB0aHJlc2hvbGQgb2Ygd2hpY2ggdG8gbWFyayBwb3NzaWJsZSBkcmlmdHMuIERlZmF1bHRlZCB0byAwLjUuCiAgICA6cGFyYW0gbW9kZWxfZW5kcG9pbnRfc2FtcGxlX3NldDogICAgICAgICAgICAgICBBIHNhbXBsZSBkYXRhc2V0IHRvIGdpdmUgdG8gY29tcGFyZSB0aGUgaW5wdXRzIGluIHRoZSBkcmlmdCBhbmFseXNpcy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENhbiBiZSBwcm92aWRlZCBhcyBhbiBpbnB1dCAoRGF0YUl0ZW0pIG9yIGFzIGEgcGFyYW1ldGVyIChlLmcuIHN0cmluZywgbGlzdCwgRGF0YUZyYW1lKS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFRoZSBkZWZhdWx0IGNob3NlbiBzYW1wbGUgc2V0IHdpbGwgYWx3YXlzIGJlIHRoZSBvbmUgd2hvIGlzIHNldCBpbiB0aGUgbW9kZWwgYXJ0aWZhY3QgaXRzZWxmLgogICAgOnBhcmFtIGxhc3RfaW5fYmF0Y2hfc2V0OiAgICAgICAgICAgICAgICAgICAgICAgUmVsZXZhbnQgb25seSB3aGVuIGBwZXJmb3JtX2RyaWZ0X2FuYWx5c2lzYCBpcyBgVHJ1ZWAuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBUaGlzIGZsYWcgY2FuIChhbmQgc2hvdWxkIG9ubHkpIGJlIHVzZWQgd2hlbiB0aGUgbW9kZWwgZW5kcG9pbnQgZG9lcyBub3QgaGF2ZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9kZWwtbW9uaXRvcmluZyBzZXQuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiBzZXQgdG8gYFRydWVgICh0aGUgZGVmYXVsdCksIHRoaXMgZmxhZyBtYXJrcyB0aGUgY3VycmVudCBtb25pdG9yaW5nIHdpbmRvdwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKG9uIHRoaXMgbW9uaXRvcmluZyBlbmRwb2ludCkgYXMgY29tcGxldGVkIC0gdGhlIGRhdGEgaW5mZXJyZWQgc28gZmFyIGlzIGFzc3VtZWQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRvIGJlIHRoZSBjb21wbGV0ZSBkYXRhIGZvciB0aGlzIG1vbml0b3Jpbmcgd2luZG93LgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgWW91IG1heSB3YW50IHRvIHNldCB0aGlzIGZsYWcgdG8gYEZhbHNlYCBpZiB5b3Ugd2FudCB0byByZWNvcmQgbXVsdGlwbGUgcmVzdWx0cyBpbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgY2xvc2UgdGltZSBwcm94aW1pdHkgKCJiYXRjaCBzZXQiKS4gSW4gdGhpcyBjYXNlLCBzZXQgdGhpcyBmbGFnIHRvIGBGYWxzZWAgb24gYWxsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBidXQgdGhlIGxhc3QgYmF0Y2ggaW4gdGhlIHNldC4KICAgIHJhaXNlcyBNTFJ1bkludmFsaWRBcmd1bWVudEVycm9yOiBpZiBib3RoIGBtb2RlbF9wYXRoYCBhbmQgYGVuZHBvaW50X2lkYCBhcmUgbm90IHByb3ZpZGVkLCBvciBpZiBgbGFzdF9pbl9iYXRjaF9zZXRgIGlzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcHJvdmlkZWQgZm9yIGFuIHVuc3VwcG9ydGVkIGBtbHJ1bmAgdmVyc2lvbi4KICAgICIiIgoKICAgICMgTG9hZGluZyB0aGUgbW9kZWw6CiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiTG9hZGluZyBtb2RlbC4uLiIpCiAgICBpZiBpc2luc3RhbmNlKG1vZGVsX3BhdGgsIG1scnVuLkRhdGFJdGVtKToKICAgICAgICBtb2RlbF9wYXRoID0gbW9kZWxfcGF0aC5hcnRpZmFjdF91cmwKICAgIGlmIG5vdCBtbHJ1bi5kYXRhc3RvcmUuaXNfc3RvcmVfdXJpKG1vZGVsX3BhdGgpOgogICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICBmIlRoZSBwcm92aWRlZCBtb2RlbCBwYXRoICh7bW9kZWxfcGF0aH0pIGlzIGludmFsaWQgLSBzaG91bGQgc3RhcnQgd2l0aCBgc3RvcmU6Ly9gLiAiCiAgICAgICAgICAgIGYiUGxlYXNlIG1ha2Ugc3VyZSB0aGF0IHlvdSBoYXZlIGxvZ2dlZCB0aGUgbW9kZWwgdXNpbmcgYHByb2plY3QubG9nX21vZGVsKClgICIKICAgICAgICAgICAgZiJ3aGljaCBnZW5lcmF0ZXMgYSB1bmlxdWUgc3RvcmUgdXJpIGZvciB0aGUgbG9nZ2VkIG1vZGVsLiIKICAgICAgICApCiAgICBtb2RlbF9oYW5kbGVyID0gQXV0b01MUnVuLmxvYWRfbW9kZWwobW9kZWxfcGF0aD1tb2RlbF9wYXRoLCBjb250ZXh0PWNvbnRleHQpCgogICAgaWYgbGFiZWxfY29sdW1ucyBpcyBOb25lOgogICAgICAgIGxhYmVsX2NvbHVtbnMgPSBbCiAgICAgICAgICAgIG91dHB1dC5uYW1lIGZvciBvdXRwdXQgaW4gbW9kZWxfaGFuZGxlci5fbW9kZWxfYXJ0aWZhY3Quc3BlYy5vdXRwdXRzCiAgICAgICAgXQoKICAgIGlmIGZlYXR1cmVfY29sdW1ucyBpcyBOb25lOgogICAgICAgIGZlYXR1cmVfY29sdW1ucyA9IFsKICAgICAgICAgICAgaW5wdXQubmFtZSBmb3IgaW5wdXQgaW4gbW9kZWxfaGFuZGxlci5fbW9kZWxfYXJ0aWZhY3Quc3BlYy5pbnB1dHMKICAgICAgICBdCgogICAgIyBHZXQgZGF0YXNldCBieSBvYmplY3QsIFVSTCBvciBieSBGZWF0dXJlVmVjdG9yOgogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmIkxvYWRpbmcgZGF0YS4uLiIpCiAgICB4LCBsYWJlbF9jb2x1bW5zID0gbWxydW4ubW9kZWxfbW9uaXRvcmluZy5hcGkucmVhZF9kYXRhc2V0X2FzX2RhdGFmcmFtZSgKICAgICAgICBkYXRhc2V0PWRhdGFzZXQsCiAgICAgICAgZmVhdHVyZV9jb2x1bW5zPWZlYXR1cmVfY29sdW1ucywKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIFByZWRpY3Q6CiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiQ2FsY3VsYXRpbmcgcHJlZGljdGlvbi4uLiIpCiAgICB5X3ByZWQgPSBtb2RlbF9oYW5kbGVyLm1vZGVsLnByZWRpY3QoeCwgKipwcmVkaWN0X2t3YXJncykKCiAgICAjIFByZXBhcmUgdGhlIHJlc3VsdCBzZXQ6CiAgICByZXN1bHRfc2V0ID0gX3ByZXBhcmVfcmVzdWx0X3NldCh4PXgsIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywgeV9wcmVkPXlfcHJlZCkKCiAgICAjIENoZWNrIGZvciBsb2dnaW5nIHRoZSByZXN1bHQgc2V0OgogICAgaWYgbG9nX3Jlc3VsdF9zZXQ6CiAgICAgICAgbWxydW4ubW9kZWxfbW9uaXRvcmluZy5hcGkubG9nX3Jlc3VsdCgKICAgICAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgICAgICByZXN1bHRfc2V0X25hbWU9cmVzdWx0X3NldF9uYW1lLAogICAgICAgICAgICByZXN1bHRfc2V0PXJlc3VsdF9zZXQsCiAgICAgICAgICAgIGFydGlmYWN0c190YWc9YXJ0aWZhY3RzX3RhZywKICAgICAgICAgICAgYmF0Y2hfaWQ9YmF0Y2hfaWQsCiAgICAgICAgKQoKICAgICMgQ2hlY2sgZm9yIHBlcmZvcm1pbmcgZHJpZnQgYW5hbHlzaXMKICAgIGlmICgKICAgICAgICBwZXJmb3JtX2RyaWZ0X2FuYWx5c2lzIGlzIE5vbmUKICAgICAgICBhbmQgbW9kZWxfaGFuZGxlci5fbW9kZWxfYXJ0aWZhY3Quc3BlYy5mZWF0dXJlX3N0YXRzIGlzIG5vdCBOb25lCiAgICApOgogICAgICAgIHBlcmZvcm1fZHJpZnRfYW5hbHlzaXMgPSBUcnVlCiAgICBpZiBwZXJmb3JtX2RyaWZ0X2FuYWx5c2lzOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oIlBlcmZvcm1pbmcgZHJpZnQgYW5hbHlzaXMuLi4iKQogICAgICAgICMgR2V0IHRoZSBzYW1wbGUgc2V0IHN0YXRpc3RpY3MgKGVpdGhlciBmcm9tIHRoZSBzYW1wbGUgc2V0IG9yIGZyb20gdGhlIHN0YXRpc3RpY3MgbG9nZ2VkIHdpdGggdGhlIG1vZGVsKQogICAgICAgIHNhbXBsZV9zZXRfc3RhdGlzdGljcyA9IG1scnVuLm1vZGVsX21vbml0b3JpbmcuYXBpLmdldF9zYW1wbGVfc2V0X3N0YXRpc3RpY3MoCiAgICAgICAgICAgIHNhbXBsZV9zZXQ9bW9kZWxfZW5kcG9pbnRfc2FtcGxlX3NldCwKICAgICAgICAgICAgbW9kZWxfYXJ0aWZhY3RfZmVhdHVyZV9zdGF0cz1tb2RlbF9oYW5kbGVyLl9tb2RlbF9hcnRpZmFjdC5zcGVjLmZlYXR1cmVfc3RhdHMsCiAgICAgICAgKQogICAgICAgIG1scnVuLm1vZGVsX21vbml0b3JpbmcuYXBpLnJlY29yZF9yZXN1bHRzKAogICAgICAgICAgICBwcm9qZWN0PWNvbnRleHQucHJvamVjdCwKICAgICAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgICAgICBlbmRwb2ludF9pZD1lbmRwb2ludF9pZCwKICAgICAgICAgICAgbW9kZWxfcGF0aD1tb2RlbF9wYXRoLAogICAgICAgICAgICBtb2RlbF9lbmRwb2ludF9uYW1lPW1vZGVsX2VuZHBvaW50X25hbWUsCiAgICAgICAgICAgIGluZmVyX3Jlc3VsdHNfZGY9cmVzdWx0X3NldC5jb3B5KCksCiAgICAgICAgICAgIHNhbXBsZV9zZXRfc3RhdGlzdGljcz1zYW1wbGVfc2V0X3N0YXRpc3RpY3MsCiAgICAgICAgICAgIGRyaWZ0X3RocmVzaG9sZD1tb2RlbF9lbmRwb2ludF9kcmlmdF90aHJlc2hvbGQsCiAgICAgICAgICAgIHBvc3NpYmxlX2RyaWZ0X3RocmVzaG9sZD1tb2RlbF9lbmRwb2ludF9wb3NzaWJsZV9kcmlmdF90aHJlc2hvbGQsCiAgICAgICAgICAgIGFydGlmYWN0c190YWc9YXJ0aWZhY3RzX3RhZywKICAgICAgICAgICAgdHJpZ2dlcl9tb25pdG9yaW5nX2pvYj10cmlnZ2VyX21vbml0b3Jpbmdfam9iLAogICAgICAgICAgICBkZWZhdWx0X2JhdGNoX2ltYWdlPWJhdGNoX2ltYWdlX2pvYiwKICAgICAgICAgICAgKipfcGFyc2VfcmVjb3JkX3Jlc3VsdHNfa3dhcmcobGFzdF9pbl9iYXRjaF9zZXQ9bGFzdF9pbl9iYXRjaF9zZXQpLAogICAgICAgICkK commands: [] code_origin: '' origin_filename: '' @@ -143,9 +143,20 @@ spec: DataFrame). The default chosen sample set will always be the one who is set in the model artifact itself. default: null + - name: last_in_batch_set + type: Optional[bool] + doc: Relevant only when `perform_drift_analysis` is `True`. This flag can + (and should only) be used when the model endpoint does not have model-monitoring + set. If set to `True` (the default), this flag marks the current monitoring + window (on this monitoring endpoint) as completed - the data inferred so + far is assumed to be the complete data for this monitoring window. You may + want to set this flag to `False` if you want to record multiple results + in close time proximity ("batch set"). In this case, set this flag to `False` + on all but the last batch in the set. + default: null outputs: - default: '' - lineno: 82 + lineno: 103 description: Batch inference (also knows as prediction) for the common ML frameworks (SciKit-Learn, XGBoost and LightGBM) while performing data drift analysis. default_handler: infer diff --git a/batch_inference_v2/item.yaml b/batch_inference_v2/item.yaml index 20422f79c..78804fe97 100644 --- a/batch_inference_v2/item.yaml +++ b/batch_inference_v2/item.yaml @@ -29,4 +29,4 @@ spec: kind: job requirements: null url: '' -version: 2.1.0 +version: 2.2.0