Skip to content

Commit

Permalink
[JOBS-11439] Add taskValues support in remoteDbUtils (#406)
Browse files Browse the repository at this point in the history
## Changes
Support `dbutils.jobs.taskValues` methods in `RemoteDbUtils` and add
additional `checkTaskValue` to allow local testing

## Tests
<!-- 
How is this tested? Please see the checklist below and also describe any
other relevant tests
-->
Added three test cases in `test_dbutils.py`. Integration tests are not
needed since changes only for remote mode
- [x] `make test` run locally
- [x] `make fmt` applied
- [x] relevant integration tests applied
  • Loading branch information
vsamoilov authored Nov 1, 2023
1 parent 2401940 commit 9955c08
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 0 deletions.
26 changes: 26 additions & 0 deletions databricks/sdk/dbutils.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,31 @@ def listScopes(self) -> typing.List[SecretScope]:
return [SecretScope(v.name) for v in self._api.list_scopes()]


class _JobsUtil:
"""Remote equivalent of jobs util"""

class _TaskValuesUtil:
"""Remote equivalent of task values util"""

def get(self, taskKey: str, key: str, default: any = None, debugValue: any = None) -> None:
"""
Returns `debugValue` if present, throws an error otherwise as this implementation is always run outside of a job run
"""
if debugValue is None:
raise TypeError(
'Must pass debugValue when calling get outside of a job context. debugValue cannot be None.'
)
return debugValue

def set(self, key: str, value: any) -> None:
"""
Sets a task value on the current task run
"""

def __init__(self) -> None:
self.taskValues = self._TaskValuesUtil()


class RemoteDbUtils:

def __init__(self, config: 'Config' = None):
Expand All @@ -175,6 +200,7 @@ def __init__(self, config: 'Config' = None):

self.fs = _FsUtil(dbfs_ext.DbfsExt(self._client), self.__getattr__)
self.secrets = _SecretsUtil(workspace.SecretsAPI(self._client))
self.jobs = _JobsUtil()
self._widgets = None

# When we import widget_impl, the init file checks whether user has the
Expand Down
22 changes: 22 additions & 0 deletions tests/test_dbutils.py
Original file line number Diff line number Diff line change
Expand Up @@ -204,3 +204,25 @@ def test_secrets_get_and_redacting_logs(dbutils, mocker):
inner.assert_called_with('GET', '/api/2.0/secrets/get', query={'key': 'bar', 'scope': 'foo'})

assert value == 'hello'


def test_jobs_task_values_set(dbutils):
dbutils.jobs.taskValues.set('key', 'value')


def test_jobs_task_values_get(dbutils):
assert dbutils.jobs.taskValues.get('taskKey', 'key', debugValue='debug') == 'debug'

dbutils.jobs.taskValues.set('key', 'value')

# Expect `get` to always return the `debugValue`` when calling outside of a job context and not what was previously set using `set`
assert dbutils.jobs.taskValues.get('taskKey', 'key', debugValue='debug') == 'debug'


def test_jobs_task_values_get_throws(dbutils):
try:
dbutils.jobs.taskValues.get('taskKey', 'key')
assert False
except TypeError as e:
assert str(
e) == 'Must pass debugValue when calling get outside of a job context. debugValue cannot be None.'

0 comments on commit 9955c08

Please sign in to comment.