Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make use of the new name filtering in calls to list jobs. #595

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions databricks_cli/jobs/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,10 @@ def create_job(self, json, headers=None, version=None):
version=version)

def list_jobs(self, job_type=None, expand_tasks=None, offset=None, limit=None, headers=None,
version=None):
version=None, name_filter=None):
resp = self.client.list_jobs(job_type=job_type, expand_tasks=expand_tasks, offset=offset,
limit=limit, headers=headers, version=version)
limit=limit, headers=headers, version=version,
name_filter=name_filter)
if 'jobs' not in resp:
resp['jobs'] = []
return resp
Expand All @@ -56,6 +57,6 @@ def run_now(self, job_id, jar_params, notebook_params, python_params, spark_subm
idempotency_token, headers=headers, version=version)

def _list_jobs_by_name(self, name, headers=None):
jobs = self.list_jobs(headers=headers)['jobs']
jobs = self.list_jobs(headers=headers, name_filter=name)['jobs']
result = list(filter(lambda job: job['settings']['name'] == name, jobs))
return result
13 changes: 9 additions & 4 deletions databricks_cli/jobs/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,12 +130,15 @@ def _jobs_to_table(jobs_json):
@click.option('--all', '_all', is_flag=True,
help='Lists all jobs by executing sequential calls to the API ' +
'(only available in API 2.1).')
@click.option('--name', 'name_filter', default=None, type=str,
help='If provided, only returns jobs that match the supplied ' +
'name (only available in API 2.1).')
@api_version_option
@debug_option
@profile_option
@eat_exceptions
@provide_api_client
def list_cli(api_client, output, job_type, version, expand_tasks, offset, limit, _all):
def list_cli(api_client, output, job_type, version, expand_tasks, offset, limit, _all, name_filter):
"""
Lists the jobs in the Databricks Job Service.

Expand All @@ -151,9 +154,10 @@ def list_cli(api_client, output, job_type, version, expand_tasks, offset, limit,
"""
check_version(api_client, version)
api_version = version or api_client.jobs_api_version
if api_version != '2.1' and (expand_tasks or offset or limit or _all):
using_features_only_in_21 = expand_tasks or offset or limit or _all or name_filter
if api_version != '2.1' and using_features_only_in_21:
click.echo(click.style('ERROR', fg='red') + ': the options --expand-tasks, ' +
'--offset, --limit, and --all are only available in API 2.1', err=True)
'--offset, --limit, --all, and --name are only available in API 2.1', err=True)
return
jobs_api = JobsApi(api_client)
has_more = True
Expand All @@ -163,7 +167,8 @@ def list_cli(api_client, output, job_type, version, expand_tasks, offset, limit,
limit = 20
while has_more:
jobs_json = jobs_api.list_jobs(job_type=job_type, expand_tasks=expand_tasks,
offset=offset, limit=limit, version=version)
offset=offset, limit=limit, version=version,
name_filter=name_filter)
jobs += jobs_json['jobs'] if 'jobs' in jobs_json else []
has_more = jobs_json.get('has_more', False) and _all
if has_more:
Expand Down
4 changes: 3 additions & 1 deletion databricks_cli/sdk/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@ def get_job(self, job_id, headers=None, version=None):
)

def list_jobs(
self, job_type=None, expand_tasks=None, limit=None, offset=None, headers=None, version=None
self, job_type=None, expand_tasks=None, limit=None, offset=None, headers=None, version=None, name_filter=None
):
_data = {}
if job_type is not None:
Expand All @@ -298,6 +298,8 @@ def list_jobs(
_data['limit'] = limit
if offset is not None:
_data['offset'] = offset
if name_filter is not None:
_data['name'] = name_filter
return self.client.perform_query(
'GET', '/jobs/list', data=_data, headers=headers, version=version
)
Expand Down
5 changes: 5 additions & 0 deletions tests/jobs/test_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,11 @@ def test_list_jobs():
'GET', '/jobs/list', data={}, headers=None, version='3.0'
)

api.list_jobs(version='2.1', name_filter='foo')
api_client_mock.perform_query.assert_called_with(
'GET', '/jobs/list', data={'name':'foo'}, headers=None, version='2.1'
)


@provide_conf
def test_run_now():
Expand Down
8 changes: 8 additions & 0 deletions tests/jobs/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,14 @@ def test_list_offset(jobs_api_mock):
assert jobs_api_mock.list_jobs.call_args[1]['offset'] == 1
assert jobs_api_mock.list_jobs.call_args[1]['version'] == '2.1'

@provide_conf
def test_list_name(jobs_api_mock):
jobs_api_mock.list_jobs.return_value = LIST_RETURN_1
runner = CliRunner()
result = runner.invoke(cli.list_cli, ['--version=2.1', '--name', 'foo'])
assert result.exit_code == 0
assert jobs_api_mock.list_jobs.call_args[1]['name_filter'] == 'foo'
assert jobs_api_mock.list_jobs.call_args[1]['version'] == '2.1'

@provide_conf
def test_list_limit(jobs_api_mock):
Expand Down