From 1024860b0e76866882d887a9b87d71550b49f703 Mon Sep 17 00:00:00 2001 From: Yixiao Lin Date: Thu, 4 Jun 2015 14:14:46 -0500 Subject: [PATCH 1/2] Truncate number of jobs by 10,000 --- apps/jobbrowser/src/jobbrowser/views.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/jobbrowser/src/jobbrowser/views.py b/apps/jobbrowser/src/jobbrowser/views.py index 5e2a6d4..7e5ae87 100644 --- a/apps/jobbrowser/src/jobbrowser/views.py +++ b/apps/jobbrowser/src/jobbrowser/views.py @@ -109,6 +109,10 @@ def jobs(request): if request.GET.get('format') == 'json': try: jobs = get_api(request.user, request.jt).get_jobs(user=request.user, username=user, state=state, text=text, retired=retired) + ## when there are too many jobs, truncate + ## When there are about 10,000 jobs, it takes about 30 second to load. + ## The page crashes when there are more than 1,000,000 total jobs. + jobs = jobs[:10000] except Exception, ex: ex_message = str(ex) if 'Connection refused' in ex_message or 'standby RM' in ex_message: -- 1.9.3 (Apple Git-50) From 028f441711d55c6afd13d4fc969a514cf0354184 Mon Sep 17 00:00:00 2001 From: Yixiao Lin Date: Thu, 4 Jun 2015 15:27:39 -0500 Subject: [PATCH 2/2] limit the number of jobs returned by the request --- apps/jobbrowser/src/jobbrowser/api.py | 2 ++ apps/jobbrowser/src/jobbrowser/views.py | 7 ++----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/apps/jobbrowser/src/jobbrowser/api.py b/apps/jobbrowser/src/jobbrowser/api.py index debe1fe..a7bb891 100644 --- a/apps/jobbrowser/src/jobbrowser/api.py +++ b/apps/jobbrowser/src/jobbrowser/api.py @@ -227,6 +227,8 @@ class YarnApi(JobBrowserApi): filters['user'] = kwargs['username'] if kwargs['state'] and kwargs['state'] != 'all': filters['finalStatus'] = state_filters[kwargs['state']] + if kwargs['limit']: + filters['limit'] = kwargs['limit'] json = self.resource_manager_api.apps(**filters) if type(json) == str and 'This is standby RM' in json: diff --git a/apps/jobbrowser/src/jobbrowser/views.py b/apps/jobbrowser/src/jobbrowser/views.py index 7e5ae87..9cd8abc 100644 --- a/apps/jobbrowser/src/jobbrowser/views.py +++ b/apps/jobbrowser/src/jobbrowser/views.py @@ -108,11 +108,8 @@ def jobs(request): if request.GET.get('format') == 'json': try: - jobs = get_api(request.user, request.jt).get_jobs(user=request.user, username=user, state=state, text=text, retired=retired) - ## when there are too many jobs, truncate - ## When there are about 10,000 jobs, it takes about 30 second to load. - ## The page crashes when there are more than 1,000,000 total jobs. - jobs = jobs[:10000] + jobs = get_api(request.user, request.jt).get_jobs(user=request.user, username=user, state=state, text=text, retired=retired, limit=10000) + ## limit number of jobs to be 10,000 except Exception, ex: ex_message = str(ex) if 'Connection refused' in ex_message or 'standby RM' in ex_message: -- 1.9.3 (Apple Git-50)