From 7127178dce93a0306599c3e75e3376b312478b4f Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Fri, 26 Apr 2019 10:25:30 -0400 Subject: [PATCH 1/2] added getDockerJobsListQuick function --- py2/SciServer/Jobs.py | 54 ++++++++++++++++++++++++++++++++++++++++++- py3/SciServer/Jobs.py | 54 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 106 insertions(+), 2 deletions(-) diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py index 76f2e70..62b7364 100644 --- a/py2/SciServer/Jobs.py +++ b/py2/SciServer/Jobs.py @@ -7,6 +7,7 @@ import requests import json import time; +import pandas as pd @@ -181,7 +182,7 @@ def getJobsList(top=10, open=None, start=None, end=None, type='all'): :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. :example: jobs = Jobs.getJobsList(top=2); - .. seealso:: Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + .. seealso:: Jobs.getDockerJobsListQuick, Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob """ token = Authentication.getToken() if token is not None and token != "": @@ -221,6 +222,57 @@ def getJobsList(top=10, open=None, start=None, end=None, type='all'): else: raise Exception("User token is not defined. First log into SciServer.") + +def getDockerJobsListQuick(top=10, open=None, start=None, end=None, labelReg=None, returnType="pandas"): + """ + Gets the list of Jobs submitted by the user using the "quick" endpoint. + :param top: top number of jobs (integer) returned. If top=None, then all jobs are returned. + :param open: If set to 'True', then only returns jobs that have not finished executing and wrapped up (status <= FINISHED). If set to 'False' then only returnes jobs that are still running. If set to 'None', then returns both finished and unfinished jobs. + :param start: The earliest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no lower bound on date. + :param end: The latest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no upper bound on date. + :param labelReg: A string occurring in the submitterDIDtype of jobs returned. + :param returnType: if set To "pandas" (default setting), then it returns a pandas dataframe. Else, it will return a dictionary with the jobs list. + :return: a pandas dataframe by default, with each row containing information about a submitted job. See the 'returnType' parameter for other return options. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. + :example: jobs = Jobs.getDockerJobsListQuick(top=2,labelReg='__turbcutout__'); + .. seealso:: Jobs.getJobsList, Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob, + """ + token = Authentication.getToken() + if token is not None and token != "": + topString = ("top={top}".format(top=top) if top != None else None) + startString = ("start={start}".format(start=start) if start != None else None) + endString = ("end={end}".format(end=end) if end != None else None) + labelReg=("labelReg={labelReg}".format(labelReg=labelReg) if labelReg != None else None) + openString=(("open={0}".format("true" if open == True else "false") if open is not None else None)) + + params=[s for s in [topString,startString,endString,labelReg,openString] if s is not None] + if len(params)>0: + params="?"+"&".join(params) + else: + params="" + url = Config.RacmApiURL + "/jobm/rest/dockerjobs/quick" + params + + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting list of docker jobs from JOBM API.\nHttp Response from JOBM API returned status code " + + str(res.status_code) + ":\n" + res.content.decode()); + j = json.loads(res.content.decode()) + # change dict 'j' so it can be used to create a pandas dataframe + if returnType=="pandas": + j['data']=j.pop('rows') + j.pop('hrNames') + df=pd.read_json(json.dumps(j), orient='split') + df.fillna('',inplace=True) + return df + else: + return j + else: + raise Exception("User token is not defined. First log into SciServer.") + + + def getJobDescription(jobId): """ Gets the definition of the job, diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py index 6a372ef..7a88684 100644 --- a/py3/SciServer/Jobs.py +++ b/py3/SciServer/Jobs.py @@ -7,6 +7,7 @@ import requests import json import time; +import pandas as pd @@ -181,7 +182,7 @@ def getJobsList(top=10, open=None, start=None, end=None, type='all'): :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. :example: jobs = Jobs.getJobsList(top=2); - .. seealso:: Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + .. seealso:: Jobs.getDockerJobsListQuick, Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob """ token = Authentication.getToken() if token is not None and token != "": @@ -221,6 +222,57 @@ def getJobsList(top=10, open=None, start=None, end=None, type='all'): else: raise Exception("User token is not defined. First log into SciServer.") + +def getDockerJobsListQuick(top=10, open=None, start=None, end=None, labelReg=None, returnType="pandas"): + """ + Gets the list of Jobs submitted by the user using the "quick" endpoint. + :param top: top number of jobs (integer) returned. If top=None, then all jobs are returned. + :param open: If set to 'True', then only returns jobs that have not finished executing and wrapped up (status <= FINISHED). If set to 'False' then only returnes jobs that are still running. If set to 'None', then returns both finished and unfinished jobs. + :param start: The earliest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no lower bound on date. + :param end: The latest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no upper bound on date. + :param labelReg: A string occurring in the submitterDIDtype of jobs returned. + :param returnType: if set To "pandas" (default setting), then it returns a pandas dataframe. Else, it will return a dictionary with the jobs list. + :return: a pandas dataframe by default, with each row containing information about a submitted job. See the 'returnType' parameter for other return options. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. + :example: jobs = Jobs.getDockerJobsListQuick(top=2,labelReg='__turbcutout__'); + .. seealso:: Jobs.getJobsList, Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob, + """ + token = Authentication.getToken() + if token is not None and token != "": + topString = ("top={top}".format(top=top) if top != None else None) + startString = ("start={start}".format(start=start) if start != None else None) + endString = ("end={end}".format(end=end) if end != None else None) + labelReg=("labelReg={labelReg}".format(labelReg=labelReg) if labelReg != None else None) + openString=(("open={0}".format("true" if open == True else "false") if open is not None else None)) + + params=[s for s in [topString,startString,endString,labelReg,openString] if s is not None] + if len(params)>0: + params="?"+"&".join(params) + else: + params="" + url = Config.RacmApiURL + "/jobm/rest/dockerjobs/quick" + params + + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting list of docker jobs from JOBM API.\nHttp Response from JOBM API returned status code " + + str(res.status_code) + ":\n" + res.content.decode()); + j = json.loads(res.content.decode()) + # change dict 'j' so it can be used to create a pandas dataframe + if returnType=="pandas": + j['data']=j.pop('rows') + j.pop('hrNames') + df=pd.read_json(json.dumps(j), orient='split') + df.fillna('',inplace=True) + return df + else: + return j + else: + raise Exception("User token is not defined. First log into SciServer.") + + + def getJobDescription(jobId): """ Gets the definition of the job, From feae2e823b8f4bd05b9178f429cfe1f6c2ca8ecb Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Fri, 26 Apr 2019 11:25:24 -0400 Subject: [PATCH 2/2] updated example notebook with new quick docker jobs list function --- Examples/Examples_SciScript-Python.ipynb | 83 ++++++++++-------------- 1 file changed, 33 insertions(+), 50 deletions(-) diff --git a/Examples/Examples_SciScript-Python.ipynb b/Examples/Examples_SciScript-Python.ipynb index c2d90ae..5c11e79 100644 --- a/Examples/Examples_SciScript-Python.ipynb +++ b/Examples/Examples_SciScript-Python.ipynb @@ -38,14 +38,10 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# *******************************************************************************************************\n", - "# Authentication section\n", - "# *******************************************************************************************************" + "# Authentication module" ] }, { @@ -119,14 +115,10 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# *******************************************************************************************************\n", - "# LoginPortal section:\n", - "# *******************************************************************************************************" + "# LoginPortal mmodule:" ] }, { @@ -194,14 +186,10 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# *******************************************************************************************************\n", - "# CasJobs section:\n", - "# *******************************************************************************************************" + "# CasJobs module:" ] }, { @@ -422,14 +410,10 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# *******************************************************************************************************\n", - "# SkyServer section:\n", - "# *******************************************************************************************************" + "# SkyServer section:" ] }, { @@ -516,14 +500,10 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# *******************************************************************************************************\n", - "# SciDrive section:\n", - "# *******************************************************************************************************" + "# SciDrive section:" ] }, { @@ -684,14 +664,10 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# *******************************************************************************************************\n", - "# SkyQuery section:\n", - "# *******************************************************************************************************" + "# SkyQuery section:" ] }, { @@ -938,14 +914,10 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# *******************************************************************************************************\n", - "# Files section:\n", - "# *******************************************************************************************************" + "# Files module:" ] }, { @@ -1254,14 +1226,10 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# *******************************************************************************************************\n", - "# Jobs section:\n", - "# *******************************************************************************************************" + "# Jobs section:" ] }, { @@ -1525,7 +1493,22 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "# get list of jobs\n", + "jobs_list = Jobs.getJobsList(top=2)\n", + "jobs_list" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get list of docker jobs\n", + "jobs_list = Jobs.getDockerJobsListQuick(top=2)\n", + "jobs_list" + ] } ], "metadata": { @@ -1548,5 +1531,5 @@ } }, "nbformat": 4, - "nbformat_minor": 1 + "nbformat_minor": 2 }