From bb076862ae4e5892831ea3289c473f34f4efb461 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Thu, 16 Feb 2017 15:51:53 -0500 Subject: [PATCH 01/79] Removing Jobs namespace for now, since Jobs functionality is not ready for production SciServer --- py2/SciServer/Jobs.py | 21 --------------------- py3/SciServer/Jobs.py | 21 --------------------- 2 files changed, 42 deletions(-) delete mode 100644 py2/SciServer/Jobs.py delete mode 100644 py3/SciServer/Jobs.py diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py deleted file mode 100644 index bb62378..0000000 --- a/py2/SciServer/Jobs.py +++ /dev/null @@ -1,21 +0,0 @@ -__author__ = 'mtaghiza' -#Python v3.4 - -import sys -import os; -import os.path; - - -def getJobDirectory(): - """ - Gets the Jobs directory, as written inside the /home/idies/jobs.path file. This directory is used as the working directory in the execution of asynchronous jobs. - """ - jobDirectoryFile='/home/idies/jobs.path' - if os.path.isfile(jobDirectoryFile): - jobDirectory = None; - with open(jobDirectoryFile, 'r') as f: - jobDirectory = f.read().rstrip('\n') - - return jobDirectory - else: - raise OSError("Jobs directory not found.") \ No newline at end of file diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py deleted file mode 100644 index bb62378..0000000 --- a/py3/SciServer/Jobs.py +++ /dev/null @@ -1,21 +0,0 @@ -__author__ = 'mtaghiza' -#Python v3.4 - -import sys -import os; -import os.path; - - -def getJobDirectory(): - """ - Gets the Jobs directory, as written inside the /home/idies/jobs.path file. This directory is used as the working directory in the execution of asynchronous jobs. - """ - jobDirectoryFile='/home/idies/jobs.path' - if os.path.isfile(jobDirectoryFile): - jobDirectory = None; - with open(jobDirectoryFile, 'r') as f: - jobDirectory = f.read().rstrip('\n') - - return jobDirectory - else: - raise OSError("Jobs directory not found.") \ No newline at end of file From 2ba663cd6787a75bff1129d19662a1627b5ccb4c Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Thu, 16 Feb 2017 15:56:40 -0500 Subject: [PATCH 02/79] Added Jobs namespaces back again. --- py2/SciServer/Jobs.py | 21 +++++++++++++++++++++ py3/SciServer/Jobs.py | 21 +++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 py2/SciServer/Jobs.py create mode 100644 py3/SciServer/Jobs.py diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py new file mode 100644 index 0000000..bb62378 --- /dev/null +++ b/py2/SciServer/Jobs.py @@ -0,0 +1,21 @@ +__author__ = 'mtaghiza' +#Python v3.4 + +import sys +import os; +import os.path; + + +def getJobDirectory(): + """ + Gets the Jobs directory, as written inside the /home/idies/jobs.path file. This directory is used as the working directory in the execution of asynchronous jobs. + """ + jobDirectoryFile='/home/idies/jobs.path' + if os.path.isfile(jobDirectoryFile): + jobDirectory = None; + with open(jobDirectoryFile, 'r') as f: + jobDirectory = f.read().rstrip('\n') + + return jobDirectory + else: + raise OSError("Jobs directory not found.") \ No newline at end of file diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py new file mode 100644 index 0000000..bb62378 --- /dev/null +++ b/py3/SciServer/Jobs.py @@ -0,0 +1,21 @@ +__author__ = 'mtaghiza' +#Python v3.4 + +import sys +import os; +import os.path; + + +def getJobDirectory(): + """ + Gets the Jobs directory, as written inside the /home/idies/jobs.path file. This directory is used as the working directory in the execution of asynchronous jobs. + """ + jobDirectoryFile='/home/idies/jobs.path' + if os.path.isfile(jobDirectoryFile): + jobDirectory = None; + with open(jobDirectoryFile, 'r') as f: + jobDirectory = f.read().rstrip('\n') + + return jobDirectory + else: + raise OSError("Jobs directory not found.") \ No newline at end of file From 5e49f7a0764fbb1d97aefc0c6f4a0b2eeccce54d Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Tue, 28 Mar 2017 16:34:23 -0400 Subject: [PATCH 03/79] Added ComputeJobs module --- py2/SciServer/ComputeJobs.py | 226 +++++++++++++++++++++++++++++++++++ py2/SciServer/Config.py | 5 +- 2 files changed, 229 insertions(+), 2 deletions(-) create mode 100644 py2/SciServer/ComputeJobs.py diff --git a/py2/SciServer/ComputeJobs.py b/py2/SciServer/ComputeJobs.py new file mode 100644 index 0000000..09c9ec9 --- /dev/null +++ b/py2/SciServer/ComputeJobs.py @@ -0,0 +1,226 @@ +__author__ = 'mtaghiza' + +import sys +import os; +import os.path; +from SciServer import Authentication, Config +import requests +import json +import time; + + + +def getJobDirectory(jobId = None): + """ + Gets the root directory path where the job is being executed. Since its value depends on the value of jobID, the path will be defined only after the job is submitted. + """ + computeJobDirectoryFile = Config.ComputeJobDirectoryFile + if os.path.isfile(computeJobDirectoryFile): + jobDirectory = None; + with open(computeJobDirectoryFile, 'r') as f: + jobDirectory = f.read().rstrip('\n') + + return jobDirectory + else: + raise OSError("Cannot find jobs directory for jobId=" + jobId + ".") + + +def getComputeDomains(): + """ + Gets a list of all registered compute domains that the user has access to. + """ + token = Authentication.getToken() + if token is not None and token != "": + url = Config.JobmApiURL + "/computedomains" + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting Compute Domains from JOBM API.\nHttp Response from SciDrive API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def getJobTypes(): + return "" + + +def getJobsList(): + """ + Gets the list of Jobs submitted by the user. + :return: + """ + token = Authentication.getToken() + if token is not None and token != "": + url = Config.JobmApiURL + "/jobs" + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting Compute Domains from JOBM API.\nHttp Response from SciDrive API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + +def getJobStatus(jobId): + """ + Gets the definition of the job, including the current status. + :param jobId: Job ID + :return: + """ + token = Authentication.getToken() + if token is not None and token != "": + url = Config.JobmApiURL + "/jobs/" + str(jobId) + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting from JOBM API the job status of jobId=" + str(jobId) + ".\nHttp Response from SciDrive API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def getStringStatus(intStatus = None): + """ + Gets the semantic meaning of the integer status value that a job has. The integer value is a power of 2. + :param intStatus: + :return: + """ + if not isinstance(intStatus, int): + raise Exception("Invalid value given to job status. Must be an integer.") + + if intStatus == 1: + return "PENDING" + elif intStatus == 2: + return "QUEUED" + elif intStatus == 4: + return "ACCEPTED" + elif intStatus == 8: + return "STARTED" + elif intStatus == 16: + return "FINIHSED" + elif intStatus == 32: + return "SUCCESS" + elif intStatus == 64: + return "ERROR" + elif intStatus == 128: + return "CANCELED" + else: + raise Exception("Invalid integer value given to job status.") + + +def submitNotebookJob(computeDomainName, notebookPath, dockerImage=[], volumes=[], parameters="", jobAlias = ""): + """ + Submits a Notebook as a job, + :param computeDomainName: Name of compute domain. E.g: http://compute.sciserver.org/dashboard/api/container/ + :param notebookPath: path of the notebook within the filesystem mounted in SciServer-compute + :param dockerImage: name of Docker image where the job is being submitted. E.g., dockerImage=["Python + Dev (astro)"] + :param volumes: volume conatiners mounted in the docker image for the job execution. E.g., volumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}] + :param parameters: string containing parameters. + :param jobAlias: alias of job + :return: + """ + + token = Authentication.getToken() + if token is not None and token != "": + + dockerJobModel = { + "command": parameters, + "scriptURI": notebookPath, + "submitterDID": jobAlias, + "dockerComputeEndpoint": computeDomainName, + "dockerImageName": dockerImage, + "volumeContainers": volumes + } + data = json.dumps(dockerJobModel).encode() + + url = Config.JobmApiURL + "/jobs/docker" + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.post(url, data=data, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when submitting a job to the JOBM API.\nHttp Response from SciDrive API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def submitCommandJob(computeDomainName, command, dockerImage=[], volumes=[], jobAlias = ""): + + token = Authentication.getToken() + if token is not None and token != "": + + dockerJobModel = { + "command": command, + "submitterDID": jobAlias, + "dockerComputeEndpoint": computeDomainName, + "dockerImageName": dockerImage, + "volumeContainers": volumes + } + data = json.dumps(dockerJobModel).encode() + + url = Config.JobmApiURL + "/jobs/docker" + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.post(url, data=data, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when submitting a job to the JOBM API.\nHttp Response from SciDrive API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + + + +def cancelJob(jobId): + + token = Authentication.getToken() + if token is not None and token != "": + url = Config.JobmApiURL + "/jobs/" + jobId + "/cancel" + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.post(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting from JOBM API the job status of jobId=" + str(jobId) + ".\nHttp Response from SciDrive API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return True; + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def waitForJob(jobId, verbose=True, pollTime = 5): + try: + complete = False + + waitingStr = "Waiting..." + back = "\b" * len(waitingStr) + if verbose: + print waitingStr, + + while not complete: + if verbose: + #print back, + print waitingStr, + jobDesc = getJobStatus(jobId) + jobStatus = int(jobDesc["status"]) + if jobStatus >= 32: + complete = True + if verbose: + #print back, + print "Done!" + else: + time.sleep(max(pollTime,5)); + + return jobDesc + except Exception as e: + raise e; + + + +# for out internal testing only: diff --git a/py2/SciServer/Config.py b/py2/SciServer/Config.py index 36981ec..e752f1b 100644 --- a/py2/SciServer/Config.py +++ b/py2/SciServer/Config.py @@ -25,9 +25,10 @@ SkyQueryUrl = "http://voservices.net/skyquery/Api/V1" SkyServerWSurl = "https://skyserver.sdss.org" DataRelease = "DR13" -KeystoneTokenPath = "/home/idies/keystone.token" #this path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment +KeystoneTokenPath = "/home/idies/keystone.token" #the path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment version = "sciserver-v1.10.0" #sciserver release version - +JobmApiURL = "http://scitest12.pha.jhu.edu/racm/jobm/rest" +ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. # returns TRUE if the library is running inside the SciServer-Compute, and FALSE if not From a9fb704d3995a60951e4ada479cb24f1b5efedf8 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Mon, 3 Apr 2017 16:33:03 -0400 Subject: [PATCH 04/79] -Fixed install script -Updated ComputeJobs module --- Install.py | 10 +++++----- py2/SciServer/ComputeJobs.py | 35 ++++++++++++++++++++++------------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/Install.py b/Install.py index 2317fd8..1fd2626 100644 --- a/Install.py +++ b/Install.py @@ -16,7 +16,7 @@ def isSciServerComputeEnvironment(): os.system('printf "\n===============================================================================================================\n"') -os.system('printf "---1) Updating local Git repository...\n\n"') +os.system('printf " ---1) Updating local Git repository...\n\n"') os.system("git tag -d $(git tag)") #deletes local tags os.system("git fetch --all") #fetches all remotes into local repo, including tags. os.system("git checkout master") @@ -28,12 +28,12 @@ def isSciServerComputeEnvironment(): if len(commandLineArguments) <= 1: os.system('printf "\n===============================================================================================================\n"') - os.system('printf "---2) Checking out latest SciScript code from local master branch...\n\n"') + os.system('printf " ---2) Checking out latest SciScript code from local master branch...\n\n"') os.system("git checkout master") else: sciserverTag = commandLineArguments[1] os.system('printf "\n===============================================================================================================\n"') - os.system('printf "---2) Checking out latest SciScript code tagged as \"" + sciserverTag + "\"...\n\n"') + os.system('printf " ---2) Checking out latest SciScript code tagged as \"" + sciserverTag + "\"...\n\n"') os.system("git checkout tags/" + sciserverTag) hasInstallFile = os.popen("ls Install.py").read() @@ -53,7 +53,7 @@ def isSciServerComputeEnvironment(): os.chdir("./py2") os.system('printf "\n===============================================================================================================\n"') -os.system('printf "---3) Building the SciServer package for Python 2...\n\n"') +os.system('printf " ---3) Building the SciServer package for Python 2...\n\n"') if isSciServerComputeEnvironment(): os.system("source activate py27 ; python setup.py install ; source activate root") # activating python = python2 in anaconda, and then changing back to the anaconda default of python = python3 @@ -63,7 +63,7 @@ def isSciServerComputeEnvironment(): os.chdir("../py3") os.system('printf "\n===============================================================================================================\n"') -os.system('printf "---4) Building the SciServer package for Python 3...\n\n"') +os.system('printf " ---4) Building the SciServer package for Python 3...\n\n"') if isSciServerComputeEnvironment(): os.system("python setup.py install")# In anaconda, the default is python = python3 diff --git a/py2/SciServer/ComputeJobs.py b/py2/SciServer/ComputeJobs.py index 09c9ec9..f7a838f 100644 --- a/py2/SciServer/ComputeJobs.py +++ b/py2/SciServer/ComputeJobs.py @@ -65,7 +65,7 @@ def getJobsList(): else: raise Exception("User token is not defined. First log into SciServer.") -def getJobStatus(jobId): +def getJobDescription(jobId): """ Gets the definition of the job, including the current status. :param jobId: Job ID @@ -85,14 +85,13 @@ def getJobStatus(jobId): raise Exception("User token is not defined. First log into SciServer.") -def getStringStatus(intStatus = None): +def getJobStatus(jobId): """ - Gets the semantic meaning of the integer status value that a job has. The integer value is a power of 2. - :param intStatus: - :return: + Gets the semantic meaning of the integer status value included in the job description. The integer value is a power of 2, that is, 1:PENDING, 2:QUEUED, 4:ACCEPTED, 8:STARTED, 16:FINISHED, 32:SUCCESS, 64:ERROR and 128:CANCELED + :param jobId: Id of job. + :return: string with job status. """ - if not isinstance(intStatus, int): - raise Exception("Invalid value given to job status. Must be an integer.") + intStatus = getJobDescription(jobId)["status"] if intStatus == 1: return "PENDING" @@ -103,7 +102,7 @@ def getStringStatus(intStatus = None): elif intStatus == 8: return "STARTED" elif intStatus == 16: - return "FINIHSED" + return "FINISHED" elif intStatus == 32: return "SUCCESS" elif intStatus == 64: @@ -116,9 +115,9 @@ def getStringStatus(intStatus = None): def submitNotebookJob(computeDomainName, notebookPath, dockerImage=[], volumes=[], parameters="", jobAlias = ""): """ - Submits a Notebook as a job, - :param computeDomainName: Name of compute domain. E.g: http://compute.sciserver.org/dashboard/api/container/ - :param notebookPath: path of the notebook within the filesystem mounted in SciServer-compute + Submits a Jupyter Notebook for execution as an asynchronous job, + :param computeDomainName: Name of compute domain (string). E.g: http://compute.sciserver.org/dashboard/api/container/ + :param notebookPath: path of the notebook within the filesystem mounted in SciServer-Compute (string). :param dockerImage: name of Docker image where the job is being submitted. E.g., dockerImage=["Python + Dev (astro)"] :param volumes: volume conatiners mounted in the docker image for the job execution. E.g., volumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}] :param parameters: string containing parameters. @@ -151,13 +150,23 @@ def submitNotebookJob(computeDomainName, notebookPath, dockerImage=[], volumes=[ raise Exception("User token is not defined. First log into SciServer.") -def submitCommandJob(computeDomainName, command, dockerImage=[], volumes=[], jobAlias = ""): +def submitShellCommandJob(computeDomainName, shellCommand, dockerImage=[], volumes=[], jobAlias = ""): + """ + Submits a shell command for execution as an asynchronous job, + :param computeDomainName: Name of compute domain (string). E.g: http://compute.sciserver.org/dashboard/api/container/ + :param shellCommand: path of the notebook within the filesystem mounted in SciServer-Compute + :param dockerImage: name of Docker image where the job is being submitted. E.g., dockerImage=["Python + Dev (astro)"] + :param volumes: volume conatiners mounted in the docker image for the job execution. E.g., volumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}] + :param parameters: string containing parameters. + :param jobAlias: alias of job + :return: + """ token = Authentication.getToken() if token is not None and token != "": dockerJobModel = { - "command": command, + "command": shellCommand, "submitterDID": jobAlias, "dockerComputeEndpoint": computeDomainName, "dockerImageName": dockerImage, From 10d1e8b1b625ad21a3aa511ea42858b474e540be Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Thu, 6 Apr 2017 15:46:34 -0400 Subject: [PATCH 05/79] Created Files module --- py2/SciServer/Config.py | 2 +- py2/SciServer/Files.py | 153 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 py2/SciServer/Files.py diff --git a/py2/SciServer/Config.py b/py2/SciServer/Config.py index e752f1b..5876add 100644 --- a/py2/SciServer/Config.py +++ b/py2/SciServer/Config.py @@ -29,7 +29,7 @@ version = "sciserver-v1.10.0" #sciserver release version JobmApiURL = "http://scitest12.pha.jhu.edu/racm/jobm/rest" ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. - +FileServiceURL = 'http://scitest12.pha.jhu.edu/fileservice/api'; # returns TRUE if the library is running inside the SciServer-Compute, and FALSE if not import os; diff --git a/py2/SciServer/Files.py b/py2/SciServer/Files.py new file mode 100644 index 0000000..8392107 --- /dev/null +++ b/py2/SciServer/Files.py @@ -0,0 +1,153 @@ +__author__ = 'mtaghiza' + +from SciServer import Authentication, Config +import requests +from io import StringIO +from io import BytesIO + + +def createDir(path): + """ + Create a directory. + :param path: path to the directory. E.g: path="/persistent/newDirectory" + :return: True if the directory creation was successful. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileSystem API returns an error. + """ + token = Authentication.getToken() + if token is not None and token != "": + + if ~path.startswith("/"): + path = "/" + path; + + url = Config.FileServiceURL + "/data" + path + "?dir=true" + + headers = {'X-Auth-Token': token} + res = requests.put(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + return True; + else: + raise Exception("Error when creating new directory " + path + ".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def delete(path): + """ + Delete a directory or file in the remote File System. + :param path: path to the directory or file. E.g: path="/persistent/directory/file.txt" + :return: True if the deletion was successful. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileSystem API returns an error. + """ + token = Authentication.getToken() + if token is not None and token != "": + + if ~path.startswith("/"): + path = "/" + path; + + url = Config.FileServiceURL + "/data" + path + "?dir=true" + + headers = {'X-Auth-Token': token} + res = requests.delete(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + return True; + else: + raise Exception("Error when deleting " + path + ".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + + +def upload(path, data=None, localFilePath=None): + """ + Uploads data or a local file into a path defined in the file system. + :param path: path of the file to be created. E.g: path="/persistent/newDirectory/myNewFile.txt" + :param data: string containing data to be uploaded. + :param localFilePath: path to a local file to be uploaded (string). + :return: True is upload was successful. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileSystem API returns an error. + """ + token = Authentication.getToken() + if token is not None and token != "": + + if ~path.startswith("/"): + path = "/" + path; + + + fileName = path.split("/")[-1] + dirPath = path.replace(fileName, "") + + createDir(dirPath); + + url = Config.FileServiceURL + "/data" + path + + headers = {'X-Auth-Token': token} + + if localFilePath is not None and localFilePath != "": + with open(localFilePath, "rb") as file: + res = requests.put(url, data=file, headers=headers, stream=True) + else: + if data != None: + res = requests.put(url, data=data, headers=headers, stream=True) + else: + raise Exception("Error when uploading to " + path + ". No local file or data specified for uploading."); + + if res.status_code >= 200 and res.status_code < 300: + return True; + else: + raise Exception("Error when uploading to " + path +".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def download(path, format="text", localFilePath=""): + """ + Downloads a file from the remote file system into the local file system, or returns the file content as an object in several formats. + + :param path: path of the file (or directory) to be downloaded from the remote file system. + :param format: type of the returned object. Can be "StringIO" (io.StringIO object containing readable text), "BytesIO" (io.BytesIO object containing readable binary data), "response" ( the HTTP response as an object of class requests.Response) or "text" (a text string). If the parameter 'localFilePath' is defined, then the 'format' parameter is not used and the file is downloaded to the local file system instead. + :param localFilePath: local path of the file to be downloaded. If 'localFilePath' is defined, then the 'format' parameter is not used. + :return: If the 'localFilePath' parameter is defined, then it will return True when the file is downloaded successfully in the local file system. If the 'localFilePath' is not defined, then the type of the returned object depends on the value of the 'format' parameter (either io.StringIO, io.BytesIO, requests.Response or string). + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileSystem API returns an error. + """ + token = Authentication.getToken() + if token is not None and token != "": + + if ~path.startswith("/"): + path = "/" + path; + + url = Config.FileServiceURL + "/data" + path + headers = {'X-Auth-Token': token} + + res = requests.get(url, stream=True, headers=headers) + + if res.status_code < 200 or res.status_code >= 300: + raise Exception("Error when downloading from remote File Syatem the file " + str(path) + ".\nHttp Response from FileSystem API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + + if localFilePath is not None and localFilePath != "": + + bytesio = BytesIO(res.content) + theFile = open(localFilePath, "w+b") + theFile.write(bytesio.read()) + theFile.close() + return True + + else: + + if format is not None and format != "": + if format == "StringIO": + return StringIO(res.content.decode()) + if format == "text": + return res.content.decode() + elif format == "BytesIO": + return BytesIO(res.content) + elif format == "response": + return res; + else: + raise Exception("Unknown format '" + format + "' when trying to download from remote File System the file " + str(path) + ".\n"); + else: + raise Exception("Wrong format parameter value\n"); + + else: + raise Exception("User token is not defined. First log into SciServer.") From 9cab5ea634ca4c65d68b7e5dbd1d284753a5c560 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Mon, 10 Apr 2017 16:14:11 -0400 Subject: [PATCH 06/79] Improved image cutout unit test --- Tests/TestGalaxy.jpeg | Bin 0 -> 52726 bytes Tests/UnitTests.py | 27 +++++++++------------------ 2 files changed, 9 insertions(+), 18 deletions(-) create mode 100644 Tests/TestGalaxy.jpeg diff --git a/Tests/TestGalaxy.jpeg b/Tests/TestGalaxy.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..3323e02ec57889879c93b6f9df90ea3bd084e7d0 GIT binary patch literal 52726 zcmbSybx<5Z_wC{mAi;tJXMx}jyEr6xu*Ee52`ug!JP<+_TP(OPzPP&t*RV)%f;+*3 z1Rme-S=IaNz1LGy(^WIo-KVCzZ{Kszef;&f26&;OpsWBuLqh|YKV5*wMZg;XE;cp} zHWn@p4h|k3EFH>xX=oULJZub1T+B2yuSMCp`1pl{gy`A8 zk{|&I9zh|2|1^SzhlfY-jDV7mkWzq=hEd@Exjc3Oh;h*Y=l~2fCIC7y8U``i;|~Bm z0Dy+|G}`|R_yI|JP&;=IPcLsDU%#-=;SpaVzed3m5|ffszNe<;JCre|j7e$6lZUfBb<&#A4*fCXv;``QS>*BoKm2CKsPu-HpdA z2>C;9;r0{%B~a)$%jtih{TH(TJ7A&zU&#I!u>Xx~0YHF(_Vn{Ghyl`oO>>kd_{1_< ze8lk^LJ3JZo!h_hanac6LCV5+SLAqa%uR10X*|=$CuP2k0<(w7KU61NS`fV`ev*LB zw`9#Q%~(9rncFhdo~1FPe$%yz@^v7j>x#xQjo{@KWXWX6vv4%t_&YL>tCnWd{0IQc z(zwT+)k2ahjO%-;47DzMqHdd`e!SV=1Dnn(CW&OunMzA-i&PVc+@>1)yme{gSE@c{ zbNV~elI1_^j~0_L^X;ISw@8$f{G5*hKSbt#%r+f-ABK$seKk-wyOZhbVLlb(0yC6@ zMVsoEv^~fCMwoatthw7fKjgTl&|Zz)S1fZQ_SVLV|8|f)O=AMPg#P+)VvkAmVMOa~ z{E$79;HO=Ij?08?_rUY_-p+40)!pISP+s+w@RXXVZB)g_>g3bH#)Tj6qDp1W+tH-* z+r|6rzHJb?>dKX_jsq})CHjj!DjbDv@9GN`e{8@Rx?9I$KdIK_Ny%->}DQQyW{$Upu zGc3mZr2HQw1i0kKuncMk1WeL^t zBY>SK-Xp*yedG}^>1yQFGG4Gk(bk-Ndu$V{Rom_RJ;R|1XmtgmkE{4#`4KEJoxUAK z#k8_6${2;w%?twi!LmmucOmtO`b)+=Q?65ZpJJjM->8cTkBv2C?vQ>eAMxhogY54_ zrw6sJ3P<~w3Lljkb0h2Pl(uE67^y7yhk$7TS?VFh6=JHG*~yoWfcY|^<~^ACsg??i zEIZ-&j+vT_kxFHmF0Jt=_I)KK4uXxDQeu*SCDdol)s*c666#&_2>2NAP1CsiE3A0Z z@i^y&`vrOJZ~IKTIkP+Y`lO8^EaY_Q$7Z=^W8YUfv5OTx5?2+fr5h5c4Y(5J5Lrw? z8M`%TkRep+Gs*MnauzvmNwW?c@m>X!p~lMe>Q|Ygqxxtufr-SCj{s^YSSmF)MxE0= z$Mvo(>e=QFWuIt#ACC;>^>%O*P1U#`#0dz{AUo^YLC+tc%2Am6_To9^fn5~2-h&H( z65tzc$np%}0xnXMND-)mw@SqHW&VO2WEEs>r~MkF+Muh0*T+rORDHXSED|S-!ZC4| z0WsJhM7M^QX-4F(RDj-zB+R(e zM(b1W5hmloY7lko|DmM#)1NE%ffC;n9mKl>hKYSvep7nN?NZ6wUYrrIte zB{}uB6%u+Ej0ImBY-O=vUSL?72l8la80dY2Yjr#VHpf4`HMud*Oz^s=N*bU!hlFb2 zi(L!9=wI4FfZz)D{ZV>I6+mZ{0R6gEnlBvy;n-ZI3eyY~ z*d{O<*DcMe2-~e+a7Xh#`r`(18n232!_$J>1_V)Qnpw7G$r~ZJ0Z%*Bv1zuaTw8gF z0yaPkM&PRX`-Gis=KW-kN4BVtY4-2A`|AsFLb=sm>X+Z3Wy?NR#@|-!4b|+irpvV= zroLRPTfq|QND~GrkL?vrPsv&~MgBbF6n|zw zr~GnZ%!DnK5o>k(BN(IwTLS>lt959mig3s)wt8q%{2TxA_R9JWT^1~X)JVF-bc6{duSy z2$K5i*>R)k=LyOqqlE?a^KOtLe|=^%7uiN+l7_UFp$uU&Ykl33g+bEMi4!kNm5&iG zI}4kWA|}QFxiC0EJE}dhyf7_;{0MdY2w>);(B#h`f{I#geML&w?Q&i_)KfeH{EjGY z6l}#J;-rqhG~JGP?K}cx`{<_LKiRtH=k$%49XNl3QYF0H4ELzROeEVZ{C(1Z=aa3Z zOvGmxO`n=2qIw3$m2PL!?>2(mYHo;oN~rrws4UQnyY7BuB^gU>MtS4|(~}Q9EP)wC zt#f}=FK8n^#V-U0LY~`+0zgLUBGl9KQ8EBx1@+~eKTrUJ%`^lp?S+Fx&ipZZ1`>4I z4`OCYAE5@kJ5Te81&^S~^c?HuY|Q=1!NmMGsmR*07=*WI`j<%3Xk}ZwwEo8*-yXu;F*HbnfUGk5bKQ(KmkG$NT z-l$Q1y+D~~>=DYOZ4$1qE*)co|%vO0NWA z+7_?sj+P`{)v0I_l*)Z*%W`X8?0Q9eIbWHXEF|D760^81(GCO$Y8%d!4A$v8^dsMY ztkZXPKG4p73niM?`*@+1R&yJ0p*JQtzr@8EJtg#xED%(NV>+90Ub<>oN!lTuDK%%{ zdi8mldi$6&@>5OTC%&7c5X^|+T>;0m(}Q4*?o#etfa9QTjW-k#*Ws%(9A&0vn^PkZN)tL_X(x!PwKcV*jWq>DF|CRkRgM# z0ruG#+fp+nl{sR|)0qS?;3J?TSb8?<>#z!~=v}#3wI>^OfUxtH^EutZR`(5jLS6Rg z18LB%ik)iN{K>t<)VPIB#2y*hS8cTTrW=`uBYppG+C^NP6el=(dDEP>Vu`*WQdzdG z_Mj`fFY5d%Dk8P-zn4xT)D$#TJ^!{+neS8zeZF7jSIw}Ys|&B;Q`|7Mkqtztc1*uX z7x#)MMO$P5XsBzDTlu)%zbK!zo9H6+$7TDooB**WnV;t#kB8d7)@{-kRONG9e1q1d z1v?(IEUR;>z`aqIsjwX`pA)Dt6?z1Oy32W%@_0;X*q3#7N2gIVHo>VB=9w+Z1v7}d zk&fVtgR<8`i}O7BLG>>cqJ*t)dMbfc4=1_Bo`-Camkd%T3kA%(2J5w78P5tf-3WCv z4cHnX$fFVk{o2((6qNm7#DbeT=tQ~hJgh4>eG8yBC+6VC7RvS$)!q1K|4(|P)$~zU zh!y^i3mn>M?;_>(SnNK|w-Ukvi56{KIPsS%=Yy%}+erH)2rj<+YHa#Nh1u=JzFWmFE!E+m^wOjU z!yW;(y1RA4*$Sd%0{5NWnhOTY`&sm%;!3m1jBRmorut~f5}wR=^;yn5%&QItd0`Rn ze3$P1a*@946;+qOb|tT-^JiiBAVs z@PP^$M$<)(!#gLbo&=gKB6da0bJbTbG-$kYgq*c-)MdL6(%x~{etI% zW5BI$V)g3cg@B*2u8?`<{P#=9>??aYD?L1}l+=Yqb#b&gT6-L$p$>iPJ=C_lD)Dee zqOr>DQ~iT`Yy4DLSORLHo8%u)?RFXY#WFS zKpf*ffsnRk?+>^szNLez5`~5oId&1y+bwofx3o;vwMIH_!$C);rulN+UL%Y{%te@b zOoaWF(1nZ!3YXftN{7$K;Z_VUWN~O)Z}oI7SrZ}-?qmE;)9Z3u=H?Vs3*Yxa35JB8 zF?e>d6_?+Lq)M4>H^hAVm;1rlGv9ET+{S0ikVSt%Vabi$HRET|E=hN>+-;f>cd>Mv zp)<4D%J}m&aEU>h7OeH^plCD}Ii=;;DjdL5d6Oh;lCH+8zC9UsNirxMN1oxPu7R_F z+!y=y9#_Y>+nEMei{!l4N>oooj8p(> z#F^}fyms*0YZh+;nl(C~Ft=QQoGJKuW_1kS;-& z$M(=&V<1Fn{99~Gcr_fCs$ZHq4AS1%0!MQss~#X=VmUVvq#<#00i4|G6!$Jv=pyR_ zJOBK6jk>xntXI`T&OK1(!p_rbt!+GuRj>S&-c?YzVpLs9C1H=_v_p4;sg5T-{lQO! z%MyT>#k+BIe9~{p9^?spj~U4^xf2YOLd)ndP;K?E2+qg zT9JJywje6TE_y3{(aP}_raLCB)zMPdT9x%XYuO>&%8G!niTMn@#)CzTZFIquHf0lr zkn>11PEXXWbL2)k<|Es#!Dk9f^gT&YM}R;f*k`)e-4pNTpu4|TW6oBML-#q0>+V#u zxv9mIi4c(-MCuyi?vuK*1@r7orfxrini51GhGJ)HHP_5z5unWyW{f%VsQi&XLx_Lg z%`lgx#?1uPhHT`B*wRlEyXz-Qn@Sr0WkKN>Tm8ADyI!_pZE&fX{mWA&Et5QX&N!+! z&wpcpnA@$AEI2jvsX80o@4dVp0R<|ZQxO@y(7t)&5k%Y9L>d#Qf*PmqdsS}*yYq}R z%`VYp8vJQGu2Tjy?H|;|0CO~dHhQG>@R>5_7!Y;_<~rQr`f&-o7Ov^k&J~@8Ww{$q z9=y@4ZmOTH0#f+mR<$nLm(&(WgA_`_Cabezxf3+OUlE=Q` z{#T@-2AS1^P?I+Nxt}h*l!nUWv>bov@?JZk63)1rpT`)M((l&uKF;ud81Z0!_!?2& z^90r;%C)EuO@Ft}*kaydcsNnoET&DR8#TaxZ<~^8tQ;zHeFRVZs~gxelpu!tBJe5E zv9aFnDX1Tg>*R}i-_+kw9A6qbGpM>d+GDD0U3JqfRZ|X?ue@>B=;F|Tu;Y#Mfr}_# zm&p?eZK%2Zpg8fHI@3uei^9bU>1RjVfV;k{mJ5TOs#A8GJ=BdC#=+>=!@vF%9jq=r z_GZ)}+MGqr7seN{w^h||| zI;YYBa!#SVBJ%KJ1jWxFU*hYmQUsYnn3a_#i`5ljzzb(@6tvP$B19&`bY|@;BPwN z#MB)fbTYV}$Pid)Olo2)K#k*`lcj>!gMqXZX*OfoNJa=Q=5)BrbIJiDA=+c%B-wsH z6b{+H#bBIzECB%!nFFU6S<@#XMWpFB{prJ_*=LR_`;_|P5OzbEiOE?XareI0-D?^6 zsd55lXUzhzk4Ry+rG>0qG4gcHmiSkpbRWU=UrTezZYQx7%_2g50XFDqQL#Ot#4rz) zo?y|9#dfGI;1S@!6f~i<75i`-{pGT}ZL0}MNNsy`*-Et?2UHdFjoxkEe9$FN+8g2- zop@2HqU;qE_fIk<(7oS&>Oy>(-yt%C@Q16Jy{nbSfsfP6XzVJ zULS=^hV7TVY==0!%{?VcD9J&R@{(47O%xuNtFgVi6~n7&Rl1+ou|lKtty9s>R1+$c z2G2hI0Pw31F5Z{&_(l0z;I3)jp-;-VD4)nMgQPP-(MusAODO)yD2(E*yMncy*+h6R zen2rZUB^Wf8gK`o$?YIymD#kNSr_WVJQW6mw^@WQ?Ig%dzG&Lq_=8viEXd=~-cE^X z9jS#K8r4)~-P)+Se1E&LtxQg?&tStPv#+>gL}F{uo2jTK8*|OEdP`*20aK<9wKyB}JmHN0i7sJfu79qHk(__U=NKKU@#<=c+K*f?t(d(EbOtuZOWTU6qU_Yo&I zXV{YL8YXFq{qAsE4EAZg@|E@KP4n56GH3FoximyYuA@M7jf4u2sI^MGm{tR6zBbd! zEf)aJL}uz1n8r{GyC}%+cE#lA=`ylT=Uu-4LtuwWx6|~iY(?Bk{V?-@vo6i{=Wt!L zzCwzCkJ*~wN(CQnb%|XHPR~i6XzHBOv2|0MoN1QWhvpB`7cV%%f1s8IQ5u3{%>Q=0 zxCZvs6oH#_BHxCdXyvy}GsSatKY7Nl{fzUN0ErTTLK#tJVlU)D<`v>|vXgwIO&=5C zM1p;oalZSnfjd%lfKA7-uqd)a#_pS>8us8|_b4K~!9AH4gXNiv#p_e-yxC-L0Sr$@ zZJKvWPCyTG>fN-Z!$MPmJ11Z6XGGG(I;+~#JQO6WCM_|1D1Q3!l3GFd_>XLeeb$T8 zynN51lzTFFRJc#K!hyyU>8RuNz|}FSP+Ph;Qe?)-lgl}1nX1i0^@WNsWTFV+zUP|T zKOBu%YA{Y<*#*)6AtQGm;9xqPt>cs4I9jA@20zq?NR)6M>pUBf)mHgzLaKI9adJPp z_u=;;>c6T(m#Vt8GrZiEyr3ZoQ=j8ua3L@ zu&J~w7Gh`{D}#RBmpFR_+~NxB9Q4M!OFBd$BEJBAvl`yVr%j>jgag*Uqm8d~s;hCl zJ_5FU)2eSaNY$NXWMBq&-y{+(mq{j7FXABV@k$&#pWCbRqc*tOF%(X;aB(%d#-#QpAXNTw|fW$O23n9A+uwTC@FW{A+fnWo%XiMDu8o_xIIMN74!cW%-U2Q7KU8yyc>uNoJ zBpxmMNOZgJ1!8_&Btc0V)9Nw@ORK&`6K4#z9;14mhP&%O8QcynTlpos`PPE}gaSAr2YM{EsI6(N^4poiTHg*sX{#l)aj{e!a=P>&$(S#` zu6Dx-$RE!-E#PG3sZ`9<+*U)J68-MXRH@%;y(dg&RS3t?1tIs3JXJdq_~Bcw!TVg(H#62KT=zP*d1G zkn^pmD9S{hRRhX11s{%dW;f>7xn-YuG{;%p@fRS#BXA>Q|4;4WUsVvw9BxyK z3%5smjw4?Bel&B*lOqKRac!kF5-A#IB@Co-GBkcOua-IdDIv<#IMGXab%F(9%yUrC z8p`R2ISO~#YDq(r!)38ekeBl8g!Bf=!19Z!K`$0b98w+u~fQQ8O0T2fI_qp%=;~QhN zU|M!xny>^l!v0kwOpHpE_=Z!|o{|g^_+U1a0$_8RLs~Tp0v#z~E7Hn?^($2XfHOkC zt8u=ipN#+!MadEYA~&f->@{>y0b2Y7t*tc!sw&5Zs_=NpcTF4HY|3%HZ8|S>Cn#C> zD{&QpOF=Lm4o%2Hk?I(g0(QLk)A38vSup!<3rn>3;1NL1nw1}SUK+HjNIzRrPz@}o zIbN@VZx^FFM#@a@XpZqcjS{?S_U8ioui{;;rSfJ+ij3yVXU}ufKUEBUPtR}~*tLuL zwfDMHs>Xv5^|?$=qRg7m4y#$iwx>KYNzYm@llg(V|Dx5PSKZ)%MYuS9SE2%ip*ELFdwdp3jA`fgw7zi05rv` z@g?L&|K(ZN4=b)>o%n+>y9r;U1e3D!3DxuRQN{BT2dmndvE*!T!ro<$Hm%`C0J`!= zDo8<=y4aMb4ZSbIsbTL1-;-0@1ZCvexPycNy3jfLq6-?YX6qz$@PG~NRv+F{4+K)OlCR6W>w4Q1$KcQQN z+dRvi{NRg1_k?&^3^vSD6=Tp2jlQZ__>Bwb9_of+6O)L7i^L>G@%;tmIPQW0<9-C* z1;5bx0~61d&IH)QV;Nis84bMT$^aZv?N~fNVaZ4}rXKNw2-2^~xWD0gJ}KErs+L*e zkc?^O(ZLceIUdOTt3ckFBl>>7?gYh#WUDr~AoCFKSbbDjxVadbL9byH-=}tZwKp-R zf={5{+3WW`&*y~+XI`a!P6+4-xO38?5MKoSo8J&lbiN43@qJ>q!|ZcTy%XVsrA) z5B6@XS@TESjy9;pQCSz-d>6OCcPS4v7p2K!A_?x|^8ecO?Sx`_#v<|a*_tFJvEFfN zhbA9vF4r}xUXgRDy>_>qGE-Z8i$vT}wFDgfD>!4#P=*4q-pZcGWW2B&h}6?K`bsrf zR_CfTtWU{JUA@uB+Q7X&Db!REUHR1-TKM?~EjKWh${Kw6KyTDFlXEA$Ah29NH9GEX_et8EhRc4>c6QSEpwNvAr_2RMpx~i5_J*E_&;PU z>s2j*#EVl&Ha~rtRXgH5Mph~>U_oLZL$Z&oWqjaN`A(MvMPDmvH7#k=5=2~+wtY>ZPlRAnXC!M zg!?-#yj}L^z zh6>&v@Wt2057AI-2o?d>YdjJX{0_E9>F9{m)!IKT+(#>4J`+V1)YhQVo(s<`uUImQ zyT^0|5tn6IQQzywa)|{boqEQpe6(7Hd)xRE@48@_e{r0rq%2HfshvekzPC|aIT^W+ z^szNwc9gnyZp><>^$4W@@~$e)QT;#}uE4$ptR0pM%ho8qA4b|HkIVDgrgFXy zkwsaOGG(@g3?%^nELl|5qcW9&5C4#@53FPp2Wb-)1u$2WKe-(7+KJ1{*(vH@Bqg}^ zyLB=Zy=}HoxxzKJw z^CN(}Ub2?>QM0b>%FF|Zo+J4%GBnZpU8gb%947)V26B+~##w=hv;Cj~Y5y48i7}aH z!DN1E-ntuPuvUEP6Ors`SI>{u-qlFugFN zwibtGLR);sZjaNMLe(kOJp?$ZXoFEQT<4{QUNlYHOdjVN66HJit^c$Cij}OqJ^RCLvfKPJU9W5^XkHi_DE&C_x9E zd~?n~WZyD3Rtltfumz22abd{=LibJ>LOITu@i0HgOTc}85t1`FUBg|?GMP1_hz%!= z{-cap`z}$K9}V?VOKUSVQ7W0fAP}3KINuOg@81S#{N%s=K8{tNN-g8by%fL7FXhR@ z7=rTd0wRc5}as!G+agMt^Cw<_sv(s)_Z8b37Y|GO!Li9GC6(G{pIl1p>uAlyuR(C*X;Q7>OM15ixtMZC#(IwIO!2^!6qDk z?jh9X{eHGF>GjZZx_Tn(@!a}IOH2vNZ4dQp3xkkfas4j{&g`;RXnjwV_!|oYpddnB z%Hp!O#d7@7`FZ5J?&Gd9ntnlIRE-2!eS|RHsp7I{;RE0?QOPgpv2AOleLOf4YtNq| zhPG?x>U-Bdw~XgATM znM4dd`BS&50`O_HitpX$deF(Gfk zMY|r=!r%eQC~Qh#D3JkyN5!W~KVMq@+2$2Go;`6klxm=&P*uFQa$UB(eW2WusfM=; zYH9fV-LH}6A<&wRFsI`}fW5A8899KS0IC);H`?-~jbYZi;{-bYDFa$=W~9Au6P!wr z=HdT4dz@rRTlWUiW-kp~9RAZM9%^My$vZTC-%C!;QR>>tic~_y3f}m3^ao~J|I6iy znj@Bp@3-=D`C)EZ6RN!3AYCgVOqd}sQM|0&$zg;ES`vc^z&vT~5Dg`s-k^GVi+a^y z%a0Mm3@-jQRGVv_WOoMTEW97*Ajyf*Avygt9uGYCvHcCXDs_vF3%9)CPfmn0^Cxcn zOXCE1^u_yUa}O(Thl>7s?GN7?vkSi$SRiE(#q$t{# zLUyk;mtLXlpjeIBxIWVY;`jj-T4NeFK;c@+HCf@CLJ|%<&LhRFY@QXI0 zx`Ic5uPwdQfkJ$2@tmQp=cBA`k+Qkcf>qk(9SgS+cBCJiQvgNjjI!4IER84220fAy6G=AeD z^G(pMVM&{b(rI#o!6RT)?o-fef4Lc6xupUj=Yzv`3|)GZ+uTfpQOk6NGYM$t?5%6E zl~RsUgfSIJQy@_5M%OcSBN=RANfY^old&7qgW*c3XI^3oR%%m?w504a$T-;8b>1Xvv51{Hff{VC7Ihd5p!S3q48$gs> zvxKWM92=a2LyPz#;k_Qw&Fm~I!&!$tlpD42Y;HF0y_R8(l-5+*t1KuQfZhmw%kBAZ z`HXtKN6q(}C zWzjp0)as;eJxa@;wNov? zm)^7A#I3E|jP&3C<{Y6PsN67=qa`$m=$L*A>AU@;xEa!Wd7u!wU|?Yid7JWTdW7af zHR`WXg>Y`|H~Ro&>1AW?Pem&7<0g@I?$SknzfZkt&7vL5P=kyynn5&h7k^F24fQaL zfMz4!$SW-?$x^_i8!LQAPD{Yg}7c{^Qi^QDakSR~@Vup9jMDG?R0|9E$G+ zV>%*=9|5*@-i{5%cg=$H2+Qo*nk#t1^~)^pdzh~BbW#aY;<)n}iQ?~N(@9I!Hk_Fp z>j#UHmAz{}j|1{0Zqk+$P1nI8s*NT3;lix>Ns7c|3twT$E9siLBF>(I_>JAI&MP~z z!n9U3X)j2NxDr#A;KEm>)i`!$({bN^F!G<;Y#JjjnlgAfO@0%V z`c<+y-|0Ecrn{2zHT4@mj}wip1T=T0`9;_i!A97aa23&l)OC=`QdK>*;}f&uY<|Sj zaNuLjq%ia$tg%SOZ=OlSz~DClqbOUeH{C0r!KYr07UzG!n#Yh@6}xs3A1*}bO- zmawUCeFX47i~UDAy{t=+l3XFZg~;rX#+?beGU_?Iwxkt$Xvu%MZ^j>8Z>oFen~NRJ z>du;5s1QJX`szuK%UWvp(y?f{zP1HyLi-c06N-izhzT+2-O|`w; zQNSAmiJ%P@u(Z-6OhFv3HNprLp?up%K<`bpIvD{GG*_0^x8PtlJ6w{`*7jF$5=~4d zJK7*Ktu3Zy{NJhgewl(-k^J$qNs-F~@4RzB{rq7^Z_X5v=+6^j9#VCV9D;o`q1Y?z z%ZspJaTI3>@xfF^xK~^dhKhSFTf1PR(6bLRIPE-qQrEwg@`bKsuzi`y06lzA#TOm-q3ptrC12^`dd^C;FcL1`d}PWLTr}%<-ae#spDI8~^r; zH_g%D{uvDHsxT?Iju0U$6b#?7kUAgEeIjl(5x61ebei z<4YlvlMF)xLQMcGVxy;x{Lj2erm-$v4zxP^U!@_W+(Lm^P{=$Jkpg9|W@f`#WQ9Y& zidP3`#6Y_{9DtTTA#X>{U!V26+4Uy@y}Y}3 zU;?>&2O7Fl2*;fO0Wd)#bhBa||6Xaf!jX>Cu)>Nukcv(ia9>X3b22n8Xb2RN_H}>9R=NSq z*sd>87iJy-+Uq59C?@-*lNvaLt^|mvrWbr~`Hj31NE)#IczIbulv+OV5BS1BB;{neRx~WBW69_4CI=zidubwDiX&sp<_sHugdktI7rh-?JTiFM z#Tz>>z3uUvcr!jG)fj{5rhhPdsry_M*^u~qjX9Pee_&w31=BQDY3Kk)3PgB>8K}_T zg0OlDLTvMkA&iyGU;!fKOGJD(jrW08X#Q+c1@8Q*mvO(}T7S^`2k8fx-8TqOwxD?} zUHTuk@>4>Eb;&0ajeAzVZSntZ7cgg?HtHj=jBN};@U82k6aaA7WQbqF%`A_A9gn2+ zV0XV0U!tg40QWg_RU>ugBOr-u8)&eZR50Hb9BFVGpM@o@pS1#Y46hyHa+Rr01*(Nd zvNdQ>G9_ye+qud7`Io~4;uxV(3J&E{9LBOKA>fD09I5lChzi|0q}&UTeSqkKcMuJ# z@3V#?1D((|Z&A3t=I0PAik+=OK49{}n4`N(_Zqdv#&(J;Gv>e7UHfVn6fTxw3fM~+ z6G+3^tcYXuW=I zRYTR!OU||8khiPVq7}gA0HgR0wl| zQ-vjf#v2u>9h^=TJsJX!!Y-V5F)~N8;CJ|PF8ndJR9cq>7z!@-s){WQjs8h(q(Y=U z7ED}D^a2^@4@Wj=S^kic9rVQCW3L3*us}_7K5XQzs35sna8_b(V?0bxYB=AdZ8f4ZV3byD zoTkEVCFT(jFhU)DvREl`O7a#RXcwuBrYjc$SKh_aWn*I&YJ$kpzJ|Op_eM_Z=yU9P zC~)v=#SChCi#D$#o^K{Y&?E&lZD_;AeHh?Gw%oSi{Tq8v`iwUL$Lyf_q3s5#f!|$a zoq98}lCFgciVv~5bzj}8qr)cZS?mdkPA$Ku)H!og4YO>j|I$i0Dbibg85#vki5il>*ey#y38%=wqZdokF8Iz&G$GO zSm9;B!7Lh$b~#mJE?x4ZgUftTYEIBJwNsR*Q1XHjF><;*A(mYfPdj=n1SNT=FX`LQ zU+Xm!Xd}hs`q^>JQUAS=&2sJj?bwe$ef1UAP!Suir(C!gJJo-zJMTcvUw>974d%UU zo~#!Z;P8zJJBQ?mAC8=DL-IXSMUGACFYAcwngOlI|^JO=ZQ1aWJQH!4(JP?xi2soU{YUfiL zFq(?e+*Y-E(%QOChQ-G+>#m7nYA%s|&_*J_2|LX9FOE9lICeT;As_-yZ>NAVIFVDdLq)s?&et z=qkgSe7pD%q$C7E=^WkNE#2K9EzKB>v>+gz(jp8PFpwCX5~GpskkJT8OZx8retY(1 z&votEeeQF9b?h`RWYaeJTajLje_dIUdWg(k>o0{lSuU@SN~#N{4khBrD`et{zqUq6 zD64C!4X>=f)Uk=0RvmWp8TaSrhE=Am^W5#Y5{EI7+MYa}>ijv#I{CGmMhVBY*h4u| zP+}h!*!(A7{L7~6pB5J1zQ16Q4Sg4gg)TDzbLDa_gUlzodU}&-C7n;Q%J`b#vFavw zAG%^f#gvyIA(u@XGh(@4cW50*{cGafA^!lFcH}r#+9Zh=o9NRi6^wLy^C>4f1ibjZ zr9p4}o+JO1FeSam1I6nY@N`OCW8Cp6YuKj7?PH=9t8iU7k#yMGi3;M^YqQ~rvFuV; zz6LyJJcDdR$_~?9R->~@5q|w@saf=f3f0W_^iU)YhMgh})h(HKcL^IA&6Nqx^LATQ zdPT9wY;g|74U1lf4IR`(O6ymwhV2hls7%I~=l|}o?(?R7Vy??5{nq*+DNs%KV_?u? z*fnvjfJ>W8FwSQbl}b08WyS>?pY=O}nUOPl#S*h612=u;l3c}!DA|Z#wNn`uFmcPk z-Ag0IXh(C%o7hCrqTgoBnu8yr^$JX8z}H2Q{{YN74vT+1x-KWcGx5!+@6~ThE>PjlQ9A}UrobV zrWoJ_zgp)TF9Q~Yz<~YI(z@kX=&-xs*cYH8xqDa6fYriJo>*U@tG1NQD?lAX-A-E7`-wZ><^zvi zNjPb+lE2?!`qHg=tKX>dEaz2~`O3%U2)hEH9oMXCI2qOX`dpOg3lM*?UhxnHRiuEf zeG-;nRjalZyn`Ni$`Y2@ZR8{WN^dt$d&>OEmgq;sd<-4|A#+hm?`JP%2(Ahh6~z=K zQNkYi0IhH5dpD;V?I2ze->omW*yF5#q8}-&gC|P|a=k3*v}UsOVs|q?0V;O!$$}@s zC@=TRQgF23G^Q+VBKzKHklv0rHr1W(wlF==3m@bA_DwlQP{gt3YEa?(*I!#10eU_| z=wjuVE2~#!S-|w^a1vIw!pq7MjQlf|Y5*hoxKxz;fodXyK+g^2HYYfRby@A?cp4J( z;&a010Q+?6RMjT~;gmd8%WNMytxCIpfWoW`1nQD_i@clnr{ot$Agh?|11iY&Va;Gf+A#p zYRi#N*OYz@=S%_p3uq$sF}S>B ze77;8rIsej(aB0P27c1||M7jLG3#7L2<_(hr)7yRSt|YfpH?^cY$8dL9)~LZ-c0qR z0qZbVc?lsP)A435_ZdUT#>Szy{zq3WtGS~U1DL<3`3<+33%?7Yue%UD*1CSIN7;!Y zWhA>J_?0Ic6E^OJh$Hu|UEdyuAs!N3^pa)j%~)I?r~d(f8f}GeFEVPg+aB`X zR`8<>hox?DE-1)@`&%L7tJ)%XTU36^dSRw?%AZo=z%LV`Lg`+H5O0aX%*t}iRX0CL zDQR=o$K_CELtm-rtv;8ZX3GfS!ZNl!U|ankG@GXbD%?SL;nC8zop8pQ{+#aXLK!p1 zG7;xog*~z+dl_BMPq##-!;yT=xf6+$%Bn)$o5%BeLb&+KYw2d)QUz_ROLIbg?@MNm zxo;M$i3WQa7i&f^wfNG9CWT_bskzo|OP9xT!?&?}ot|3x&NT+UdJqboo#Y46s6sYE zJ*;Q>{o7O*@eQ8k))r;^oHZ=^Qzda0vbMG-sqv53>Gm9JJyZzmx=6ynuSG)Ypv z>*+C@e|*qe%1Cwtd%+`JRnh2@80Iq-(TFZ!M^oTZGt82bY3wKEg(Fl|tS!{Bkcbf@ zrkEd19u&#D8fu7_dtKkhV%R049s>v^uYQ%5p1{yun>u-_q^@_xHk`1PkWSBIso%iftn0lU=F|n~qMxb`7h# zpda!cb-%6CbSd4A=UZP*Uz0w*(C_|kp3Yrfri@}p=uvFhOIHDpb##KDsTn#NHt)G3 zp1!9Sv<=mlyVguk?zx(!4Cb03Oo)^BJ3yNfa0nxMax9!i<OU&qHXNl-mlx_}mRE*^9{=us zB_yU-Q0Y=Qc`~&W%KQTFVZAIrJVVlpm9BO}gBUfC`~&>Jp<7T~UOfD{c%~mV;8g1+ zI?K)X4-iJt@Y|Q8QwpeJuc)flz|o(CL*9EEfmoL*usUBd(3hN*I@DG4O_%n6- z@|~zTq)kR%r?m;}Bpg%>TdZ~^%NIWF?nBNd=Dv28 zYnr;OB=-`xcu?9%%Aatg;qp;(JulM3ra1}w0gTG`o+PEMmu{ko&z&NiHW1jDYcX>hY-2@MZndO zo=e)na>#a8T*=Ca-HWU=XG#`8{_#5*c6B0h!Q!eA*x$pqAfMgoPaVr$Su+dP0tH6B z!>8-bJluk=N2y@0x%2jKo^Rpu=+6Pa5(-PEzf#P;Z_%XGkQaaTsvA}PGF&fWDCxc@ z)2`Khx`VN?r}?XG2t8@&UpKg@nytG&6xp0xJ2x$djMo(MBxFAqk%SoFm;ZE{y{XpC zlsYpXt}ZKakg9>9&JBa=3q`kbvQ_M!l?BH0N{Y;)MNh{l{~U^@*P8AApu&7)EsVsP z!z&AQ30xo9OG>D=sH;i^wI9!q{w&FV#^&!K#p?E)>Oun}DRbI4kT@YFNKe3~4Wwy= zz)DpUL%^9@F5xMw_G#j{-c(KuG3dY=FIGHo`gGo`r6Q#kzig>Z3ikHhY^u41)tDW; zIm;hi!d4zZ(?aSP!f4nOfx;8pV@L+~(q8NEgJzTUb@oVpua|j4pK}|{jnEOqgkz#0 zHnmWVgs9y2o7;xDXw+u|>dB^d+K-vgyEe{3`Zgy~V&3$pJG5@tn;)lGE*i{)O-8L$b2+Im= zZnE@V-7V6*;@2A7@<-MnC_Q0*;CRc~4@+aFwpIb*eJqf=^UUiFnUUCi=MgE6(kd^u z7f2oV90<>gMZK)BiF!RFNU74LY*JzGvt$HXl zukFLKP~wBXB>OnTub_!Hx z!j>I+cG$_1AfXr>=kd2^s7z zt3$K9H?T==`jzW&k{(u8?e2>f%SW%um?nOC7r**Zw|JC~*|vc9-^Yf65bynYQG&|(+9w@;*ODc0O^vVcI z&fOM@#}2FGO0$sW#ZF4Z0r!&TjK_W&o;Qi0tP7ofl`{0SSJi(i z46-J`0}>v5N#)@j&*eGZZfra$5%nptFZf-#p4UwU+^sR3p+v)JxWJiF#%JJ*z$Bc9 z0fYKKJN-?mD5Z&wGgZ%*<*glq@6G9O!)fI z2`4p|U>;?w!|d;J^E|&KK97_8FZ@L($YvNAmV-TfXY>>o!he`09%uq=435UDmras& zMEkA+1ns-NjB5Er#u&ezEa|``W%UnGzVBl*i)2&sI8_eo^gF*`c>f;^_bdgIK;>93 z=EZqx(TABK*?)k5hd#KE@YMWq2sU;PQw&(v5|Ln|DCuMaMAsyHF&QLIV9Cj7({NeQ z!(7nll;rBK7`fhWewZzi`nJ6khbXb&OEjWP{QHrv-nWmQt(h)`6K~)fifdnh=xGv0 zN9q-aG;fNIdHlV&P8c|LU%OZ6X&|Kl%j}G`u4yIwhsM~S7e?HR?g%)n`|hHTFO7dr z27^T%(1}OJ=PZMZuA{-WOhN?yq>#4TJE{PvVWQ0YtwvYaT`dswfIcMsBLjVi`nIfY zJ6J!$ZIfu+uQ2=z)%L}Kd@XcK|oNtC)*K7-n--l1D$ zJmw{|7Fqfp-@18C8$fVc+Ye7`=0nZhRIBAs1}PqQqImMJs`9V78dHacZ^oUX8qdty zy3=5GxE6g9iY~;;v1!H4t%EAmvDGC>-+6ht}`g8yvHMCdl|-_O?IX%JdbkRAEP5 zk!)?JrXns%#q4^H4veuy*P4TK^`a723s(Feh#xUTSpfR+AE~`V*z%(eoHGMd^ zi&kc4zWPF2vQh3c+j=QpTG%JxqiZ($>{eRms(tm(49FOg$+3F`nekALii=@L?I32; zAzm~SCiQShSi6LMriCGA{Bh4QCNf|W^TGq|! zB#EN0W%pb8T01>f?^%^~c;$}N%6AYu@^%0m)1lv} z3~d?;>r)_o6=Uw#Gx;OaX5%xETtJV5yICS@bHO0tB64ErDn^1bV`}R-H#CH4u!N~} zsweA@h3b8at-+os5DD!ejicF-_@p}on>@xA%2Q7b6Ur=@Dgc^BL-mk^9mMgfz)ZH2Mdy=I<)Kc9=D#H$c$c{p?4LDtt6blI?^4xt7EJ z=B7zlveQ-jej6*7X=_Ab7R5y7ZM(Hn#j^~Sy4;Q;?cd^l4sRnawnNYSlAt#s>`aCz zq*e27Ews>fz}8aj^Acjxv7_)97SX43)ZdCo3l&tBZTb#W zG4bX2mD_(=i-t8q94oz_M@K)0i|KM!FUrKhHsI;Ts-`#p0O5(f{uPPp?F5)3%jS%;ZEaQ*;@7O}wVLG5L@gC-aM94#DMUi=IDnSTq(NiB#)sjCLrX zF$f+B#+DbsEiJ$5L0~XFVC$U4cVJ&X0&)3Ts@g~4L&K) zYcRc1?`93*Dth4XSB>b9DX~~xT@+b~Z>Ut;QJM`#=)c*jymL-xiZhMw8Ce3LI(J{ZsW63vpg)%ZV-MNgjAH)h&jCS1Bt(pD-2%J1d zpEgBmF^I|l|K!9c!| znyK6I7(k?wK)ee?m19sQ*!giDdmhFrCQtj{qrh*E<%t7s+ZZtlUO80(cm&2vFS-2? zm(`#{*~Dvi<{vn?^`CSZNikl6S|@Ftd5)_I3aEDru2ae!P zal0-Qw#&`eT}Pj&2{r-PN2n7?a{s zBUxuqo#Pn}3%9Yu&XXT+ zTn|lGMh;V}=r9C(04gpblj!E-A(@03IG`ku^jxB9(~C`xjuc^rmatuTMZYDTNp_^K zv`>^8qq-l$ijhdo?#%7`VlULp+&sopi_sA^*0{2!Oz`vNP8ao?Re$fGbz!4~8ktP# zybNI6L60A^GVzc!^4jzEF0wA#Pxx9Xo>ri_f#R3nO18=yDvA`!N#=t0B;VcXtMSI8+fbk+zz{eU2-%GP?9qT@0$iD)f7{dfl92@cvW$z)Qj zqQ~puB-=%5dSbdO^O>`(WKvM{bfs{1yR<-(FdVUi<-8#*YF7L~K6#p$ifM+}A0EEu zq7*9gmeCH5Xfeuf8AYWVufI>fiJ7nH3{w!&}1QePLB=c#{76_~_H5_@fw>`4&chmx}FBFrF3xVz1Q zYf9%^^VcKht=r+IT|?{ghJKp~1d9VUiIrO5V>7*sF7lJ^S8BwC-=S?stD2?$RaxRb zH^=Nfwz|Zp{#lBOx+(ms(xrrhCG`z0v&`r229jSm<-ES4&2Fb~=K^d_EDPWNY}8+IaoaG6{I;??IXLrjT7(zcOkqU48YM}>2F?FkXX2l1Jd9)>YwGl{~GNfywLjMm#7jswrp zhj4}~<`hIyIXTpTp&UFJ1Rf;Y^9Oo8HxHs1T&N>1dHw@Q?d^O-BSuG&X-l6xzEZ}n z7+24WC0HY25|<*%F8VAw_T&UKvkpUeCPap|K)*F_0tltpz^3>xwI6tUNs(nqv%@Ki zgW`sZ$4Puqh)zxFb`RvO(T4>JBi7G~*qCALLCg7<2WE9<#DSy^oa*nk(!X4qzdU*m z+v-%?VyNzR&zwz?yB!{?xKej-gD+Pc2wb3IMTEXJ;c2fuxEvqH1uvP?2E@q3j@zzP zGHw3@C=i$%#-E4fibc4nNMxUt3P*(fXw&|^i&zt!{Jlj0G6Pdn2;>QOcoDF5I|8#D zhH4TP3+1TU(#>q?x~!4fMRH8gF;x!<^l_tr?>l~H$Ch|m2q|9n_GKC^_KX_Lw-kXb z?iB=J1k9+tF+fC_?zYOC`nvu$R^=UgQZap&9V6|RpRKJA@i*=j^Rql~e9XVrtC}*I zoZN+WPP89rciDLemsTay)&bv^Y;xIvez<7wzYvtKf2Pbc`PLqDIletfMp>-W|FY*2 z=!XgW`|X7#`=LAW#A<9wrh|ZFbk4B0#=DTy>AI{mA6wY?4x~E$cOd?)*z2d?By! zJ@H+#pX9IhPCsfT?S?6eY3Hy(f;~E5+jd$I`&o9TX65B|^;R_?2SZwE#rFJDH>hN? zeEBo>$awSKxCy+mDO=6S%@Z9pcM1T2bTz9qvze7zkIC|+QUxNEEMsQLSbfHfGKZ(A z;8R4J_7ii11qoA#E7+`XdCqCZof8=&{Jzv#q+Q^bw|G8bR}k)xE38_W;hV91nEllQ zQSYbp42xs?wT0D0nD~3nMkr6(2+oO3TOw&0&-so?!t;0HwIAl*|MuWXXu!T`{oFLw zGp_2`-iR;9H*I47&|Ld8Zbo6$ap#`@oxLjXb5yEH*ZWpcj`pY68A9x~25?B&+mT$q zVqeHOb(>0|T)qHfKWq|kF2@+L{PiCoS?1O8Cbnl7r_4Wq**6ZU%ilK5Y06Z`iK>hn z@kRYax1qppFTbo-e{^d8r6;%xN8y^8|AWlfpiX2plC}|5Q1PG6#a>f}m>n?7kEDwu z_ccW{a%>mewyiVuwuGtfc&la(a}CG0Z}7kEsuq;G-)P7JxBq+v8U6#nVK!eR#ks=S zy&<`dPu_Ny#cJN=BU{%fsDW2MAL`vAvAiZT2K9fF8UlKAWA8wIEo4xmif~Cg+&MYf zX-xr~xBB6V4vWF0_}?Gxnml(%$;Z2^fKlbsIj%su7>CBwk6{3L}2kZFV@LOti zB-kLwX3m-@C85lWykVe{nm}Uh;nSla?;zT4sV`-hm2={xASTCvn~(d~at~hRah1^_ zM0t=x!kHRD&`)}~RoMyl{vw`{O(*KA?^7R`p5J`r@^$$04-hMq7w54XJ)@q%{-dHo z&U1Ch+rLOF6LF}*Fzdl!ZvNRYNxo2V)FVxj%H0kgNPChD!8cue)z)e;j6kQdP5&0P z(J9Wl?uZ=)%m+y1&ZPI{=9|7!#1KW%qzfQsnbR^-VkovVb87yfe)(Br?dqtGL=9LT zgYd!!rTUWqgE+k&+ck~-a7&J3R|_}TglxBKKA{|9Zj(o?f&Fgk93d^u;z<~?iFHQ)Y78jp|YE_05A z;7*d$^OGP2=X6`DK=F&KxGyn-D6}{CLmO!|cC*uIt=RB$n?b|UIi-^C&)p-C62?+Y zs3@@|=*H4=|5>yq_tirAD0fjeANX`U%&`RR}~0;o(iR+YXG?h>pIu5$64_g>eM zo?`De{nEt``dJvXAfX=_Y-)LRB}P%D#bc1xet5S!ooV}%i~iNY1?+LIx^mYP*KO5O zdUy44l?Fbs2?#ivhU>h?<()qCPCp=VMy1~gX_pUtvpVEmf|Py58+$3S#)5H)9NoKs zkOvEH&xC9|NHFBj{r6%_0ayiJFni5Ri zwA<#F=6F36W|An6q=>krG+kp5C`-(&&&nljXM@$@xA-3${c0kk zGF)^&2gOfCA{e8`6$&!r)m!tZq14YUV#aQ)N8s;fMkCHItB#y^!tia_b6=ONjz(!Z z^%w%@OM_Gu$w~bd#uFnHNlR0-d0D88w24OGg!U^xhmwBK{= zBSEdBLn7!2XqFCF1vT(6=K#S+Gtb7i(S^?`-_-3nAH?sJy(LG~idLAN8@^4n_lAx@ z?II?Ebj=bfMMt<%irLqU1c$0>{0+trVZTx#;9rS`PN{l>UZg`tUL-FGp#zQ^OXCi_ zF~2apNeP5o-pQS&s8j|0HAH3Qlz!-TPO1G_2Zhzp%@Vnl*F`LIh;_8ys8EqA`|hs< zolQdwD!!Io8SQs(rc%J|p6eO8Ech7(2R4w1=`X}DXHjxFSh)srcdg*(n||yOu2Xy~ z`mVn)=fI@+#~`NoOmR!@)WcEWc8tb5K{o^BA%{zf_dNGFwU7ocOV1MSs${9H)ft1;-Q1OR$okRkz`K$XV{^NNYIBD8UKtlKwF@q~ zbq&ydiu~4)0BMiv9(@=?8bRPux8_&&X1i8T%(Mt+kB#KCoJX)}L^v5jC|JMi-gl&a zrIbCj`hBb_01W|5H<6vRa^o5-q}m4 zOME>_O&Woys)^q+dzpn{mn@`dB?&%)zmJCzNWBAT$JyNO@>`s#595QH)-yAejE(8@% zw;>R{kne6jE0YVVeiRJN#olYiuBkii!y#(3_tTT1sN5hZSlIns^EkRv`7mXsy5;I) z`NiRcLmr!(@z3tpaZh%oX$6J`dcfnaUQ`&b(RJc48A2Cz$i#A1>HM^+S95jNeldm$ z5J-&h910lnkUP*iE6c{u2+AmO>&POjrEjcL&TvnC{c@b1`kaEFFS%P+Vr1C8{eG_N z{>gqquM$bdCDN`9DY{zxHHhk{Qx|rTWnD%XzaK5|F>PFBxit?McFmgRNZjdq==@>j+WK`y38oe`Q#lX*RPIB zdM|I3l3zS~hW*}JbWtJ+{@Lq#QgU#f&8CUv1~kr^Ivf1m5aa1t)iFzIx@ybG@GkUF zFl(^uOA}~ub$!)bRX|QqEjH~pEw8t#sMvkUMFlxcq_?>BJxujTV z9dNxo?xP6^ir!7$dV_v8dmE| zt#mgL9FwEr-N6jNq#{tj1YS|MPv_6F4~$lZUiqlWXsOFgepz17WBqr&r%Zn{;Wk^# z_t)3W*n{?Is?1;bm<+!QN0Z&}Le7PYsjWv{iw!fy#^rHDy0VspK$HXPK!cUX?#jo| zCFZaK-hk%*D6A)9YJ>ZbChfCoQ2OwJT2;i*UGDx){Jzh*a$VIw0FHkg7-cB_f68-x z*qOoUU-bu7#Djr2$}`qHEHha5WO$nYl;Onk{(-Gx;F3V)TKK=$;3HyvZDo$Gwa=<@ zJwJEI3z_pVcUFp{JYujwdJGS83?~S<7wvwLWRQ&P^3lZsyKzbY@;cva_3-q{lk5OY z&Mjo=z{t;E9Nfyg@%ILz-+WdcdwfOa0i1`6>4}jpzwStP4F@+~{JgBZ6Fj-DU-}2I zSFNB=b@<^q@)G-rCWv;r%4L4op?br7P{yCMZ!k)%2T!)z#&L}e7FH`!lAD=dKoMts zVIAxWE=EDM7dnPx-iONTmS(@Y({Z8x$l8|{KT;7;E3=QBOEKpk>O>Dshmll-;3JED z3Aqz-y?1FSb@sF%UBU6+CU-m`*7lWC_b0biYtp}Q*+nm1`kppWS}zKd2g5CSIcd7N zfTO28mcR32%aj;)Bs=Nf?@M6T?7!V9yx_~?7-!0{rr(7g^^?s}zi5rz7SGEj5!c*8 zZSBienZee@q32$z&e8G-@7-YW!9M^k&y!FwCcIj7c+T@5 zfHE8P+P=6pr%tPQ4OR^Q8fT^{3U;04!Apvo3Y9hJdZ-BglTf)QY@J!Rrp7W^-qG&S zHIKkl3E!kKS6d*3m*Wj({hGKRJgLC5w$z9)Pw_In z^mAxPy$4SxhSxl&W$^yZ&Y5i(KdiSN=oEUB6}FI8M*e2PKDH~F@6g;LLD6zGE-DGB zEiVG;(7oBNZYnOdU;T}7cf^J)z(tHX^UO}=(}&nPfnep>)wfeORL1ISJJt=gvFbQJOx`=Y99EAI4iK z>DN$iRoJFm5fZi21;;6_d*5abrN}N{LKT(K{67I-i6+8u#q1WXCa4o5woZwO!CU`N zm;dird7XvD38V7Ja4=CNDk3c`(H3n|@K-@ulF2oy0*MzW4A9Wu6J#CK9KN4)TN&y| z@vz~HVkvA{G5C5W6KBuz#Li{R*t~-Jsx4@P$(rT98Q9!hQ!EjpbdKa|PWjK&Hn>*T zt5ogG+S1tE{;(87t)F4FEOJ}t6i8!~X5^G6Z(5L~{%SUTb?))iZcCf?>C)uiEn&~c ztcD>@FL5WBJ7nu%S*W_{I%_a+{{6a>p7l>;K4E4jdzHxWNLH%m(NheW zt+vKbt>AY-xP&soeDRk3Nr0^;5phpES7HT-t2j?K#pk~wFLKDcry?=A<-`47%a1O@ z{(dr@_8pf|ZAyDmdFpjzNc{!pbn<$3Dv{&m2Xx&yx!V2i^O^^GjO4*MLq6{3hnY<# zkMgyEW2886z-lEox%HME4@rIy_8Pu)wNY2!4WD~PEe>ktfST9KCs#F$(L2`sYr0xf zX`_}jtby=&}0fM`D90_m3#l#*!trPX~x!S!R< z5FJbeN8o*k-ZcD9ms?7Bm(Cbuhgo0erck+&CTqj`bg;#Es+Iksfcb0 z%UrvRIJ#}-&ll!gEFDfEvl=J2m{0-ibgsNK!e^%XR+pV&Z*5e$1Mi+woFw71K$hev)VqFO>f`NwyKhR8^XK!51^K# zpvWT#HqJ!8Zm&gX&;AVleIuk>n{+tr+QqA_tNU2O&g*K31rCN6mv9}u4BV|0+TB#U zq;~l{(IX6=RI_b9QQf?okH?{x*#GR#wvJ+1)oKMIUp&JZH|D2GENi?RXQr0_aeaOG zce&R^KhUJ(hd&hozr^fFsdJ4p`4;hRs z`+nKNS@U2kPTgNM!XGMW9ER3q52G$>^!{k$ACFjauS9U*yh);zyhOq*yFMIH*s(aI z$qb`1UzaLf^D3@_U);C4xfi#o>M3P0+`9M23^_7Fsrf7f)5}>{D%~I8^+$Dma^eEu zHQ|Na`O%sU#+SmYK6`bgEWoJgsNro2+lss&%T6qiyVCe9E^4iF?eY%88VK);Ec zj#Rc$Mr|DY5_G0bjYVE|ES>TE<1jVbJ~$^f9X>HB1v7Y2m9{%U#tkWB>POhs-CX8#FZm6MChCTY*z&1?fs<^(N-{%rf=;o zGV5$=ZEq={YfaU)&-iY^M}_JbM`Sv+#Qjm5fo}8bKb_9RUoA9ek4qzQwW2q?(QgY-)EoS_ux^ z?x9+P<6Y1)6O{&YXZ{=jy_IX{nv(c)8hAWeB%ikKxt-J@KQBm&1o?XTVdVjV%NM}D2BUn=s z9=I=OE;&`-5WLpwe}{{KmhCk(E);>vv-uRSRq&n^QAK(vKFx=*t)CI~AUNJJW%Id^ z7p=FHiXxDOj$n3htvz-8nZmY?7#2gyJ16qH>1R0P2ij{NIg&K-JoVKT9h_?!5$g?k zm+~9;T=oex@YY{z;^*eQl70;xTqIMCpUK%pl={+VI&EkONfRi1f!W!IHaI-)z;>S5 zbgz|gasQHFn$AjTZITB70W#hEy{Dg;2^b*-Reu>sMl&0~q7U#dk)iLEXpm)s{rt0? zB98o8#bjx~r)0=@QCU~;Re&I-4W``h=O0Q1uTy884tTmOftFqy$+x#o=HO;Ust52;<|p@7$|l(qkz<#?aL}mSrv^}r#kZ`8g;w*z7oz)ehMgyLnRd|X~}ydKS|tp4_ZmbpLmb zAe#Z+1tF&rDt@G@xZ{-7l#KTZxfFZE!n6N0pM0WQ9abOV-p+A%DW0HhmIYJr*D_f z_?H(Jc8*ipxI-wR3%Yf=U^Eq7`atZE5%7g_^H-_+nUXW%GIz&_kNkafl*oXWwNy!b zN$|*oc5l_SA~PTFwkrI7Jn*SPU+)o<|DFv3OUpZSVLcSph%;Kv=yZk%n4&Is##R*T z0T^jKg*1xjTn z+}3o+;TNOEXPqt=0WKK+WwImv1ks(~^0hK&ZFKpYHBHT966`9!yUI}SDJB_#g0-C4 z?#mMvtUZN`wpvaY#JxW2#SDEZbFId7*uRLL zue)7&%+z^Yn^S;iU=UK^?>xY<2K#X7mgGiAu=IV;VfE-QX>9V5YFU$;(8uFz2hbHU z;t>Z%-k290LSi-!J5YR@+9l+6Oiyf)@zwrXxtvwT*p#eQ*V63jNv~%d4`^ScDl}D~ ziOg`CXnj`;ySwSO`4KwhEoZ&nd8HnF%|~7@W2VSLEkmJ1i)wrN%1KwiQ|2)uB3MN( ze(2wIHF_sWN_McK+Tqmb25b`GL(u`jy|0owa)a9XY5{ zl&D>T&NxwfQr;Zu*0ybrNR<9V`9kLOD<#U+B^RzO)F2Ek(;DU)OgkUn`6edvmTkEh z9NytX1J6{4LYYGD1O4j~wH$;*2k)d{J8Lxqlv3g$vaO#+3}T<1at@POT<*j^vTFvt zO}Cx}DjTbEQ};1GBhU(vNfg&KHz>quIqO72*EvA(>tv;!qx zuu%=3X%3sh<}1t)fmlTTI6>`>729Sh8yXE`_qnrzI1{zB&eVMS=w+XVX^3^GXxtcm z9HO47!ZPaXomzyEw z8VV63c1dDo+aBAX1NjHoIHUJ0w)f+2ADY?H0tCjp88NI?y%Z$@S%jNshlKt&8n+y9 zvc3jaM6^`UOdyF%Ps+FL@F1GY{mm@De%_9PE<@(LUW;v?1GD*?X5lBY?B5Fl3QcDU zf!?GO+fpEuxm-+iaDSw6hon{KXrCMQAoVUCAt=_8@9xXd3L?V7su-5Zw9VYRH8_@R zmK^6y6Q#C{lYtozW2GyyiVjXrp5f|%5+R){_iMvuud&IqX)edYyXO;6R$p&5HO9Jg zy%mSet)^DLQEd|awQiD}ci%@oK*};qSzkGC@GF>HcV%24beAQ4$)IS*$Zkp~hOQ?4 zL*J}<>r=*o@3E078EW_O2H6c2W=m*MfkAGdhgg;~ReE~oFcf9r`}PnvG%kKxf4^}{ zq0lEjdhuAqU)cMn^a*2L_m>=Fl(CQMTeICf&ID58U13`UI%N#_uf7mI`(dB!=#C)g z4rxgakeqDum<0>N7eYR)o#r0sbGkd6sH#-XWYp2--)&?-*F{K1vePpto<1(PjYj}l zoaX)w8g#X*lGjh{e@=c?Qa0U!S%T$^agnZcTC_{D)+F!VsJY84I&mmRKdBJ!Xqrs^ zRR%zGC764TaW$JQrX_xt8+!aX(wk4m^WixgecgrKkz$S+&?5jg`?uSI3UzZlY!=#JmI^w6F#2ksqezbLsdC+In z(iS)ft}$oZ<$l}!I=#M8Y*^|(prj-|vl(L!MaFP;q_w=&or~MV?LaL!O>DZ$_(Znr zqk#fDVj%F@%N35;@+g(`vFN@Mbw4d^F=`UMjJ;9cH(}Yg@rEwm_uoBFP-cg=fsUqS zs{)eAH~ERI4yuNK_)DFfrOVH&<0#7rLKG#!a!*vO0(C^by~8D4zN60Oy^rL zI-XoTSPoWRN)o#4^IRn=4Q^;B&7&AsS3}XyJHr@JLwClKp}OsW_?cPLJdJ5e#yls5 zziNB8M7m~t8OX8P+WoO3dHxKs%&SM*+n1wNhax&1#^TYdQzC|Ge(%DbxuiI}7?!9Q zoIRQZq2lbk#Dq%i$f}s^-uEh~RAK%Jg9zaaQzx6XLzWdopi3(nW5mvBB5b{Y?^rYs zbE)7`S=4IWnu0bWrhS<^JX^@b#IVRsD2#lhL9br3n*;R(cvw8vdq*@(Wwf3roE=iR zi~=ZaWj{-f5;+#?^??vK;giyt;XTkvmr7r;yw67W-AV}DByZ<|GFWHZMnlLa+BOw2 zl6da`%p}#Gl5nt*tItQVwF9lSXfg!JGy$7DA96n2)_}qPG~t-n)5Tf$U-Cp9VA9wv4Q|Ae5iH~|FDSc6eVL4%xBR1O>dyYZ2*x=R5S+)Hh+c`pOmy}7&;M#eK%X|j4A#kXvi9(P5 zLa1d$3d5-W6-v-xn<{uj8C#>6c{MhpK$LA~_S_f{E-NuSDP2&-Lz%(N>-^JV)nt6v zd=kiSM_J-sPLo3-?)!S$|28b8fITj_!T+N97C&G|iH)6~cMp$O$<@U5HSX82A3~*d z2;i{q*@T-+kIjg)5`e-zGC@}>2?*VPK+U|Z!JXc0v{t~DzA{4w~Y%G8W3FJ^{$lGc>e z&sow3*k&5$fhdtig8`Ff6)P$e1g`4F65Dm@9a*5zD9@)+Q_s;&qgl&?svI5n>BN3& zFGWga@O1Y7SbEEVsM@ys8w5dG6p+rLOG>&#hLY}36T{HM1`K95U? z@BLE4Hvcp&ZY|!WdQa9lyzTs0!2%DTHOrZ^s@OYYWJBxsXBKazccOMOc>(ndayJVr zQ`?xm=|kNKR#kkMzAKl_&xlr6X?&-R*E^LipA|KIMq%k2dNOqjy)zC;x`hW#zv`=# zvo%&9HOXl`wf9H+nT|7Ff=9KfC zf6QCnL@6_E7WYsgLQ>JLY!3lro4o|hcJydXewK2 z4X38tF+!xN&MjYR%e+85H)l^bCS*U`B<9oqDz>Y&f}B-H?**-|Fl?w1#@$4z>3Lh6!^7V9>s&$%O6|`(0E<1D zaF0UR$0Rd~sKl*X!dLiY>+*EpVtU8*C-A3zAk61*e)I#Rk3sX z&w5)~?>iFFb8z=F%q86E#=<{9yNy>Z~6vrAF8(|r3gh|ER2JemQHRVMIsrZMq^ZC;k;FeZI z=AvOocRUwWfG!<)YRZ6HqHp9R!Lj++pJ-X%rZaDJHbFoEEBsLzT8u`}UdZveI|X!> zPY0xwNBmo1Gg9gXD7(4VwnFPy9*3$OUx&ZAAT-FoZ_)m8YIe=J$4q@qu!$131BJ|d zXIw%?o!%q0tLtZHOi}iy{=91VD{3jdtlaZ^Hgf`{Y|@0|t;qcwf`(U8P`i)&r zoA#0F#5HX<@IjF*yt4bKokX9QDdZLAV*vXB3l?AxE_X*tXW2o^fQmu=o$buR`J(k2 z$@VdJ{}rI;;!FRz=s_fb?2R53F#!X<842sXVI;IBbY$sF<-8j*#Cq;)^zi{hoHV-%X`=7t$S${Pn6918A zfsB*q&k@RuODq2b(JV+(_C_+>)kT99q(<+`@_y%ED?$RWlSn;>17Czmw<7q=oiXB6 zCF|5;G!g85v=sYkaUq0cSOjuW`-;cu`CfR|~pBli|XPgh*l zDK6lKi5;7k$s`t5F2&tfXO&T zpU3nMhw43zw7;yl*A(RIBZVJdW!9{Jdld08S{jpJM%w8AOg-~?cAkf^!$O^`qQrVm z*@F49#Seiu^%OcXowiT_ z@dHe#TCcPaBzm-ZCk0AV?9l?5t0t9YeXG8=shElHiOe`V$cis9u-s*j3EO#|K2Q1v^-lSv?>>>aIWBN)uUF z()3`86H+ynxEOLV(X86T(3bb+_y%(&g5FH1E=l0yJUlj$K0)Sp&yXa5RU}0B^z`OD zE7zpp6muJ{yY-zSW+X%wguhT8&ja;&{ujGy)RQZeD=c*(2$%s|d><`zPR#jaOEyj* zOMO`;ms2ZfUi`u5S~x6DgInDD05~I-V6vdoH$s9NiP5l#)oN( zjf!G#ARA9xj0z7EH%W+??vS`+Pl#Z3! z@|OaJmA3di?67@D9@S`9xujM&_ICQqujV9O)OZDl`(#q2nqXA$@wro&M`j=uqV?300Hmg9+Dgeo?cYC5c4O$Swan+AJK_BK zG}Xvp$tPeZ30WMB<@20PuEo{RM}l9#?fTA#>o#Km&$a-?1qCu)2< zH)UTV*5j@K%|*kE1EUri?Ns56QHw&+44E_nO=0ZLzlFFB1SUaIkD^vN8d?+Sy~D4J zQTmLjodA@I$5qnsgEenRjt2P?Iim|X;wpfMVmxyLq>%Z>vT3@!rh7$()Rw+H7c?S+a;qC_z8sFn))-& z9Oe8U=xsEGzz1hq%bh^IA{{aR^rNT(bb^m>JT_}2y`oaQyUf}KCOLl+nJmvjv4+a^ z&Wk~T1dt*Pe!7Qv<5n(9UX+8% zOW%z1s3VQ}#v($c$>dJg6(7)b5jiz=LNhQ@a8TLQ#b6<4gfT^8m`OKpFTN-5DJ_T(eoty%4f64O>D-pU0~y@wU^#buFE zQ_3=>U^)+-U?y(K(K=G!1k|_gGt*G1u2vuGz`N&6PNpLnlkb7-tcD)i?CdEnLLvl0$4{_Dw(Qjw?fLC`E@dNP9 zbg> z6tv`Qy?99Op5mKfX583atNd~JyL1*Uz^2w(vT5R%cUaMF$-5B_fmf)O`Yrpl_tD+>&S3 zB*(v#KJe%N6+ZhHhlVF+cl^~_Z22XAO=HQU?8x!edN}8E zsT7k%H4yY=sIl^^j9Sg#w~wI%|1vTi>`g4z-hX%|bQHm6C~D%~U~iut>20E4m79JX zUg+yc9UN*pZmeBVp!wiv(X1h<%hv2pZ+~7-@)qwKl1|p$R2!|qc|K&&lgQfcm()0d zskbCqAcgIueie;K(Fe@gM$M2mvDI_O45nT(>)D)ykdqtzu0D#p!bH(t>azYP1PuEc zw8hf;%^5qCEmVb-N)?K};ev0|nt!tp7P*i37V0jPnu_YIapb?h?+G1ZDoVP+SS|dA zMd*2RCS>^a?@(`NWi4212460{SdE5y$W3fJGjiiiQquLlFIxmFkDBPu3v+YVL|HfY zO`+ag4+K7YdY+Bx7oWD`dG)0eNbus_SWtnvtdSz%gR~zwgEDDN)0Pxe-&WR)JpU)qYM~bB z@5+-PRl}h%vYUpIRkjf~g2Bx^9V&rbd(BrhcBk@(*pm7zQyhWwl|$)q?FH-aW8muOV? z|DkW-REvBb?(PrM^p{%;FSou8s9070Ql>zc>RpI=I>N0oI}fDP%Xq`Dj4mOjA!di? z!nthGK}8p%)D#b!QNui95@HzWKEZHs>8`q$c@!9l=ZuAufKJITZa-8Yg8NU0yiFXz z**XTVx0e^&93AOP3PR+WRDbnen$Age27UAg6kBH;1)S|d{q$QE+K7WmoxK-E^R^gX=W-~obHckH zzq7Y~bz(0kg3_wWO3zy0oN9&%X#c%KcmAs*Cj_ojH?u|Ketf9l#8?Hd`{*L~^^ zBTFJdHOf}b2KAlCM7{tqf;ClfMhR~WKrE_EYcf<{Ix{JB+1Dsrj z|4-p6Z)}g|vVTUFodD&{(2vQ>jraHut3+Z0oL8ec&fR%=s{BbT#n;9gQm-B1m&B?; z^X!Pn;H7-1#MCjede2Dc=brde@K9;u=)1F3{tc!gGt zsMm{*HCJ(pm}yhguS*xz6!(Cc(u~k7CzV46&;Gfb+og9z^^!Ej-17=1IKt1z+-Z@Q znA6QgzJ9{qOzRA}n4hqk1(oeyQOhuv2(jx@rGZN$@wowOSi5ptCoAfKLs)HelclPrl<&tPa7JiyN-65Tz za4SozAC>eabmiW&)GzNyEayv1E+&^k93E*c0JQ^ZqxpG@rN*$z`l48tAjPo5)GN+;v4@vsC9fu^~NacmwHdcf1z&k(R z!y8F_Ogjb2*6WHL4+@fI{}{Q}E`tqkmA_vRmu{Vb_*8a`4&BnCSa{TGGMm*ZmS3j6 zdd5cS58OY|BpTk;@FR1Nl{-U$Su=*Gx?4s+dqeFggq{%=mOGkGh-ILC`(l11B1i_A z?;-V7qkmo?-eWKy*;9WlX$DU`X7Sb4JBjHj-Aq^&1UtLHVx~`MPX{#2@!~YKOtjqD z`1(dw_!K3t8t6CD;&=VWj08UiHgOG4T7fxth3e@3e19=da6e}1eJ)z93^wCO-fNk? zOyG2`N8YPngF=ngcju)V5-Y#RbFaKg6;705^HIbhR+JpuHBn;N8%mt`W!)e6RPV29 z6PHc0swB;2y~_|hPuzMFVkVN>#aL??0q)h)P|npUSQ#SpOo?W(ixk;&!PKzCPiP~c zr$kwwEiL*?svb8cRkQdR^X{@4Hd9rF(8MMf{>Un9Wjc`z6A z3T`V>GW9@aAMSVpiS7JV4^2pQ6GE3@bRQPVqB?>Da|-8H17T^>M%L6AS1<2sy72-P z9QSi;{>KXM$S}zstkFJ-!`$l0-Odt58!*K)aZtPs`h^4=CKno9iC8Xm#sSgWU6iby zlUsb%N*9#8ZdC2bo|FyPXm!RSN=Zlvtays2Ah}a7oxzw>ad=gPe_?H>NpxmMv96vc zGeX)ilK;wc(H%{OPDk z>?%gDl9Z1Mq%PVPCsD)PHDv{>W&$j2lKn*+64mj;tTd7~E7>)5&OAet0(&b+Namq~ zGe>NRCof%`Ds#dK+F(2Ev8s(jm7KKLg5YTa#vB-6MBR6sH!)f;&s^)S0Rz_U<<*t> zO;-jLRo&QGaVArbJYWoEae)@U2lmN%woqnrJClv6M>VkwYO+|zhM(BJnipO!_Aw{i?jawx+Qp3NzOCo0cq3sVNj6%bFy{w8QtUFS z7XcDu{rl!3dp8afIXg&NHFLVLPUdG=G+10C`z9kS-U;zXaTILJOEG#TUV>cc1N>Z0L-`&oU9UeXs$?g(ljMy#TXuJDVCDtvx=O_g$ zX=y}6gS}Z(g+Gx^e3iM|%M=)qN3R@xq9|C_ny zdLg;xingfkq6;+s+jsg#2uN}(4r}ok(|)Ms*MUB2&{ZE1oo2<`!?snByZE{xt9}zf zxo^~LwWQ_&`>@CgAy6LtMr{>CInDUJ)z=L9;p-%u%gg-h&3l>5g6PM`1zi0Ws4HTd zkiZJEb(;FAGLE34&9w~*E=D+g8eo8U^wLGIme{-A+&$q@Aq7zWl0P62SW6k>UXxOOCG zw24vplv$wmT)md5K(M-IwW+)o=9O5t!HR{ps~eE5cKVF?p4`{?l?JCIrjPS$y#pol zEumsm3QQWuSPNFkXp*I#4(SK8@;>!du^edwfYdt1Bx5zA^ku6rw8&YlM=#E zG=>>LPl$cMO@o6khdz1ACLJjkS8CM_*~RY6UIYUSnfNDZ`8tc ztJhSPSQLO;?OYmIWIRR+ruM%5 zBPWs-gAIi(3XaXFOmf!iNRwfooK2sy^E+30h&bJ_eA1odk6MHD;8L=Yy9OKojf+Yr z-d!`Z%v^U0Cb#Au=sH#Q@ay3TL_$&*zRQFpJ`DL>KXJBRDh1X7bOQWHPJXiBI|NqJ zZyWZ(21=b`FjcdxQUMzYgoeZ69_RUgh)6;1)cY%SyP1$G>mj#OxRs6kOUv(G+HVsc z{0;I*+L+81HNlq?gH$C`dnPy*Q>npj#`1jjemAh;y0VPoPSoF*U%7{T1_LWAk}Pu` zHALqD4sA~4iYPGGEmXG6TA8Z~`hAh!t+!81h)oM0s8eAFWA))R6+t;aPxR1-&F&lD z$wjOep*F^dKzoq^*U;5{T`?|Bi3wm#L2}efrI#KWTz1Z*#)qvXX=uubQWoN-RB_jI z&T^9HoX<*Y$&;T9LGnW_v+Z57;^ltu&)s4m$8SUiuBT_Zq zyv+hFAyf6b-*uV!ZpSxlH0avLlM=g_DPV|_Y#Dhi0(<=`k2cu7K6X3J%WiakrGqq+ z^Ksa2A2teM=0U1T5wcT{y)C7s%|=nkwVvSMqgMJj{T-B>xVh4r=tZG_m;Q(-m=mhk zN1hzRSw+pm`#Tu&@~W4Eg9$qb$M4T1Ldhq0kFmi<${rw1J+k8Jqb~BtTzt_{yaD8ZZ%p z#cWSqorb#p&07L?3;?MJD73>gBtuVs;mT2Hy1R<~=w$i4<>utV&3Rs1Tja{ODv%VG zaPjImX_}HF7WCOWcuN{QJmp3yAp=NUnN>w+Flxsq{^cr72c}V$ z^2u1py7&1y(|cT$dMIo|Rfi#x!(3##eNUIJim=-vL;T8ccc4MrDT@4&A*g5c zWtr37O60)hNhq8)vd+DTNFblYQ34O`NCG|tj|sBzujYRi@;7<;d-8qx+`>(taqvXY z+qSs9m-lu#)vW#{_-i9})vf7Z;xW4luDI1pUYhwzqUHU?>e#oMT-JvE$h{r0l`=gn zS?qd4w7XrJi!&@}`l@T^!N<1F%4Bh`5vcu^O?bxNEYn&=qowcdMofMM4ng|Y-T#62 zHVpVLg-gJJ_4}a3&e)PO%|+RWyqVCXhlhNgJ+vhEDydto?GT4KDI}?Q4%2rcy^gPpHt`YqO2f2C*WNa&v*DsUd_f2NP6nG}6riBG{y(R$RqyOpgRg_1!% zIe@6qbIa(=xVGDlXmz43W6L=7Vg%2yuy)ayqsHI2o%tR|zXjkWspHo9Mn4&YWoTqw z1-1Gc=Vk)oMohY*5h$^|fDPIG$CEsXm#E`rNo0Dhr1fv$>4-&-RI$ST29?+70ntey zhb=RZjlQNa0D4UtVEo2f-JLL`bdGIMGy%gvm2^|Mp2(=#m?J+L1I@lXghaS{`!gW& zKIv{6^R`>izb`7tIW#r0+Ac5;xu&)i*u7YG58u0W{Z@5cwWmMLrE~QRorv7( zO5zv}t2I?CTQXZ&8uNI2G9D-K{>PdG8x8dyPiDC@pP&c7_;nnOne4ye19~UM3)i;@ zI83_z_{~ATEs}q=-k0Ud~SZztl}*6=5;wtMpNM}2{j8ADaH{gANI|{rwETv7bCiMzLB-^ zS`Azd9Uup8h(79jcQeu$3MEtQ@T^1^TqQ;nTjkia8pS#1yfvB~R*>$$tr0={pVuik zBU-eCkt-2@mxu1`0n}xo%GT+T=k)JHBi{B64RLbbZ>H=H3JNmxsgE^?l@ywxd#>g* zp2$tn&?A|Fcs#{Lx*OPLUgohx z?E>AWmU}6sJ8@kg0vSe_@Arq7YFeZ8ROY}9Zy$9AV3bpVZ9q$LFInrakyZ|SyNd8}?b zNgCCXIgQT&&!z3KLAzv9FYgpJxE=F(X+V%@4W3sJCC0zLg;s59&nJ2*1Z85=xKQxH zeHgHO3*cAbo@in{^$OALli68FW_)r{Bu-NH9O|vnZ&fK;&7%Ac&v#$7*KdQug_cS_ zz@~2u&}K~j234}T@6gm@I-C~jCN~kSt&PEPLOU+pjYFAaKqYZzFt4?u-MbFnsgR>xyg)KL+VWH@29)O=NmBpD~8$G58Cl}kqzJfYWF`i46jld8C86xZ6!p~{gjM4`>*M5n9vU?}BmHkeS66{2{K z@V)0bcwyYyL*U#FJp2Qx!L>J(4eOUm^~}Vmh|8LeEsSvj zSJEc`c5Yb3r^&6lZ^1dgwgtrPXddj&YJS}ITuKGhhjBd2jCfY8{3QA`IkmQJq}kb8 zy*qytKh^zltSj@W{9Grewo<%TUhWob{%^z=fef|f0awE-`Jo4PZ>VoUdj75!7xlFI z&tK2*?JejF)n{hm!|1QS!O?4r%{D9SyFhzf)IzPv;{8`01VY&D*?42-a6q)e6tCXp zC2uH~-=x&`#rtY|2(7~NfVCoGUiy2~Jmwp4{HD_N4*uBrDrx;ofBE8Y+pwo#;ewyJ z8Q_8=z>3?WG*A=7J6V@dOAfWCl=_nbX||wz4WQU}&W~RZXf#@1-_uDgx%;wjVpvbh zo5^WO%dL_TVW};ctLdEOOWaoX$DvTW-E__249lTkVUUAea%w$PbF1Jebn~NX4?EV$kpM zD{BrPDfL9vHs^vwUUSDE+p@9TxP(}bd#s!#aB$zxA)&ePgjFnA+ZPp0?>ru8vH6y{ zO%AU^BAv0Op7R!lyp3)fk%Mm?$G;{_{_bCd4Sj-zjD75*%)H-rUTGI;40Qg^DBm*? zU(N0u|C+cV0isj7Rz_Kh9pp9n=F$CauaSfccv$S@MryGB*+d6|vzALgg8 z3pcAUGz(~5GXH+zxy&PV*#J>PXz+4;@a-3M}Zh?7ZxS6j2-Bg}JWp~*7vE3h-Xv#l{*+f`(c3l~o*CI-_GF`!h}OHyl1f6nCW79lnKvRZo?T08yl zNV1d=eh#fIUs*9@UEU0UiG64-p=%BfhA^J75Pr@ymY|;UY1=p9&Z9TTdOji3V}B~s zpu%^cb~hbQ;qZtx8HfH1nqEfoAqJUZAoA|yP<^5$ehm}% z2Z_>`vZ||m2+QcZ(O^2~XtvFo0Iktqipl2RWAb^`+?iDzelDh%kSBXbKRMvpJ>h8T zOFE=MJ+!}?zQ?Eh$geu~i=(jc4yRzw?$vsl_SEvTgIj`ksd6c`X>_=!&lDDeMl=>ZGvN<1l9#teAyocVr)gM?#92qZ(h!P@0DoDXO*v8#_?sdmk^ zzxA}Yq;hRR6|vpo6qYimHze+Oy5WLOl?{bCl^961>F$2zww?I6cN7fU);A$Ob9=>H zBQAQ6nrK^foBlKVzgy40@ zeF6&PGCA!BV_n&(0~JB`Jv&7H-Tj*X;6KbzEXQ5%+#)>;}(B>b{gaaA;xBU zCA3PJ`cev$oadqfZini$z5xf`a(!L7@PJ_=i)N~5+K^SO0d)2p& zrmoa=!B-~bDsk*->7fGF1^kG}e`yvO<&PArreguBMWShl zQFX9QAffrHHU)=+lj@d((WVnKJtMEi(1PV%fhgv~tH9?tVIny6ZxtufeumD4y_N-% zf(|XO#F1Sj@XQW_>EBxbhG=Z+S$u4JLszdpjaJ@z^&)KFUEW%sS62zII`HX+{Zwxn zZJn^>h*@)(|K*|j-eEH50z2eYOmgREiF=n;?-5NZ+^hdBesu<^d3_7jF3%C|5l{eoInrFK>rgQ zeCm1(^3h6ZY6w0y33-5JzDVt(e>KdTacrtrt0jE-@{hm%K#f6a{7h*j1Zh!*0B^4> zGutnn3DM_mP~+cLl+JMgWseUgOn!piY~VgSpG*n)3{8iu*$aE7fkw5Y(nDaBtmOkb zF`_-n%5EV4IDAl=bLqK@+Wjy2RLkEUQIBk>6CPs0JV*h8*@&X!lp@Kw@PAm|gG_yx ziuUr=U)C1WVx8>O?G9?BFM~m$r;lx`sd*&C0tR)UNX+^|s6<0a&g}T;L7M|HHl6Cj zNgWRua;9P}xSMULucsf^I@@q_Q=dpgH}v6X840gAc^+&|*<*E)6Ig8}F70Yk=B15b~GKlNaPi9`2j^>g8%}DZS8AGd)s?c#q11 zxj2eBGuBWrWtQ3Wr(U5RNiLmdkozL%{Mt4GiDE-Qu8j$iTdPF(K{ zXR8;dUg0^`9{O3!m|LUqX0aU#?TUQS1AY2xo+Pi^kj=S7(ReKx=DJh zg6qU~TEWu4xgIBje9Z9$Uw8AdJg>Jpj?nbJ+P65y0{G#UISz^&3s!Ive%{MV(Uq8TL9O62kKU=spW=3TzFIzB z)CJlhj@nu|<^6wDkdbYSIn~c)7y}K0uKPqgN8N~L`U2mXV{(SImM+$#yHdIIr?`Qi zn2GOfD!8Hn%MA)K?ASzwG>(Kk6|EC*;n^t?Emvh|ua|u!v1pR$kO;cACSZ-z3!t}W zB$FtY^iiH@%vU>AoNf|%sn}o7PvIH*^Tx9|>zocSZZuNWhA+aX>(X%k@!2pn0E`qe zgE@a$6Uf<8r;0c3Np{NXxF6b3EQS+{i%R`pV{o|fDq?i!`*=+$w?r+F9CiKqy`?eq zQ}!{v-&hSwBx*}Ep+AVIli3I^McP)Iba4CWY<|+R@P>*T6C$!`CAyIMi&{_8d?Ix8 z!STsrknvKudlzB&3aoQmdpnUczIehPw6N>%Q}I6=YS;VA|I5B)vn9J6-tVu6l!^TNG|UZ0xMEyUnNl?^~Mte1G7MyrA_2=P-<(EBnn zDzc%vdDYjx$7UjSF5}y`hA*_k>a9RtL))VVhumx-4KGasZB|MjCL23iyF+;=I&(ig zRckqX{qgaf){_AFp@Iy7+Nx!aLMJEsXlc`WHhx;x#P9U<$-=>A(`2$SpWOxQV{=}b zaq@WYrdTv4ENp*4WEMg{Qc4)4`hMA(S1yMimtUZ3PR2aB)$HuO=PceO4dmp(=H?l; z+FSE~c4{~g@Qr|yqeA|}>T~5E7CPcB$;&G#n}X^4M5l>eR2X6k^u&3~e8?Y8xgT1x zZ;3y?>pL{3g5?=sq842A%wG38nyly7D>`uL2>z%sKdPp|$hlbk-> zfUi^Q!Zz@gG&h6vyo?i%w~ni!17Y5smPTUEj`(Uns5*$*AeE?Rt5%L2~)j48=JJSG#jdWr;#t@ z!4WObC9O3MFwMN$E-tEu>s{owKUYTmITT^SybaGF-|k)#yN&={ug00k7!-0f$8llQ z1QO`!9q3jc>fB?X{(UqeD1lTc>PkCgz>h7}*T|pf+Mnn{#zwpl?UPws%NihPaDhqB z(t%t`g-^a32g4R8Asj}w=AqROd=X^j67Bt#dz=4ZY5&91^{wdM3Rb&@R9>?YfP*SU zyyO4y3pRKPywM#jH;YVS3H+8AX{a~ykNO<1VZUQ}xz>b;X!KIev0)k)8>eSmug@Jf zY(Wctji&Gq_q_WYR1@z4i|;&(*z{3U#f{4IFB^5(Ju|v5sI7J(Uo~G|S(>CXQ+?mg zZriTgDIpWk5aBtI@((kUp6jAMNr$TD7N_AAJG+0ntydI^fo6b6oFZIh(E%$yoSCP* zluPvgw~>pay5k;~Sb&Vnj1NuGpR@pXFOI(P0j(a{ifg>~y-+KF5SVF2j^4OeZeN-$+v&o08csu;TI^#%erw*~0ks7l*|dF*hvQR`%V!Z3i?YWfDMUC7(m%Bx%E)CPtF|8bt!)IfnW9=6 zK~L;oJUYtYt%@f(GNDTSd>X&S1u|<&B6oCjO4C5B2jI&lwh~?)qud4^xld&nf6%2?q(B1Pd(?tokLxuY^#L+cza2l34lB|mHEE9hLtS+ z$fUYN_Ip2z`_RsAYCjbybX03Kpdur0()GiPck)w|$=l;RyK z#7~s|xuuXAFIx?_GN=PD+-*I7F64n?1y7NJbd!`Dy8gE|B?ji9Mm;GL`C|X=(4!^m z>Ro`VjPx<*NI$%Drp`-K&a`C+?s@`gvRIQ|IAU+KuHLUl_JN7)!DD6#L#{poY#@DUoGN>E7rE z!vq74wZ1rfT*i_@SFah1@BQ&Lqjd^0UM^v_g1n|^wm*8?$|JLh29l)$YSOWC&J(}J zmOVPtqozTHF0(V#5d$g}F@3J-dfLUzOgpA|F;RM;IGR+G<&3`z%WG+fK>GO9B(xl= zDJI!^=bate#wwTkY9+Wf**q(Y+D9f;A+$Bo+roC@AdmuZpE0=8jEmm_|xIQ|v zDl1Pr+rtjGVLf#lpAj<79^*aRv=l9dtul0lv(FVWCAOTfHrp(iQs)>plsLS>W5)1fq{CDe1 zpn1*=P$?=zem6}~S{-|XvZWZ}7@F%^`<2B_=07X}BiL^?Tw*HvZ=6`r47o$8>b%K$98sq;1&0HlxDltHE$V$+-6Lao#`1k^|e+eh98lrR`w^ryHVuo~{R4l`{V zi-C`I#UI{SJ0cMA42ybLKQ}jbK(GdT_Xjg7UH}b_9!N0j%IrA)s;fWP-{6umhv?~! z7(MViblZO^kiGr8VP$HazpARSeUVKN+xO2)A!OtEJ>K89$h{Ctq}dQLbj3D1iYLL~ znTikA9R%}j@lR&vLgHM@?~e5GPq`mlUKZJJsY$Lbv~S_aoS2}>>oWL(R}|^aMSWNz zUf%u3&KFylqswL^!9DTX_cPh`v0b`vSPe+cv7QBU?i-kpbkVP;@ANLq?tWzGl{)-j zV3@KCCjVey_iYVN&Ze&tyPaU;QEuYWCQ6l!Vi=2_UccR8MD$&7@G<~WD-g-kXQ<=q zzsROnV_%mrQo*H@D)p#Vz=-W30ZL~@tvDO#z&(PzXJYQ7zJ3i$?>LEFT zEqI68nolbDmk~pvgAu95a)9%Ay|~y>rx;?{Q$%~v0W{AF4FKq-eD2@uf0w@VzMi|+ z)&@+`_As&TiyuOLC($!oZ7H9pmZ0^fJ^(l`*Qd2QxFwFT?g#cAc1_B%MV;JRU-ia| z1^Nwn^)Y1C9#2+FheyXvpybxaTAuiNBtmoEntf>r2@kWP%VF<89Tk5(K$XhO3S)P0 z;8b0(ws&1pbZcSvm522XF0_8IFWce_1xA=(vxGi+K55;D$VKbPE8OwBfqYg3$uw*B zhos>wPDz?cXE#y2A%SfrOj??j@qp1#FbC-b6@REeSv z)o)+jb^E@>cg0b#THlO9P!u)UG3lz3fi^@pxHl5s>D#xmTtg#1&Gdi7@8*1;G7s~% z{W?tQj9v6hf-+DGovRJWRwAAd4)&xUH&tDtt5ET$G?t0^smD+!EU{got7nv4{+NA& zKaq3ZG2pwAnVZtWIN->ciK+<<)VH?GX|Q*MURh@Lcoq}AS6llwvpi&^Me=-4h?>xy6zRQa@SzeA8~p2GIWL_Et1>& zOfQb*lneWPlhldjGuA%=foz8X!q!sX?X;OkN69#|G)tMGan;UAF%jHxG-dH8RFp*Vte&{r}@$>V2Ljj%BG8kxR2}%s5SR%b}9p66k-bnMJQ3dy%9WK&XqATILF(DC}z_sprX+)lSzi~GIC zw1dAdAtpn0nCaQsFH5V33SI$oFWDPHJG)?6Wx^-yA+#bf(Y$s03Pqm#LoC|4^%C^l z8wKdTpWDfalX6Jcd%V*pZyYw-WBYN)@6YtrG{38|GA{KqNZTvZ0t@rGMjRjfn22c!xNlr(Pf%CShqH48Ee-F`Q z-t))G$zEc3i_d$0p~jP46^CYD6W3#>Tml6-6ZhFgc^Ar*2VnW{l$$o8J>1F@OJIh| zUI+1;ctH8dW}e<5*7MaCixOC~fR4VAtCFqxB>540@-Ct>lxonbiazh09N2Sx(LwVc z@s+l;G?_c6DzV(}OM)V$?;ktWRMX{M6o2~FcZ_K^u!V)}ZvTh%V@j0y|DHY>74Hl_ zk>APRh}amF{|EcQ7U^pCePOKJ;@8s3klUe*UU3RT6t;!rNuaU>6|X6KAzt_smw8Z8x;!tFaj=M;os;;rf0 zZQi9W+k08(L}BI_l%1gZp4|;T{>1tSc}$#SO^GJkxM-yR|(Lq#zGccrHH?;)dPR(Bm)RICjaHG&*A)~huAw*p3;;aFrPjmd*ObcnSO7`g>S{|nDX*@U-VK}BF$J6` z00P94cmM!u0>-yux`$2DZ0;ekf$k?+qiLMEVf-fmFkiafl@(^mTS{5{|GTzR4zTK=?MI5j^Ok)`f zNeNZSC#_x7ygRzy&gVjo`u@^%Y>^3KtgVC4a4Sk(24%I-HT|9;L`Ys&T@21aBC)c5iVPz$?jor<|K^!+0kL8EBj5~A26Fa)D zI(DosPkxnmP`wxL0841?B0nv<0qhM{Ny#~^uk zAK)l%$d>G5fLqd+<^b{2iZV8I=72IeG!!7f>q*qph$D~3G@Jv+G+YM}fHO#O)OVon zJwVgCRj857+wr}G0fPVZH}y+Iw-_0_~+=WMex4E+hJndc{( zKJ$83aidNy>M4ATV~3>|c*QT07XgnPQ;7L@3<}rrQiSo2xRIy|FXsROj1MuZ6-e(^ zwO2KZy1A!m^XNK~`bi9*mgWBd*{Ul*JgHdYAaPRIYAs@u1+{{8nL@Hm2~&ayJQ2=C zUc0ojxR>n~`i`Az6wyZJ5M;_RLAYU5hULw9OyF8qleIY?BHNgj5x zvW5g?1JI75oppB%fl@faC>)n7*W^GiChofDxMGf)sjZsbBub{o|H2dq@?G4n{oc@D}U^@b=SGB+J0qX z73u5c`JvJ4nvNNu7I9<#dDz0I^9fUpaj$IdZ1sz~gB$H=+9*}_us_}0qo!(IGT>Wy zKU9~?Qcdlu0!K{8ai8QX5*ur+Lg2@1Z!Gd2c0`4aK7zA!X=?11y68lxP_uSPuC`ir zEZ%8yN2xufqRn(dB#KfaxT`2&N`i6shkChxZ=`8=rq){v`DH~?#jceGN`v=&ypF!L ze&g*H(_h5f;hgCZ1_0rdua%McDXv%Mc{ed6o}~Awq~%p7C!$|_ve!d8E>+TUdL`3K zwwAhE%+wl&lNrQ%8k6fa)3^SEww{`t2pGko>BH(JZ~Y0)WHPP~Cm_^|Czlz+WS%Qu zV;gn*-}#*j4^L-*z5f82t0mg%8a452J4jjwiIBknLIETmdXdQMSjwz^VmKbP*?4{n z9X+q0lHy73Z!~G#N8|vAO}j`A?cCsT)2(s60|I|qnhks0yL9`Jo*Bp9<=Xmv%k>nT zD*$?$jOV3ExCrn%(>C$DHPCecLRTZVTGcuRr3d=ewu?QcyM+(r%qqo>m|@edwJl3c zovqR&ctbIXWZcR+<0pfjJu5Q)_wD+G*G3^2kmq1I1B`X9e`zVo)b?Aw_0vtxUu!Cp z<)v#a-umgL{{VS(Xl?hT)U>$3{{W|;{{Z?9Yd-ef?5{2@W!hwlLX1@85)W@`gdPYS z16_xS?hW3r;+W)u+x-^x%xsqEtN@g+ECx?e)X?Qp5?6Om<<%f^t4>Rq-9ML9zf+My zUAEI%zJ4iHfFm3lLNanc8t6K13~2%Fno4V}}3$ literal 0 HcmV?d00001 diff --git a/Tests/UnitTests.py b/Tests/UnitTests.py index 4c1a585..76aa138 100644 --- a/Tests/UnitTests.py +++ b/Tests/UnitTests.py @@ -7,7 +7,7 @@ import json; from io import StringIO from io import BytesIO - +import skimage # Define login Name and password before running the tests: Authentication_loginName = '***'; @@ -21,8 +21,6 @@ SkyServer_RadialSearchResultCSV = 'objid,run,rerun,camcol,field,obj,type,ra,dec,u,g,r,i,z,Err_u,Err_g,Err_r,Err_i,Err_z\n1237671939804561654,6162,301,3,133,246,3,258.250804,64.051445,23.339820,22.319400,21.411050,21.119710,20.842770,0.664019,0.116986,0.076410,0.080523,0.238198\n' SkyServer_RectangularSearchResultCSV = 'objid,run,rerun,camcol,field,obj,type,ra,dec,u,g,r,i,z,Err_u,Err_g,Err_r,Err_i,Err_z\n1237671939804628290,6162,301,3,134,1346,6,258.304721,64.006203,25.000800,24.500570,22.485400,21.103450,20.149990,0.995208,0.565456,0.166184,0.071836,0.124986\n' SkyServer_ObjectSearchResultObjID = 1237671939804561654 -SkyServer_ImageCutoutImagePy2 = '\x1c\x1b\x16\x17\x18\x12\x1c\x1f\x16\x15\x1d\x12\x0e\x19\x0b\x0e\x1c\r\n\x18\tz\x8b{\x10\x1c\x0e\x0e\x19\x0b\x17\x1e\x16\x1e!\x1a!\x1d\x1c%\x1c\x1d\x1f\x13\x17\x0b\x06\x00\x13\x1d\x02\t\x1d\x00\x07\x18\x00\x9f\xaf\x94\r\x19\x03\x17"\x11\x19\x1f\x13\x19\x1c\x13\x1c\x1d\x17"\x1f\x18$\x1f\x19&\x1f\x15&\x1c\x12&\x1d\x0e+\x1f\x11%\'\x11\x8b\xa7\x81\x02 \x00\x1f#\x0c-\'\x17\' \x16\'%\x19\')\x1c,-\x1b=2 4-\x13#\'\x06\x19)\x02\x97\xb3\x8a\x13-\x06&1\x1172\x1cB,!B\'\x1e=%\x1b8%\x164\'\x162*\x13.*\x11+*\x0e,,\x10\x84\x84h-)\x0e4.\x14,$\x0f7,\x18/ \r<-\x1a\x15\x16\x0e\n\x0c\x01\x11\x17\x0b\x0e\x16\x07\x0b\x18\x07\r\x1c\t\x17(\x16\xb8\xc9\xb7\x19&\x15\x18#\x13\x1a"\x15\x1c\x1f\x16\x1a\x17\x12\x18\x10\x0e\x1e\x12\x12%\x1e\x16\x16 \x05\x16\'\x03\x1a+\x0b\xb6\xc4\xaa\x1f+\x15#,\x1b\x1b!\x15\x17\x1a\x11 \x1f\x1a$!\x1a)"\x1a) \x17)\x1f\x15) \x11."\x12(*\x14\xa9\xc5\x9f\x186\x10(,\x15.(\x18\' \x16*(\x1c.0"76$;0\x1e:3\x1926\x15.>\x17\xac\xc8\x9f!;\x14,5\x1661\x1b?+ A)\x1f?)\x1c=,\x1c:/\x1d60\x181/\x16+.\x11-0\x13\xd4\xd7\xba,,\x10)(\x0c<6\x1e1)\x144)\x155*\x16\x1c \x11\x14\x1b\x0b#,\x1b\x15!\r\x17%\x0e\x12"\x08\x08\x1a\x02\xb7\xc9\xb1\x10\x1e\x07\x13\x1f\x0b\x16\x1d\x0b\x1c\x1e\x10#\x1f\x14\x1d\x14\r\x1e\x13\x0f,&\x1a\x17 \x05\x1b*\t$3\x14\xc1\xce\xb4\'0\x1b\'.\x1e $\x16\x1c\x1e\x13%%\x1d(%\x1e,%\x1d-$\x1b-#\x17.%\x162&\x16+-\x15\xb9\xd5\xaf!?\x19*.\x17.(\x18\'!\x15+)\x1d.0"54"9.\x1c<2\x199:\x1a7F\x1f\xb5\xd0\xa7&@\x19,5\x163.\x18;+\x1c>*\x1f>.\x1f>3!=6#:8\x1f26\x1b.5\x16\x1e%\x06\xd9\xe0\xc1\x1e%\x06$)\x0b7:\x1f33\x1975\x1e:5\x1f\x1d&\x11\x19#\x0b\x1c(\x10\x00\x0e\x00\x10 \x03 0\x13\x13&\x08\xc3\xd6\xb8\x13"\x05\x16#\t\x12\x1a\x02\x16\x19\x04)&\x15,$\x17&\x1c\x12("\x12\x1d#\x07\x1f-\x0c\'2\x14\xbf\xc9\xb0"*\x15\'+\x1a(*\x1d,,"+(\x1f,(\x1f0\' 0\'\x1e1\'\x1b2)\x186+\x19.0\x18\xb3\xcf\xa9\x1d;\x15)-\x142,\x1c/)\x1d1/"-/!.-\x1b>1\x1e=3\x1a67\x171@\x19\xaf\xca\xa1$=\x16/6\x1772\x1c:-\x1d=- ;/\x1f<5"<:#:="4=\x1e1<\x1c8F%\xb5\xc3\xa20;\x1b=H(.7\x1a;A%/3\x1a7;"\x10\x1d\x01\x10\x1f\x00\x19(\t\r\x1e\x00\x1d.\x0c\x1c0\x0b\x12&\x01\xc3\xd7\xb2\x1b+\x07!/\x0e\x1c%\x08\x16\x1a\x01" \x0b3,\x1a5,\x1b0)\x16\',\x0e+4\x15.7\x1a\xc6\xcb\xb4(+\x16-.\x1e00$86*0,!1* 1(\x1f3)\x1f6*\x1c8,\x1c:-\x1c22\x18\xb4\xd0\xa8\x1f>\x15.2\x1994!82$97*34&10\x1eD7$C7\x1f:9\x1a5A\x1b\xb2\xcb\xa3)?\x195:\x1c>8"=1!<0"90\x1f94 8:"8>"6A!3B!;L(\xac\xbd\x992C!:I(,;\x1c8E\'/< 8B\'\x12!\x00\x12#\x01\x16\'\x03\x1f0\x0c 5\x0e\n\x1f\x00\x13(\x01\xc3\xd5\xaf\r\x1d\x00\x13!\x00$-\x0e*/\x11(&\x0f71\x1bA9&5/\x19..\x1227\x197; \xcd\xd1\xba.0\x1b21\x1f1-!3/#6/%5-"5+!6, :.\x1e<1\x1f=0\x1d44\x18\xbb\xd7\xae&B\x1904\x19:3!60"97(78(96#D5"F7 >=\x1e+5-\x180.\x1525\x1888\x1e\xce\xcd\xb81.\x1b30\x1f2,\x1e3+\x1e;3(:0$8.";/!>2"@5#A4!88\x1c\xbb\xd6\xad\'C\x1a27\x19;4!60 ;8)<:+>;(F7$G9\x1f@=\x1e>F!\xbb\xd0\xa91B\x1e:: ?7"I=-H<,E<+D?+CC+AE*=F\':E#BR.\xb9\xca\xa6IZ63D EV40A\x1f2A"+:\x1b\x92\xa5\x85\xc3\xd6\xb6\xb7\xcb\xa8\xb9\xcd\xaa\xc3\xd4\xb0\xc3\xd4\xb0\xb0\xc1\x9d\xc9\xd9\xb5\x8d\x9bx2=\x1b\x19\x1f\x0303\x18+)\x1470\x1dJA091\x1e62\x1964\x1b53\x1c\xcd\xc8\xb22-\x1983 :3#<5%?6\'=4%=1#>2"A6$D9%C8":;\x1c\xb6\xd1\xa6&C\x177<\x1cD>(@9)B?.BA/E@-P?+N>%C> >D \xbc\xcd\xa94C"B@\'K@,NA1NA1NC1MG1KI0II-BG\'>E$\xa0\xab\x89\xc6\xd4\xb1\xbe\xcc\xa9\xbb\xcb\xa7\xc1\xcf\xae\xc6\xd4\xb3\xc1\xce\xb0\xc6\xd3\xb5 0\x15$4\x19\x16&\t!1\x14/>\x1f\x10\x1f\x002A"\xa4\xb1\x93\'2\x14(1\x14+/\x18/.\x192/\x1e81!<2&>5&C=\'/+\x12D?)\xb7\xb1\x9b5.\x1bE>,1(\x17G>-A8\'D<)F>+C;&?7">6\x1fC;$AD%\xb2\xcd\xa01N 6;\x1bE?\'F?-B<,GD3ID0RA-D4\x1bWP3DH%\x9e\xae\x8aJX7FB)L?,UC5WE7WG7TI5QI2OI/LK-LM.GK*\xb2\xb8\x96?F$?I\'EL-?F\'GM1BH,\x0e\x1c\x05$2\x19\x1f+\x13\x1d*\x10,9\x1f\x1e+\x11\x19#\n;E,,4\x1c.3\x1c02\x1d21\x1f52#:4(>6+?7*B;(60\x18<6 PJ42+\x18@9&H@-C;(C;(F>)H@+E?\'B:#A;!F?%BH&\xb4\xd1\xa15R"<@\x1fJB+JC0F@0KH5MF3[I5VD,NE(34\x12_mJ?J*HB*XH8VE5YF7ZI7YJ5UI3SI0RK/RM/@=\x1eonO>?\x1fDH\'AD%GJ-NQ6PS8\x19"\x0f*3 )0\x1e!)\x14,4\x1f08!%*\x14\x1d"\x0c.1\x1c/0\x1e./\x1d1/ 3/#70&:3+<4)5.\x1cA9$A9$&\x1e\tG?*H@+LF0B<&F@*HB*JD,HD)F@&EA&JD*EL*\xb8\xd5\xa5;X(CG$PI/OH5LG4PM:RJ7WB/eQ9WL0FF$BM+HQ2PH3[J:\\I:^K<_N<^O:\\M8YM5YO4ZQ4TM0E@ UP2XU6JF)SO2SO4XT9,0!(,\x1b-0\x1f47$25"14\x1f9;&>@+21\x1d21\x1f10\x1e2/ 3/#70&92*:3)5,\x1bIA,D<\'IA,NF1KC.>8"PJ2IC+JF+LH-JI-IH,IH*LK-GQ,\xb8\xd6\xa4>\\*IM(TM1RL6QJ8VQ>UM:VA.gQ:gY>__=IS1W^?_T@_N>bQAcSCdUBdU@aU=`T:`U9aV:lcDMD%kdGc\\?\\U9VQ4WQ7YS91/ $#\x110-\x1cA>+97"/-\x1842\x1d97":8#:7$96#96%:7(<8,>:/A;/F=,I>,?7"VN9=5\x1eG?(D>&YU:KG,KJ,ML.MN/NO0NO/PQ1JV0\xb4\xd5\xa0A_+MQ,XQ5VN9VO<[VBZO;lVAdN7gY>]Z9OV5PU7h[HiXHcVEdYGf[Gf[EdZAcY>cZ=d[>bY:`W8i`AYR5jcGYR6d]C`Y?94!0+\x182+\x1870\x1d:4\x1eA;#@:"3-\x15?9#>9#>9%<9&=:)@=.B>2E?1LC2C8&IA,JB-D<%PJ2[WWQ1qjM]V:leKg`FC;&?7"7/\x184,\x15@9\x1fNG-MF*D=!A: @: ?;"?:$?=(B?,DC1FC0I@/MB0SK6ME0UO7\\V>^Z?JF)SR4ST4UV6VZ9Y]Z`>Sb9\xb6\xd7\xa2Ig1Z^9d]@c[De_IleRi^Jv`Ks[CtdJkfHlqQ`cHm^Km\\LkfPhjRllRllPkjNjiKmjKojJpkKtoOmgGhcCrmOfaCniLkfI:6\x19=9\x1c>:\x1dD@#KF)C> >9\x1bE@"E@#E@#D@%EA(GC*JE/KI2OK2QF0^S=OD0SK8PI6VQ;MI._^@VW7WX9YY=[^A``D`cD_c@Yf:\xb9\xda\xa3Nl6^e;jdBlaEoeLtmSpfMr`H|hOrbHibEghIllPphSwo\\oq[mu]qv_qw]ptYpsTstTtuUyyWccAnnLttRlkLklLlmNmnO:E\x1b8C\x197?\x187?\x1a;A\x1d@D#CG&EF&ML.NM/LH+FB%GB%NI+VO2YP1ZP-cU8RG3SJ;meZMG7aaEPQ1W[:ccGZYEvsbd_KqkQfbJ"AK&DL\'GM+HL+:;\x1cCB$OL-VQ3VP0VM.WN-ZP-XL$TG$TG4bXLg^UPM>eeKkoNbfEUX=mk\\b^Sg_RskTzrMip=\xab\xcf\x92W}@hvAwsF\x7fmE\x90}U~pI|tMytN{sO\x7fpS\x81rU\x83uZ\x7f{^x\x82`t\x84gv\x88ry\x8ax~\x8f|\x83\x92{\x84\x92x\x81\x8er|\x87gx\x83a\x7f\x89f}\x89ez\x86bt\x82^p~ZjzUeuPbtN/<\x105B\x16=H\x1eCK$CK&CI%EI(HI*baCWS6PK-TM0[R3[P2]R2cY6\\P(gZ7k^K\\PD\\SJ\xb4\xb1\xa2\xed\xed\xd3\xff\xff\xe3\xf6\xfb\xdb\xe0\xe3\xc8\x8b\x89|\xc4\xc0\xb7\xd3\xcb\xc0\xc8\xc0\xab\xe9\xe1\xbds{J\xab\xce\x94\x82\xa8mr\x80M~zM\x8byQ\x93\x80V\x8b~T\x85~R\x81|R\x84|U\x89zY\x8c{]\x8d~a\x8a\x83f\x84\x89i\x80\x8do\x83\x92{\x8a\x99\x84\x93\xa1\x8a\x96\xa2\x8a\x92\x9c\x81\x8b\x94w\x84\x8dn\x80\x8ah~\x85c}\x87b}\x87b|\x88bz\x86`u\x84]q\x80Yn~W\xb6\xbf\x94\xbd\xc5\x9c\xc6\xcc\xa6\xc9\xcf\xa9\xc7\xcb\xa8\xc5\xc9\xa8\xc9\xca\xab\xcd\xcc\xae\x88\x84ge`CJC&PG*_T6eX8eX8h[8na7m_:qbMeYKnf[\xdb\xd8\xc9\xb0\xb0\x96fiJx}]rv]\x84\x84x\xe9\xe6\xdf\xed\xe6\xdc\xc8\xc1\xaf\xd1\xcb\xabnuI\xc3\xe6\xb0\xbf\xe4\xae\x87\x94f\x88\x83Y\x95\x84Y\x96\x83Y\x98\x8b^\x91\x8b[\x8a\x86W\x8c\x85Y\x91\x82[\x93\x80_\x92\x82a\x92\x87i\x8f\x90p\x8d\x96w\x94\xa1\x87\x9c\xa8\x90\xa5\xaf\x97\xa3\xad\x92\x9c\xa2\x86\x91\x98w\x8c\x91q\x8b\x91o\xc0\xc6\xa2\xc3\xc9\xa3\xc5\xcd\xa6\xc7\xcf\xa8\xc6\xd1\xa9\xc3\xcf\xa9\xc1\xcd\xa7\xbf\xcb\xa5AH\x1fFM$LP+LP+KL*JI*OK.TO2kdHWM2NC\'^P5l^Ak\\=eV7eW4uf;bT-m^Gxl^wod\xd1\xce\xbf\x96\x96|dgHpuWlpY\x92\x92\x88\xff\xff\xfa\xff\xfe\xf8\xb5\xae\x9e\xda\xd3\xb6\x86\x8dd\xcb\xed\xbb\xe2\xff\xd4\x94\xa1u\x93\x8ed\x9d\x8ca\x9c\x89^\x9e\x94c\x96\x90\\\x90\x8dX\x92\x8aY\x95\x84Y\x95\x7fX\x93\x7f^\x94\x85f\x95\x92q\x96\x99z\xa7\xad\x93\xab\xb4\x99\xb0\xb6\x9c\xaa\xb1\x92\x9f\xa2\x83\x95\x99v\x94\x95s\x96\x98s\x8a\x8ce\x8a\x8ce\x89\x8ef\x88\x8de\x84\x8bb~\x86_y\x81Zt\x7fWMO(PR+RR.QQ/PM.PM.UP3YR6VL1XM1aS8k\\?j[l\\Bn^Dl[?hW9kX8s`@{iCwi:\x7frFzlQ|o^\x94\x8b|\xe0\xdb\xc8\xb5\xb4\x98\x86\x8ai\xc7\xca\xab\xe2\xe4\xcc\x9f\x9d\x91\xff\xff\xfa\x9c\x95\x8f\xe3\xdb\xce\xbf\xb8\x9c\x95\x9bu\xbd\xd9\xb0\xdf\xfc\xd0\x95\x9dt\x9c\x94m\x9b\x8a_\xa4\x92b\x98\x8fX\x91\x8cR\x8d\x8bN\x93\x8aQ\x9a\x86S\x9a\x7fR\x97{T\x95}Y\x95\x87d\x95\x8fm\xa2\x9d}\xa4\xa1\x80\xa5\xa2\x7f\xa2\x9ey\x9c\x97q\x99\x91j\x9a\x93i\x9d\x96l\x93\x8c`\x91\x8d`\x91\x8ec\x90\x8db\x8f\x8dd\x8b\x8cb\x89\x8bc\x88\x8abZT2ZT2]T5`W8cZ=f[?eZ>eW\x83vJ\x8e\x7fb\x7fr_\x9a\x91\x80\x9f\x9a\x84\x8d\x8dk\xa9\xa9\x83\x9d\x9dy\xff\xff\xe4\xca\xc5\xb2\xff\xfc\xef\xa7\x9b\x8d\x9f\x93}\xd4\xc8\xa2\xe8\xe7\xb9\xd0\xe6\xb7\xdd\xf3\xc5\x90\x93h\x9a\x8dc\xa1\x8ca\x98\x84R\x9e\x95\\\x8c\x88K\x91\x90P\x9b\x93U\x98\x83L\xa4\x86T\xa5\x83V\x97wN\x9b\x84Z\x93\x82W\x92\x83X\x93\x86Z\x95\x86[\x96\x85Y\x94\x83W\x94\x83U\x96\x84V\x97\x87V\x9f\x8f^\xa2\x94c\xa2\x96f\x9d\x92d\x98\x90b\x9b\x92g\x9d\x96j\x9d\x96jbU3fY7k^h[;h[;l]>paBveGwgF{hG~lH\x82nI\x85rJ\x88uK\x88wK\x8a|K\x8b~R\x89{^\x8b\x7fi\xb7\xaf\x9c\xe7\xe1\xc9\xc0\xbd\x9a\xa1\x9fv\xda\xd8\xaf\xf4\xf1\xce\xb9\xb2\x98\xf1\xe6\xd2\xb0\xa1\x8c\xa0\x91r\xaa\x9co\xe7\xe4\xaf\xd8\xe9\xb5\xf4\xff\xd4\xa8\xa5x\x95\x86[\x9b\x84X\x94~M\x93\x88R\x9c\x98[\x88\x87G\x9a\x92T\xa1\x8aT\xaa\x8aW\xa8\x83V\x9czL\x96{L\x9b\x86W\x9a\x85V\x9a\x85V\x9a\x85V\x99\x83R\x96\x80O\x94~M\x96\x7fM\x96\x80N\xa2\x8cZ\xa7\x93a\xa8\x96f\xa5\x94f\xa1\x93f\xa1\x94h\xa1\x96i\x9f\x94ghZ7l^;pb?qbApa@pa@tdCwgE\x7flK\x80nJ\x81oI\x84rJ\x87tJ\x89vK\x8byK\x8b{J\x89{J\x8e\x81U\x8c~a\x96\x8at\x95\x8av\xe3\xdd\xc3\xf9\xf5\xcf\xff\xff\xd4\xfb\xf9\xc9\xa7\xa2x\xa8\x9d}\xfd\xee\xd1\xa8\x95u\xa4\x91g\xa0\x8bT\xc0\xb6x\xee\xf9\xbd\xf8\xff\xcd\xab\xa5s\x91\x7fQ\xa9\x8ea\xaf\x97g\x9b\x8f[\xaa\xa5m\x9c\x9a_\x9c\x93Z\xa5\x8eX\xae\x8e[\xc2\x9el\xb1\x8d[\x95zE\xa0\x89S\x9e\x87S\xa0\x86S\x9e\x84Q\x9b\x80K\x97|G\x94yD\x96xD\x94yD\xa0\x86S\xa6\x8f[\xab\x95c\xaa\x96d\xa7\x95g\xa5\x94h\xa0\x92e\x9c\x8eak]8oa\x95t>\x95v?\x9d~H\xa3\x88S\xaa\x8e\\\xa9\x92`\xa8\x92c\xa5\x90c\x9e\x8c^\x97\x84Yn`9qcxkA|oE\x81rI\x84uL\x86wP\x88yP\x8d~S\x8d\x7fR\x8e}O\x8d}L\x8d}I\x8c}F\x8b|C\x89|E\x8a~J\x86{N\xa3\x98z\xf8\xec\xd4\x9f\x95|\x95\x8ck\x8c\x86T\x84~B\x8f\x85G\xb2\xa4g\xa7\x92[\xae\x95_\xb2\x93Z\xbb\x9aU\xc4\xa2K\xb2\x9b>\xcd\xcbv\x9f\xa0T\xbe\xacl\xff\xe1\xab\xcd\xab}\xcf\xb2\x88\xf4\xe2\xba\xde\xd5\xac\xfa\xf6\xc9\xbe\xb3\x86\xb6\x9ep\xdb\xbb\x8a\xf3\xce\x97\xa8\x86G\xa2\x85?\xa3\x88A\xa3\x81B\xa5~C\xa2{@\x9ex:\x9as8\x98r4\x9bs8\x9bt9\x9ez@\xa0}E\xa0\x7fJ\x9e~K\x9b\x7fO\x9b\x80Q\x94|N\x8cuIzmA{nB\x7frF\x83vJ\x88{O\x8c\x7fS\x8e\x81W\x8f\x82V\x94\x86Y\x94\x86W\x93\x85T\x91\x84P\x8f\x82K\x8e\x82H\x8d\x81E\x8c\x80F\x82vB\x96\x8a`\xbc\xb1\x95\xfb\xef\xd7\x9b\x91x\x8d\x85a\x9f\x98d\xa4\x9c^\xad\xa2`\x99\x8aG\xa9\x93W\xc5\xa8n\xbc\x9aZ\xc5\xa1U\xdd\xb9[\xb7\x9e:\xb4\xb0U\xa9\xa8V\xc3\xb0k\xff\xe2\xa9\xca\xa5x\xc3\xa3|\xef\xdd\xb7\xcf\xc6\x9f\xe9\xe4\xbc\xd4\xcb\xa0\xb7\xa0t\xfb\xdb\xaa\xd2\xadv\xa9\x87G\xab\x8fF\xa2\x86<\xa5\x82@\xa7\x7fB\xa3{>\x9ev8\x9ar5\x98p2\x9ap4\x9ct7\x9cu<\x9au>\x93r=\x8en;\x8eo@\x8fsD\x8cqD\x85m?xvEywF}{J\x82\x80O\x88\x86U\x8c\x8aY\x8e\x8c[\x8e\x8cY\x92\x90]\x91\x90Z\x8f\x8eV\x8e\x8dT\x8b\x8bO\x8a\x8aL\x88\x88H\x87\x87I\x8f\x8cY\x81~S\x96\x96r\xf6\xfc\xda\xdd\xe8\xc0\xec\xfa\xc9\xfe\xff\xcc\xfa\xff\xbd\xed\xf1\xa5\xb6\xb2h\xab\xa1Z\xbc\xabe\xcd\xban\xc3\xafY\xc4\xb0M\xcd\xc4[\xba\xc4a\xc3\xc9o\xc4\xb0g\xfc\xdc\x9f\xde\xbe\x8b\xdc\xc1\x94\xf9\xea\xbf\xc3\xba\x91\xe0\xd9\xaf\xe9\xdd\xb3\xad\x9cn\xff\xf4\xbf\xb6\x9e`\xb4\xa0Y\xae\xa1R\x94\x878\x9f\x89@\xa1\x86A\x9c\x81<\x96{6\x91v3\x8ft1\x8eu2\x90w5\x8ev8\x88t7\x80m3wf.wh1yl7ym9tj7\xb3\xc7\x8c\xbb\xcf\x94\xc2\xd6\x9b\xc2\xd6\x9b\xbd\xd3\x95\xba\xd0\x92\xbd\xd3\x93\xc2\xd8\x98\xbc\xd2\x92\xbc\xd2\x91\xbd\xd3\x92\xbe\xd5\x91\xbf\xd6\x92\xbf\xd6\x90\xbe\xd5\x8f\xbf\xd4\x93\xc8\xd2\x9e\xbf\xcc\x9e\xb6\xd1\x9e\xd7\xfe\xc5\xa8\xdb\x99\xac\xe6\x9c\xa2\xd9\x86\xae\xe0\x89\xb1\xd5}\xcc\xe4\x8c\xc5\xd3|\xc9\xd3|\xc7\xd0u\xcc\xd7w\xb6\xc9`\xbb\xd7i\xb6\xddp\xc3\xdcw\xc6\xb5e\xfa\xd9\x96\xf9\xe4\xa5\xe2\xd6\x9c\xfa\xf3\xbd\xd6\xd0\x9e\xda\xcd\xa0\xff\xf4\xc4\xe1\xdc\xa4\xf8\xf7\xb7\xd2\xd2\x8a\xcc\xd1\x81\xbf\xcfw\xcb\xdb\x82\xd3\xd7\x83\xd5\xd4\x82\xd4\xd3\x83\xd0\xd1\x81\xce\xd1\x82\xcd\xd1\x84\xcb\xd2\x85\xca\xd2\x87\xc9\xd3\x8c\xc8\xd5\x8f\xc7\xd6\x91\xc4\xd5\x91\xc0\xd3\x91\xbe\xd3\x92\xbf\xd3\x94\xbf\xd5\x95y\x8dR~\x92W\x82\x96[\x81\x95X}\x93Uz\x90P|\x92R~\x94Sx\x8dLw\x8dLx\x8fKy\x90L{\x92N}\x94P~\x95O\x80\x94U\x86\x92`\x8c\x9en\x95\xb8~\xbf\xf4\xb0\x94\xd7\x8a\x8f\xda\x85|\xc5k\x87\xc8j\x8a\xbb_\x98\xbd`\xa8\xc1e\xbf\xd2v\xbf\xd4u\xad\xc7d\xa1\xc4^\xac\xd7l\xad\xe2p\xba\xdbr\xca\xb8f\xe5\xc3|\xff\xef\xab\xf1\xe9\xa8\xc9\xc3\x87\xb9\xb0y\xc9\xb5\x83\xf2\xe1\xad\xe5\xdf\xa3\xc5\xc6\x84\xb0\xb2i\xa6\xad]\x96\xaaS\x85\x98?{\x80,\x7f~,|~+{|,z}.w{.rz/nx/bo)_n)\\m)Wj(Ti(Ti(Uk+Xn.\x80~M\x80~M\x80~M\x7f}J\x7f}J~}G~}G~}E}{B}{B}{@~|A\x81\x7fB\x85\x83F\x88\x86I\x8a\x89Py{J\x86\x90^\x98\xb4z\xc5\xf2\xaf\xa6\xe0\x94\x9d\xe1\x8e\x8c\xccu\x9a\xd2y\xa4\xcfv\xae\xcct\xb7\xcbt\xb7\xc5n\xb1\xc1h\xb7\xces\xbc\xd9|\xb3\xd9t\xb0\xe1t\xb7\xd6o\xce\xb9j\xd0\xa9d\xff\xf4\xb0\xff\xff\xbe\xb2\xa7i\xba\xaao\xc4\xa6r\xff\xe1\xad\xff\xff\xc5\xc1\xb3t\xaa\x9bX\x99\x8fG\x96\x97I\x83\x822\x89w-\x8ft-\x8du-\x8fw1\x90y5\x8bv3\x84q/~l,ue)sd)nb(i^&e\\%f]&ha+le/\x8b\x7fU\x89}S\x87|O\x89~Q\x8c\x81S\x90\x84T\x8f\x83S\x8f\x81P\x90\x83O\x8e\x81L\x8f\x80I\x8f\x80I\x91\x82I\x97\x86N\x9c\x8bS\x9c\x8fZ\x96\x8e]\x9a\x9cj\xa3\xb4}\xc6\xe6\xa5\xb3\xe3\x99\xae\xe5\x94\xa3\xd9\x85\xb4\xe4\x8e\xc1\xe3\x8f\xb2\xc8v\xb9\xc6v\xe7\xf0\xa1\xcb\xd7\x85\xc4\xd3\x80\xb8\xd0z\xc3\xe4\x89\xb5\xe2}\xb6\xd0o\xca\xb0e\xc0\x95Q\xee\xd3\x8e\xfa\xe7\xa3\xb1\x9e\\\xbd\xa5g\xb8\x91Z\xe2\xb9\x83\xff\xe4\xab\xae\x91U\x9f\x7fB\x94v6\x8cy4\x86s.\x85d!\x88b!\x8be&\x8fi*\x91m/\x8fk/\x86c)}]${\\&z\\(x\\*vZ(tZ)u[*v^.ya1\x81yR~vO}vL\x81zP\x89\x80U\x91\x86Y\x91\x86X\x91\x85U\x8a|K\x88{G\x87wC\x87v@\x88w?\x8e{C\x93\x80F\x94\x83M\x9e\x8e]\x9c\x95a\x9b\xa3j\xb6\xcb\x8a\xb1\xd2\x8b\xa5\xce\x82\x94\xbdn\xa3\xc6v\xa8\xbfq\xb3\xbfu\xb8\xbbt\xe5\xe6\xa0\xab\xb1k\xb7\xc1z\xae\xbbs\xbf\xd8\x87\xba\xdf\x82\xb5\xc8n\xc1\xa0Y\xbc\x8bI\xba\x97S\xba\x9fX\xa7\x8cG\xa4\x82B\xb3\x82H\xacx?\xaa}D\x8d`\'\x98g/\x9cm5}V\x1byW\x1a\x82^ \x84`"\x88d(\x8di-\x91m3\x90k4\x89f0\x82a,}\\\'\x7f_,\x82b1\x83d5\x83d6\x83d6\x85f:\x87h<\x81}X\x80|V\x81}W\x86\x83Z\x90\x89_\x95\x8eb\x97\x8fa\x98\x8d_\x8d\x81Q\x8c\x7fK\x8b{G\x8ayC\x8cyA\x8f|D\x94\x7fF\x96\x81J\x9d\x87U\x9f\x8f\\\xa4\x9fg\xbb\xc4\x85\xc0\xd4\x8f\xa7\xc1x\x8a\xa4[\x8f\xa3[\x9f\xaae\xb6\xb9v\xbf\xbb|\xeb\xe5\xa9\xa8\xa6i\x9e\x9fc\x8a\x90T\xa9\xb8s\xbc\xd9\x85\xb0\xbch\xb9\x92O\xc2\x8bK\xa0x3\x91p)\xa3\x82;\x92k&\xacv8\x9db(\x86R\x18\x90]$\x9c`,\xa0f4\x7fP\x1c\x86]\'xU\x1dyY\x1e~^%\x82b)\x83b,\x83b,\x80a+~^+~^-\x81c1\x85f7\x88i;\x89j<\x89j>\x89j>\x8ak?}\x7fZ~\x80[\x81\x81[\x85\x86^\x8c\x89`\x8e\x8b`\x8f\x8b\\\x90\x8aZ\x8b\x81P\x8a~J\x8b{G\x8ayC\x8dzB\x8f|D\x93~E\x96\x80G\xa2\x89S\xa8\x91]\xb0\xa1h\xc2\xbe\x81\xcd\xd2\x90\xaf\xbav\x93\x9eZ\x9b\xa2_\xb4\xb4v\x9d\x97]\xb6\xabu\xff\xfc\xc8\xfe\xf4\xc1\xb2\xacz\x8e\x8c[\xbb\xc3\x88\xc2\xd8\x8d\xa9\xad`\xaa|>\xb8z?\x9eq.\x8dh!\x99w-\x8fg\x1f\x93^\x1c\xa3i*\x8f[\x1f\x9ag.\x9b_-\x95[-\x89Y)\x8ab/y[%xb)|f-|f-|c-{b,}c0\x80f3\x86l;\x88m>\x8bpA\x8cqD\x8cqD\x8cqF\x8drG\x8ftI\x82\x7f^\x85\x82_\x88\x84a\x8a\x86`\x8c\x87a\x8d\x86\\\x8c\x83X\x8f\x82U\x89{L\x8byG\x8bwE\x8dvB\x90vC\x92yA\x95zC\x99|D\xa0\x81K\xa5\x87Q\xa5\x90W\xb1\xa2g\xbe\xb8z\xa6\xa2c\x98\x94U\xa9\xa3g\xb8\xadu\xba\xadx\xb5\xa3s\xa9\x97i\xed\xdc\xb0\xff\xfc\xd2\xff\xff\xd8\xf0\xf0\xbe\xca\xd9\x94\xa1\x9eW\x98e,\x9d\\$\x96f&\x88c\x1c\x7f]\x13\x82\\\x13\x8d[\x16\xacv6\x97i+\x98k0\xa5q?\x9fl?\x9bpC|Z,\x7fe4\x80k6\x84o:\x81l7{e3zc1\x7fg7\x84l<\x86n@\x87oA\x89nA\x88mB\x86k@\x87lA\x89nC\x8bpE\x89{^\x89|\\\x8c}^\x91\x81_\x95\x82a\x95\x81\\\x94~W\x93zQ\x8ftI\x91rD\x95tG\x9ezJ\x9fyJ\x9evC\x9dsA\x9evB\x9e{E\x9e\x7fH\xa1\x87N\xa1\x8dR\x9b\x8bO\x95\x86K\x9c\x8dR\xa7\x98_\xaf\x9bh\xc1\xad{\xb8\xa1u\xa6\x8ff\xa3\x8df\x94\x82^\x97\x89f\xb5\xb1\x84\xf7\xff\xc2\x92\x8dK\x99d0\x98U!\x95e\'\x8ci%\x86h\x1f\x89i \x9ao*\x9dq0\x9ax8\x9c|A\xa1yF\x9dwH\x92uI\x8duG\x8bsE\x8evH\x8duG\x8arD\x89qC\x8cqD\x8aoB\x87l?\x8frH\x89lB\x87g>\x87g@\x8bkD\x8dmF\x8ajC\x86f?\x8a{\\\x8c}^\x8f\x7f^\x90\x80^\x92\x80\\\x91}X\x92{R\x92yP\x92uK\x92sE\x97uG\x9f{K\xa5}L\xa3yI\xa0vD\xa0xD\x9cyC\x98wA\x97zB\x9a\x80G\x9a\x81H\x94~D\x94}G\x98\x81M\xa4\x8d[\x9a\x7fR\xa0\x87^\xb5\x9bv\xb2\x9av\x9e\x87g\x8dzZ\x82yP\xf5\xfd\xc2\x90\x8aL\x9ad5\x98V&\x94g,\x8eo.\x89s*\x8du-\x9ay6\x9ax8\x94|<\x91}B\x95yG\x94yJ\x8d|N\x8a}P\x8f~R\x92\x7fT\x91~S\x8cyN\x89vK\x89vK\x89tI\x84oD\x82kA\x7fh>~e=\x7ff=\x81h?\x82i@\x80g>}d;|xS\x7f{U\x83~X\x84|U\x81xO\x81uK\x82sH\x86uI\x8auH\x88rA\x8crA\x94xF\x9c~J\x9d}J\x9c}G\x9d~H\x9e\x7fH\x98yB\x96w@\x9a}E\x9f\x82J\x9c\x7fG\x96xD\x90tB\x95yI\x9f\x82V\xd7\xbc\x91\xf0\xd7\xaf\xbc\xa4\x80\x90yY\x98\x81b\x9b\x8fg\xe8\xf1\xb8\x88\x86K\x97g9\x97Z-\x95n7\x91y;\x8d~;\x91\x82?\x9d\x85G\x9a\x82F\x8d\x81E\x89~F\x8d{I\x8c~O\x89\x85V\x8a\x89[\x91\x8ea\x94\x8fe\x91\x8cb\x8a\x85[\x85\x81T\x83\x7fR\x81zN|uI{tH{tF{rGzrDwoAtl>qi;og9syMw}Q}\x80S{~QwyJvuE{wG\x80zH\x90\x86S\x89}I\x84wB\x89zC\x8f|D\x8f|D\x90{B\x92|C\x9f\x86N\x9b\x81H\x9b}G\xa0\x81J\xa4\x85N\xa2\x83M\x9bzG\x94tC\x90qBz^/\x98}R\xdd\xc4\x9b\xff\xf7\xd1\xff\xfc\xda\xff\xfb\xdc\xff\xf4\xcc\xd3\xe2\xa9|\x82F\x95l>\x99d:\x98xE\x95\x85J\x92\x8aL\x95\x8dO\xa0\x8dU\x9a\x89S\x8e\x87P\x89\x83O\x8c\x7fR\x8e\x83V\x8b\x8a\\\x89\x8fa\x91\x98l\x92\x9bp\x8f\x96l\x86\x8fb\x80\x87[}\x84Xx\x7fSrzKgo@iq@lrBkqAhn>el9bi6bi6\xb6\xc7\x93\xbb\xcc\x98\xc0\xce\x99\xbf\xcc\x97\xbd\xc7\x92\xbd\xc6\x8f\xc2\xc9\x93\xc9\xce\x96\x94\x97^\x8b\x8cR\x81\x81E~|A\x7fy=}w;\x80w<\x84z?\x98\x87O\x9b\x86M\x9f\x84O\x9f\x81K\x9e}H\x9ayD\x96r@\x8fn;\x84d3\x8erB\x8cqD\x8fwK\x9e\x87^\x9b\x83]\x90xT\x8a\x81V\xc1\xd9\x9ds\x83E\x97xJ\x9ftJ\x9d\x87V\x98\x91[\x93\x92Z\x94\x91\\\x9c\x8e]\x99\x8b\\\x91\x8b[\x8c\x88Y\x8f\x83Y\x8f\x83[\x8b\x88]\x85\x8c`\x8a\x97l\x87\x99o\x85\x96j{\x8eav\x87[s\x84Wn\x7fRhzJ\xb4\xc8\x97\xb8\xcc\x99\xbc\xd0\x9d\xbf\xd3\x9e\xbf\xd3\x9e\xbf\xd3\x9e\xc0\xd4\x9f\xc0\xd4\x9ffs=hu?ku@it\x82l=\x9b\x84Z\xa1\x8aa\x9c\x82]\x9f\x98j\xb8\xd8\x99p\x8aK\x9b\x86W\xa4\x83Z\x9e\x91d\x96\x95e\x8c\x93`\x8d\x8f`\x93\x87]\x94\x86_\x8e\x87]\x8c\x85[\x8e\x80Y\x8c~Y\x86~W\x7f\x80X\x80\x88a|\x8bd|\x88bs\x82Yn}Tl{PhwLcsFXh;Wi9Wi9Xj8Wi7Vh4Se1Rd0sm;rl:pj8oi5rh5si4wk7zm8qa-ra-vb-ze0ze0|e1\x84m7\x8dxA\x8b|E\x8e\x81J\x91}H\x8cs=\x89i6\x89e3\x86b.\x7f^)xZ&~d1\x82l:\x88tB\x92}N\x97\x80T\x99\x80W\x91\x8d]\xb4\xdb\x9am\x92O\x99\x8f^\x9f\x86]\x93\x8fb\x88\x8e`\x7f\x88[}\x82Y\x88~[\x8b|[\x87}Z\x85{W\x89uT\x86rO\x7fqL{qMxrRytVyrUsnNpkKplInjGhf@nlEjh?cd:`a5_`4\\]1WY*SU&{j\x83pF\x85rH\x86pIwtG\xb8\xda\x9ed\x8aMx~P{sN\x81{YwrRniIzmM\x81hJ\x80eGyfErhEmjGmhJqbKo`IeaDabBa`Bb]@dZ?cW=dVs_bfCbeFj]Jj[H_^@\\`?]^?^Z=^W=`TweAuc?hf=\xb6\xd3\x9bXy@glCmdCriLlaEf]@m^ArW:rU7jW6a[7Z`>Z_AbWEdUB]X;YX9ZW8[T7\\P6]O5]M4]K3\\H0[G/WE-UE,TD+SE*RG+RG+eS-dR,cQ+dR,eS-eS/cQ-aO+bO.dQ0eU3eU3eU3fW6k[:q^=oX6pV3pV1nV0nW.oX/oY0oY0o\\2n[1m[5n\\6o\\;m];k[:`^7\xb7\xd4\x9cSp8ac;h]=j_CeY?bX=fW:kP5iN0aQ0YU2SZ8SZ;[S@\\O$K="G< F;\x1fD;\x1eB;\x1eC<\x1fD= [Q-ZP,ZP,\\R.^T1_U1^T0\\R.]P.\\O-]N-]N/_N0aP4cR8dS7dS5bS4bR1_P/^P-]O,]O*]O*aT1_R/^Q1^Q1_Q6^S7^P6TT2\xc0\xd8\xa6Ia-VT.`R5]N7YM7]RE$5H(5H*=@+A<)D>&D?"B>!B? ?>\x1f=>\x1c:@\x1c9A\x1c3=\x182<\x17/;\x15-9\x15-9\x15-8\x16.9\x17.9\x19?I\'CN,1<\x1a=I%FR.KW1LX2\x86\x92lDN);C\x1eDH\'ON0IE,E=(NC1QI6IG0GG-==#\xb0\xb0\x94JI-A@"IE(KH)LH+JD*GA)F@*IA.LD1MD3GG-\xbf\xd2\xa58I\x1cKE%RB+I<+LC4HB2C<*G6$N=)A="9B#\x94\xa9\x88$8\x1c9<)85$>9#><#<< 8;\x1c8>\x1c6@\x1d.=\x16&6\x0f0B\x1aj\x7fV7L%)>\x17"3\x0f\'8\x16#4\x14->\x1e\x9e\xab\x8d\xc6\xd3\xb5\xc7\xd6\xb5\xc4\xd4\xb0\xbe\xce\xaa\xbe\xd0\xaa\xaf\xbf\x9a\xd6\xe6\xbf\x8c\x9bt]iE;B!49\x1b<:#GB.MF4GB/EG1DH/=A(\xb5\xb9\x9eJM2AA%CC\'CB&DC\'EA(C>(D=*E>,HA1KB3GG/\xc1\xd3\xa98I\x1cID&O@+E8(E=0@=.=7)F6\'E8%:8\x1f;F(\xa1\xb9\x99,@$7<(41 85"99!69\x1e49\x1b1<\x1a/?\x1a(=\x16 7\r\x83\x9cr\xe5\xff\xd7\xa9\xc4\x9b\xb7\xd0\xa9\xba\xd0\xac\xc2\xd7\xb6\xba\xcd\xaf\xbe\xd1\xb50:!6C)-<\x1f1@!.?\x1b.C\x1cEW1\xaf\xc1\x9b=O)0>\x1a4=\x1e@F*BA,?<+?7*52#=@+#:: <<"><%@;\'?:\'@9)B:-F<0CB-\xc0\xd5\xaa6I\x1bEB#I=\'?3%=7+75)40$?3%9-\x1d00\x189F*\xaf\xc7\xa7/C(36#/)\x1b30\x1f43\x1f24\x1e.4\x1a*8\x17*;\x17";\x13\x1b6\r\x195\x0c\x99\xb5\x8c!=\x15(B\x1d\x1e6\x16$7\x1b\x1c,\x12\x1f/\x15:?+2:#,6\x1d7D(3B!&7\x13=N*\xc9\xdb\xb5"2\r*8\x15?H+EI054".(\x1a80%@:.38$2<$3=$\xbd\xc5\xad4*<;\'\xbe\xd3\xaa0F\x18?<\x1dC8$7-!40%..$/-!7-#.%\x16\')\x133@&\xb4\xcb\xae\'; (+\x1a*$\x160*\x1c3-\x1d0/\x1b*/\x18\'2\x14&7\x15\x1f8\x11\x1a5\x0c$@\x18\xc2\xde\xb6\x160\x0b\x1c4\x12\x12%\t\x1b)\x10\x1a&\x10$-\x1a)\'\x18:9\';=(37 1:\x1d3>\x1e7E"\xb2\xc0\x9d:E#3:\x1b48\x1f87"3-\x1f6,"?2,:1(-2\x1e(4\x1c+7\x1f\xbe\xc8\xaf4="39\x1f48\x1f46\x1e66\x1e77\x1f94!83 7/"8.":-$43\x1f\xba\xd1\xa7)@\x1276\x18=5"3+ .+"*+#--#3) -#\x17(*\x15-9!\xb3\xca\xae\x1e0\x16"#\x13,$\x190#\x1a2(\x1e1+\x1d)+\x16&,\x12#0\x12\x1f3\x10\x1b1\x0b\x0b#\x00\xc8\xde\xba\x10#\x03\'6\x19%/\x17*.\x1d$&\x18*+\x1d7-#7/"4.\x1e.,\x1777\x1f87$*&\r \x04\xc4\xc5\xa6??#:8\x1f=7!;/\x1f2!\x196!\x1cA\'(:($\'+\x1a\x1e.\x14&4\x1b\xc6\xd3\xb9)3\x1a,5\x1a.4\x1a/3\x1a33\x1b0.\x17/*\x173,\x1a9/#>2&;,%.-\x19\xbb\xd2\xa8$>\x11/2\x1381\x1e,(\x1d\'\'\x1f&)",-\'-$\x1d2*\x1f/0 %.\x19\xbc\xd0\xb7#2\x1b*(\x194\'\x1f0\x1b\x184\x1f\x1c2!\x1a(\x1e\x12!\x1e\r\x1e"\x0b\x1c\'\t\x19&\x08\x14!\x03\xc4\xcf\xb1\x16\x1b\x04!#\x0e*$\x18\'\x1d\x14,\x1a\x165"\x1e8 \x1cB.\'@.$&\x19\t.#\x1171\x1973\x18\xcc\xcb\xad,(\r)#\x0b/\'\x125(\x185!\x1a<"!@#%0\x1d\x19#\'\x16\x1a*\x10$2\x19\xc7\xd5\xbb%/\x16(1\x16(.\x14(,\x1311\x19,*\x13+&\x131*\x18=1%A5)>/(/.\x1a\xbe\xd4\xad&@\x1303\x1492\x1f+\'\x1c%%\x1d"\'!*-&) \x191)\x1e./\x1f )\x14\xbf\xd3\xba(7 -+\x1e3$\x1d/\x17\x155\x1b\x1a1\x1e\x1a\'\x1a\x11\x1f\x19\t\x1b\x1d\x07\x19"\x07\x17"\x04\x16!\x03\xbe\xc7\xac\x1d!\n(\'\x152(\x1e*\x1b\x14*\x15\x142\x1a\x1a' -SkyServer_ImageCutoutImagePy3 = b'\x1c\x1b\x16\x17\x18\x12\x1c\x1f\x16\x15\x1d\x12\x0e\x19\x0b\x0e\x1c\r\n\x18\tz\x8b{\x10\x1c\x0e\x0e\x19\x0b\x17\x1e\x16\x1e!\x1a!\x1d\x1c%\x1c\x1d\x1f\x13\x17\x0b\x06\x00\x13\x1d\x02\t\x1d\x00\x07\x18\x00\x9f\xaf\x94\r\x19\x03\x17"\x11\x19\x1f\x13\x19\x1c\x13\x1c\x1d\x17"\x1f\x18$\x1f\x19&\x1f\x15&\x1c\x12&\x1d\x0e+\x1f\x11%\'\x11\x8b\xa7\x81\x02 \x00\x1f#\x0c-\'\x17\' \x16\'%\x19\')\x1c,-\x1b=2 4-\x13#\'\x06\x19)\x02\x97\xb3\x8a\x13-\x06&1\x1172\x1cB,!B\'\x1e=%\x1b8%\x164\'\x162*\x13.*\x11+*\x0e,,\x10\x84\x84h-)\x0e4.\x14,$\x0f7,\x18/ \r<-\x1a\x15\x16\x0e\n\x0c\x01\x11\x17\x0b\x0e\x16\x07\x0b\x18\x07\r\x1c\t\x17(\x16\xb8\xc9\xb7\x19&\x15\x18#\x13\x1a"\x15\x1c\x1f\x16\x1a\x17\x12\x18\x10\x0e\x1e\x12\x12%\x1e\x16\x16 \x05\x16\'\x03\x1a+\x0b\xb6\xc4\xaa\x1f+\x15#,\x1b\x1b!\x15\x17\x1a\x11 \x1f\x1a$!\x1a)"\x1a) \x17)\x1f\x15) \x11."\x12(*\x14\xa9\xc5\x9f\x186\x10(,\x15.(\x18\' \x16*(\x1c.0"76$;0\x1e:3\x1926\x15.>\x17\xac\xc8\x9f!;\x14,5\x1661\x1b?+ A)\x1f?)\x1c=,\x1c:/\x1d60\x181/\x16+.\x11-0\x13\xd4\xd7\xba,,\x10)(\x0c<6\x1e1)\x144)\x155*\x16\x1c \x11\x14\x1b\x0b#,\x1b\x15!\r\x17%\x0e\x12"\x08\x08\x1a\x02\xb7\xc9\xb1\x10\x1e\x07\x13\x1f\x0b\x16\x1d\x0b\x1c\x1e\x10#\x1f\x14\x1d\x14\r\x1e\x13\x0f,&\x1a\x17 \x05\x1b*\t$3\x14\xc1\xce\xb4\'0\x1b\'.\x1e $\x16\x1c\x1e\x13%%\x1d(%\x1e,%\x1d-$\x1b-#\x17.%\x162&\x16+-\x15\xb9\xd5\xaf!?\x19*.\x17.(\x18\'!\x15+)\x1d.0"54"9.\x1c<2\x199:\x1a7F\x1f\xb5\xd0\xa7&@\x19,5\x163.\x18;+\x1c>*\x1f>.\x1f>3!=6#:8\x1f26\x1b.5\x16\x1e%\x06\xd9\xe0\xc1\x1e%\x06$)\x0b7:\x1f33\x1975\x1e:5\x1f\x1d&\x11\x19#\x0b\x1c(\x10\x00\x0e\x00\x10 \x03 0\x13\x13&\x08\xc3\xd6\xb8\x13"\x05\x16#\t\x12\x1a\x02\x16\x19\x04)&\x15,$\x17&\x1c\x12("\x12\x1d#\x07\x1f-\x0c\'2\x14\xbf\xc9\xb0"*\x15\'+\x1a(*\x1d,,"+(\x1f,(\x1f0\' 0\'\x1e1\'\x1b2)\x186+\x19.0\x18\xb3\xcf\xa9\x1d;\x15)-\x142,\x1c/)\x1d1/"-/!.-\x1b>1\x1e=3\x1a67\x171@\x19\xaf\xca\xa1$=\x16/6\x1772\x1c:-\x1d=- ;/\x1f<5"<:#:="4=\x1e1<\x1c8F%\xb5\xc3\xa20;\x1b=H(.7\x1a;A%/3\x1a7;"\x10\x1d\x01\x10\x1f\x00\x19(\t\r\x1e\x00\x1d.\x0c\x1c0\x0b\x12&\x01\xc3\xd7\xb2\x1b+\x07!/\x0e\x1c%\x08\x16\x1a\x01" \x0b3,\x1a5,\x1b0)\x16\',\x0e+4\x15.7\x1a\xc6\xcb\xb4(+\x16-.\x1e00$86*0,!1* 1(\x1f3)\x1f6*\x1c8,\x1c:-\x1c22\x18\xb4\xd0\xa8\x1f>\x15.2\x1994!82$97*34&10\x1eD7$C7\x1f:9\x1a5A\x1b\xb2\xcb\xa3)?\x195:\x1c>8"=1!<0"90\x1f94 8:"8>"6A!3B!;L(\xac\xbd\x992C!:I(,;\x1c8E\'/< 8B\'\x12!\x00\x12#\x01\x16\'\x03\x1f0\x0c 5\x0e\n\x1f\x00\x13(\x01\xc3\xd5\xaf\r\x1d\x00\x13!\x00$-\x0e*/\x11(&\x0f71\x1bA9&5/\x19..\x1227\x197; \xcd\xd1\xba.0\x1b21\x1f1-!3/#6/%5-"5+!6, :.\x1e<1\x1f=0\x1d44\x18\xbb\xd7\xae&B\x1904\x19:3!60"97(78(96#D5"F7 >=\x1e+5-\x180.\x1525\x1888\x1e\xce\xcd\xb81.\x1b30\x1f2,\x1e3+\x1e;3(:0$8.";/!>2"@5#A4!88\x1c\xbb\xd6\xad\'C\x1a27\x19;4!60 ;8)<:+>;(F7$G9\x1f@=\x1e>F!\xbb\xd0\xa91B\x1e:: ?7"I=-H<,E<+D?+CC+AE*=F\':E#BR.\xb9\xca\xa6IZ63D EV40A\x1f2A"+:\x1b\x92\xa5\x85\xc3\xd6\xb6\xb7\xcb\xa8\xb9\xcd\xaa\xc3\xd4\xb0\xc3\xd4\xb0\xb0\xc1\x9d\xc9\xd9\xb5\x8d\x9bx2=\x1b\x19\x1f\x0303\x18+)\x1470\x1dJA091\x1e62\x1964\x1b53\x1c\xcd\xc8\xb22-\x1983 :3#<5%?6\'=4%=1#>2"A6$D9%C8":;\x1c\xb6\xd1\xa6&C\x177<\x1cD>(@9)B?.BA/E@-P?+N>%C> >D \xbc\xcd\xa94C"B@\'K@,NA1NA1NC1MG1KI0II-BG\'>E$\xa0\xab\x89\xc6\xd4\xb1\xbe\xcc\xa9\xbb\xcb\xa7\xc1\xcf\xae\xc6\xd4\xb3\xc1\xce\xb0\xc6\xd3\xb5 0\x15$4\x19\x16&\t!1\x14/>\x1f\x10\x1f\x002A"\xa4\xb1\x93\'2\x14(1\x14+/\x18/.\x192/\x1e81!<2&>5&C=\'/+\x12D?)\xb7\xb1\x9b5.\x1bE>,1(\x17G>-A8\'D<)F>+C;&?7">6\x1fC;$AD%\xb2\xcd\xa01N 6;\x1bE?\'F?-B<,GD3ID0RA-D4\x1bWP3DH%\x9e\xae\x8aJX7FB)L?,UC5WE7WG7TI5QI2OI/LK-LM.GK*\xb2\xb8\x96?F$?I\'EL-?F\'GM1BH,\x0e\x1c\x05$2\x19\x1f+\x13\x1d*\x10,9\x1f\x1e+\x11\x19#\n;E,,4\x1c.3\x1c02\x1d21\x1f52#:4(>6+?7*B;(60\x18<6 PJ42+\x18@9&H@-C;(C;(F>)H@+E?\'B:#A;!F?%BH&\xb4\xd1\xa15R"<@\x1fJB+JC0F@0KH5MF3[I5VD,NE(34\x12_mJ?J*HB*XH8VE5YF7ZI7YJ5UI3SI0RK/RM/@=\x1eonO>?\x1fDH\'AD%GJ-NQ6PS8\x19"\x0f*3 )0\x1e!)\x14,4\x1f08!%*\x14\x1d"\x0c.1\x1c/0\x1e./\x1d1/ 3/#70&:3+<4)5.\x1cA9$A9$&\x1e\tG?*H@+LF0B<&F@*HB*JD,HD)F@&EA&JD*EL*\xb8\xd5\xa5;X(CG$PI/OH5LG4PM:RJ7WB/eQ9WL0FF$BM+HQ2PH3[J:\\I:^K<_N<^O:\\M8YM5YO4ZQ4TM0E@ UP2XU6JF)SO2SO4XT9,0!(,\x1b-0\x1f47$25"14\x1f9;&>@+21\x1d21\x1f10\x1e2/ 3/#70&92*:3)5,\x1bIA,D<\'IA,NF1KC.>8"PJ2IC+JF+LH-JI-IH,IH*LK-GQ,\xb8\xd6\xa4>\\*IM(TM1RL6QJ8VQ>UM:VA.gQ:gY>__=IS1W^?_T@_N>bQAcSCdUBdU@aU=`T:`U9aV:lcDMD%kdGc\\?\\U9VQ4WQ7YS91/ $#\x110-\x1cA>+97"/-\x1842\x1d97":8#:7$96#96%:7(<8,>:/A;/F=,I>,?7"VN9=5\x1eG?(D>&YU:KG,KJ,ML.MN/NO0NO/PQ1JV0\xb4\xd5\xa0A_+MQ,XQ5VN9VO<[VBZO;lVAdN7gY>]Z9OV5PU7h[HiXHcVEdYGf[Gf[EdZAcY>cZ=d[>bY:`W8i`AYR5jcGYR6d]C`Y?94!0+\x182+\x1870\x1d:4\x1eA;#@:"3-\x15?9#>9#>9%<9&=:)@=.B>2E?1LC2C8&IA,JB-D<%PJ2[WWQ1qjM]V:leKg`FC;&?7"7/\x184,\x15@9\x1fNG-MF*D=!A: @: ?;"?:$?=(B?,DC1FC0I@/MB0SK6ME0UO7\\V>^Z?JF)SR4ST4UV6VZ9Y]Z`>Sb9\xb6\xd7\xa2Ig1Z^9d]@c[De_IleRi^Jv`Ks[CtdJkfHlqQ`cHm^Km\\LkfPhjRllRllPkjNjiKmjKojJpkKtoOmgGhcCrmOfaCniLkfI:6\x19=9\x1c>:\x1dD@#KF)C> >9\x1bE@"E@#E@#D@%EA(GC*JE/KI2OK2QF0^S=OD0SK8PI6VQ;MI._^@VW7WX9YY=[^A``D`cD_c@Yf:\xb9\xda\xa3Nl6^e;jdBlaEoeLtmSpfMr`H|hOrbHibEghIllPphSwo\\oq[mu]qv_qw]ptYpsTstTtuUyyWccAnnLttRlkLklLlmNmnO:E\x1b8C\x197?\x187?\x1a;A\x1d@D#CG&EF&ML.NM/LH+FB%GB%NI+VO2YP1ZP-cU8RG3SJ;meZMG7aaEPQ1W[:ccGZYEvsbd_KqkQfbJ"AK&DL\'GM+HL+:;\x1cCB$OL-VQ3VP0VM.WN-ZP-XL$TG$TG4bXLg^UPM>eeKkoNbfEUX=mk\\b^Sg_RskTzrMip=\xab\xcf\x92W}@hvAwsF\x7fmE\x90}U~pI|tMytN{sO\x7fpS\x81rU\x83uZ\x7f{^x\x82`t\x84gv\x88ry\x8ax~\x8f|\x83\x92{\x84\x92x\x81\x8er|\x87gx\x83a\x7f\x89f}\x89ez\x86bt\x82^p~ZjzUeuPbtN/<\x105B\x16=H\x1eCK$CK&CI%EI(HI*baCWS6PK-TM0[R3[P2]R2cY6\\P(gZ7k^K\\PD\\SJ\xb4\xb1\xa2\xed\xed\xd3\xff\xff\xe3\xf6\xfb\xdb\xe0\xe3\xc8\x8b\x89|\xc4\xc0\xb7\xd3\xcb\xc0\xc8\xc0\xab\xe9\xe1\xbds{J\xab\xce\x94\x82\xa8mr\x80M~zM\x8byQ\x93\x80V\x8b~T\x85~R\x81|R\x84|U\x89zY\x8c{]\x8d~a\x8a\x83f\x84\x89i\x80\x8do\x83\x92{\x8a\x99\x84\x93\xa1\x8a\x96\xa2\x8a\x92\x9c\x81\x8b\x94w\x84\x8dn\x80\x8ah~\x85c}\x87b}\x87b|\x88bz\x86`u\x84]q\x80Yn~W\xb6\xbf\x94\xbd\xc5\x9c\xc6\xcc\xa6\xc9\xcf\xa9\xc7\xcb\xa8\xc5\xc9\xa8\xc9\xca\xab\xcd\xcc\xae\x88\x84ge`CJC&PG*_T6eX8eX8h[8na7m_:qbMeYKnf[\xdb\xd8\xc9\xb0\xb0\x96fiJx}]rv]\x84\x84x\xe9\xe6\xdf\xed\xe6\xdc\xc8\xc1\xaf\xd1\xcb\xabnuI\xc3\xe6\xb0\xbf\xe4\xae\x87\x94f\x88\x83Y\x95\x84Y\x96\x83Y\x98\x8b^\x91\x8b[\x8a\x86W\x8c\x85Y\x91\x82[\x93\x80_\x92\x82a\x92\x87i\x8f\x90p\x8d\x96w\x94\xa1\x87\x9c\xa8\x90\xa5\xaf\x97\xa3\xad\x92\x9c\xa2\x86\x91\x98w\x8c\x91q\x8b\x91o\xc0\xc6\xa2\xc3\xc9\xa3\xc5\xcd\xa6\xc7\xcf\xa8\xc6\xd1\xa9\xc3\xcf\xa9\xc1\xcd\xa7\xbf\xcb\xa5AH\x1fFM$LP+LP+KL*JI*OK.TO2kdHWM2NC\'^P5l^Ak\\=eV7eW4uf;bT-m^Gxl^wod\xd1\xce\xbf\x96\x96|dgHpuWlpY\x92\x92\x88\xff\xff\xfa\xff\xfe\xf8\xb5\xae\x9e\xda\xd3\xb6\x86\x8dd\xcb\xed\xbb\xe2\xff\xd4\x94\xa1u\x93\x8ed\x9d\x8ca\x9c\x89^\x9e\x94c\x96\x90\\\x90\x8dX\x92\x8aY\x95\x84Y\x95\x7fX\x93\x7f^\x94\x85f\x95\x92q\x96\x99z\xa7\xad\x93\xab\xb4\x99\xb0\xb6\x9c\xaa\xb1\x92\x9f\xa2\x83\x95\x99v\x94\x95s\x96\x98s\x8a\x8ce\x8a\x8ce\x89\x8ef\x88\x8de\x84\x8bb~\x86_y\x81Zt\x7fWMO(PR+RR.QQ/PM.PM.UP3YR6VL1XM1aS8k\\?j[l\\Bn^Dl[?hW9kX8s`@{iCwi:\x7frFzlQ|o^\x94\x8b|\xe0\xdb\xc8\xb5\xb4\x98\x86\x8ai\xc7\xca\xab\xe2\xe4\xcc\x9f\x9d\x91\xff\xff\xfa\x9c\x95\x8f\xe3\xdb\xce\xbf\xb8\x9c\x95\x9bu\xbd\xd9\xb0\xdf\xfc\xd0\x95\x9dt\x9c\x94m\x9b\x8a_\xa4\x92b\x98\x8fX\x91\x8cR\x8d\x8bN\x93\x8aQ\x9a\x86S\x9a\x7fR\x97{T\x95}Y\x95\x87d\x95\x8fm\xa2\x9d}\xa4\xa1\x80\xa5\xa2\x7f\xa2\x9ey\x9c\x97q\x99\x91j\x9a\x93i\x9d\x96l\x93\x8c`\x91\x8d`\x91\x8ec\x90\x8db\x8f\x8dd\x8b\x8cb\x89\x8bc\x88\x8abZT2ZT2]T5`W8cZ=f[?eZ>eW\x83vJ\x8e\x7fb\x7fr_\x9a\x91\x80\x9f\x9a\x84\x8d\x8dk\xa9\xa9\x83\x9d\x9dy\xff\xff\xe4\xca\xc5\xb2\xff\xfc\xef\xa7\x9b\x8d\x9f\x93}\xd4\xc8\xa2\xe8\xe7\xb9\xd0\xe6\xb7\xdd\xf3\xc5\x90\x93h\x9a\x8dc\xa1\x8ca\x98\x84R\x9e\x95\\\x8c\x88K\x91\x90P\x9b\x93U\x98\x83L\xa4\x86T\xa5\x83V\x97wN\x9b\x84Z\x93\x82W\x92\x83X\x93\x86Z\x95\x86[\x96\x85Y\x94\x83W\x94\x83U\x96\x84V\x97\x87V\x9f\x8f^\xa2\x94c\xa2\x96f\x9d\x92d\x98\x90b\x9b\x92g\x9d\x96j\x9d\x96jbU3fY7k^h[;h[;l]>paBveGwgF{hG~lH\x82nI\x85rJ\x88uK\x88wK\x8a|K\x8b~R\x89{^\x8b\x7fi\xb7\xaf\x9c\xe7\xe1\xc9\xc0\xbd\x9a\xa1\x9fv\xda\xd8\xaf\xf4\xf1\xce\xb9\xb2\x98\xf1\xe6\xd2\xb0\xa1\x8c\xa0\x91r\xaa\x9co\xe7\xe4\xaf\xd8\xe9\xb5\xf4\xff\xd4\xa8\xa5x\x95\x86[\x9b\x84X\x94~M\x93\x88R\x9c\x98[\x88\x87G\x9a\x92T\xa1\x8aT\xaa\x8aW\xa8\x83V\x9czL\x96{L\x9b\x86W\x9a\x85V\x9a\x85V\x9a\x85V\x99\x83R\x96\x80O\x94~M\x96\x7fM\x96\x80N\xa2\x8cZ\xa7\x93a\xa8\x96f\xa5\x94f\xa1\x93f\xa1\x94h\xa1\x96i\x9f\x94ghZ7l^;pb?qbApa@pa@tdCwgE\x7flK\x80nJ\x81oI\x84rJ\x87tJ\x89vK\x8byK\x8b{J\x89{J\x8e\x81U\x8c~a\x96\x8at\x95\x8av\xe3\xdd\xc3\xf9\xf5\xcf\xff\xff\xd4\xfb\xf9\xc9\xa7\xa2x\xa8\x9d}\xfd\xee\xd1\xa8\x95u\xa4\x91g\xa0\x8bT\xc0\xb6x\xee\xf9\xbd\xf8\xff\xcd\xab\xa5s\x91\x7fQ\xa9\x8ea\xaf\x97g\x9b\x8f[\xaa\xa5m\x9c\x9a_\x9c\x93Z\xa5\x8eX\xae\x8e[\xc2\x9el\xb1\x8d[\x95zE\xa0\x89S\x9e\x87S\xa0\x86S\x9e\x84Q\x9b\x80K\x97|G\x94yD\x96xD\x94yD\xa0\x86S\xa6\x8f[\xab\x95c\xaa\x96d\xa7\x95g\xa5\x94h\xa0\x92e\x9c\x8eak]8oa\x95t>\x95v?\x9d~H\xa3\x88S\xaa\x8e\\\xa9\x92`\xa8\x92c\xa5\x90c\x9e\x8c^\x97\x84Yn`9qcxkA|oE\x81rI\x84uL\x86wP\x88yP\x8d~S\x8d\x7fR\x8e}O\x8d}L\x8d}I\x8c}F\x8b|C\x89|E\x8a~J\x86{N\xa3\x98z\xf8\xec\xd4\x9f\x95|\x95\x8ck\x8c\x86T\x84~B\x8f\x85G\xb2\xa4g\xa7\x92[\xae\x95_\xb2\x93Z\xbb\x9aU\xc4\xa2K\xb2\x9b>\xcd\xcbv\x9f\xa0T\xbe\xacl\xff\xe1\xab\xcd\xab}\xcf\xb2\x88\xf4\xe2\xba\xde\xd5\xac\xfa\xf6\xc9\xbe\xb3\x86\xb6\x9ep\xdb\xbb\x8a\xf3\xce\x97\xa8\x86G\xa2\x85?\xa3\x88A\xa3\x81B\xa5~C\xa2{@\x9ex:\x9as8\x98r4\x9bs8\x9bt9\x9ez@\xa0}E\xa0\x7fJ\x9e~K\x9b\x7fO\x9b\x80Q\x94|N\x8cuIzmA{nB\x7frF\x83vJ\x88{O\x8c\x7fS\x8e\x81W\x8f\x82V\x94\x86Y\x94\x86W\x93\x85T\x91\x84P\x8f\x82K\x8e\x82H\x8d\x81E\x8c\x80F\x82vB\x96\x8a`\xbc\xb1\x95\xfb\xef\xd7\x9b\x91x\x8d\x85a\x9f\x98d\xa4\x9c^\xad\xa2`\x99\x8aG\xa9\x93W\xc5\xa8n\xbc\x9aZ\xc5\xa1U\xdd\xb9[\xb7\x9e:\xb4\xb0U\xa9\xa8V\xc3\xb0k\xff\xe2\xa9\xca\xa5x\xc3\xa3|\xef\xdd\xb7\xcf\xc6\x9f\xe9\xe4\xbc\xd4\xcb\xa0\xb7\xa0t\xfb\xdb\xaa\xd2\xadv\xa9\x87G\xab\x8fF\xa2\x86<\xa5\x82@\xa7\x7fB\xa3{>\x9ev8\x9ar5\x98p2\x9ap4\x9ct7\x9cu<\x9au>\x93r=\x8en;\x8eo@\x8fsD\x8cqD\x85m?xvEywF}{J\x82\x80O\x88\x86U\x8c\x8aY\x8e\x8c[\x8e\x8cY\x92\x90]\x91\x90Z\x8f\x8eV\x8e\x8dT\x8b\x8bO\x8a\x8aL\x88\x88H\x87\x87I\x8f\x8cY\x81~S\x96\x96r\xf6\xfc\xda\xdd\xe8\xc0\xec\xfa\xc9\xfe\xff\xcc\xfa\xff\xbd\xed\xf1\xa5\xb6\xb2h\xab\xa1Z\xbc\xabe\xcd\xban\xc3\xafY\xc4\xb0M\xcd\xc4[\xba\xc4a\xc3\xc9o\xc4\xb0g\xfc\xdc\x9f\xde\xbe\x8b\xdc\xc1\x94\xf9\xea\xbf\xc3\xba\x91\xe0\xd9\xaf\xe9\xdd\xb3\xad\x9cn\xff\xf4\xbf\xb6\x9e`\xb4\xa0Y\xae\xa1R\x94\x878\x9f\x89@\xa1\x86A\x9c\x81<\x96{6\x91v3\x8ft1\x8eu2\x90w5\x8ev8\x88t7\x80m3wf.wh1yl7ym9tj7\xb3\xc7\x8c\xbb\xcf\x94\xc2\xd6\x9b\xc2\xd6\x9b\xbd\xd3\x95\xba\xd0\x92\xbd\xd3\x93\xc2\xd8\x98\xbc\xd2\x92\xbc\xd2\x91\xbd\xd3\x92\xbe\xd5\x91\xbf\xd6\x92\xbf\xd6\x90\xbe\xd5\x8f\xbf\xd4\x93\xc8\xd2\x9e\xbf\xcc\x9e\xb6\xd1\x9e\xd7\xfe\xc5\xa8\xdb\x99\xac\xe6\x9c\xa2\xd9\x86\xae\xe0\x89\xb1\xd5}\xcc\xe4\x8c\xc5\xd3|\xc9\xd3|\xc7\xd0u\xcc\xd7w\xb6\xc9`\xbb\xd7i\xb6\xddp\xc3\xdcw\xc6\xb5e\xfa\xd9\x96\xf9\xe4\xa5\xe2\xd6\x9c\xfa\xf3\xbd\xd6\xd0\x9e\xda\xcd\xa0\xff\xf4\xc4\xe1\xdc\xa4\xf8\xf7\xb7\xd2\xd2\x8a\xcc\xd1\x81\xbf\xcfw\xcb\xdb\x82\xd3\xd7\x83\xd5\xd4\x82\xd4\xd3\x83\xd0\xd1\x81\xce\xd1\x82\xcd\xd1\x84\xcb\xd2\x85\xca\xd2\x87\xc9\xd3\x8c\xc8\xd5\x8f\xc7\xd6\x91\xc4\xd5\x91\xc0\xd3\x91\xbe\xd3\x92\xbf\xd3\x94\xbf\xd5\x95y\x8dR~\x92W\x82\x96[\x81\x95X}\x93Uz\x90P|\x92R~\x94Sx\x8dLw\x8dLx\x8fKy\x90L{\x92N}\x94P~\x95O\x80\x94U\x86\x92`\x8c\x9en\x95\xb8~\xbf\xf4\xb0\x94\xd7\x8a\x8f\xda\x85|\xc5k\x87\xc8j\x8a\xbb_\x98\xbd`\xa8\xc1e\xbf\xd2v\xbf\xd4u\xad\xc7d\xa1\xc4^\xac\xd7l\xad\xe2p\xba\xdbr\xca\xb8f\xe5\xc3|\xff\xef\xab\xf1\xe9\xa8\xc9\xc3\x87\xb9\xb0y\xc9\xb5\x83\xf2\xe1\xad\xe5\xdf\xa3\xc5\xc6\x84\xb0\xb2i\xa6\xad]\x96\xaaS\x85\x98?{\x80,\x7f~,|~+{|,z}.w{.rz/nx/bo)_n)\\m)Wj(Ti(Ti(Uk+Xn.\x80~M\x80~M\x80~M\x7f}J\x7f}J~}G~}G~}E}{B}{B}{@~|A\x81\x7fB\x85\x83F\x88\x86I\x8a\x89Py{J\x86\x90^\x98\xb4z\xc5\xf2\xaf\xa6\xe0\x94\x9d\xe1\x8e\x8c\xccu\x9a\xd2y\xa4\xcfv\xae\xcct\xb7\xcbt\xb7\xc5n\xb1\xc1h\xb7\xces\xbc\xd9|\xb3\xd9t\xb0\xe1t\xb7\xd6o\xce\xb9j\xd0\xa9d\xff\xf4\xb0\xff\xff\xbe\xb2\xa7i\xba\xaao\xc4\xa6r\xff\xe1\xad\xff\xff\xc5\xc1\xb3t\xaa\x9bX\x99\x8fG\x96\x97I\x83\x822\x89w-\x8ft-\x8du-\x8fw1\x90y5\x8bv3\x84q/~l,ue)sd)nb(i^&e\\%f]&ha+le/\x8b\x7fU\x89}S\x87|O\x89~Q\x8c\x81S\x90\x84T\x8f\x83S\x8f\x81P\x90\x83O\x8e\x81L\x8f\x80I\x8f\x80I\x91\x82I\x97\x86N\x9c\x8bS\x9c\x8fZ\x96\x8e]\x9a\x9cj\xa3\xb4}\xc6\xe6\xa5\xb3\xe3\x99\xae\xe5\x94\xa3\xd9\x85\xb4\xe4\x8e\xc1\xe3\x8f\xb2\xc8v\xb9\xc6v\xe7\xf0\xa1\xcb\xd7\x85\xc4\xd3\x80\xb8\xd0z\xc3\xe4\x89\xb5\xe2}\xb6\xd0o\xca\xb0e\xc0\x95Q\xee\xd3\x8e\xfa\xe7\xa3\xb1\x9e\\\xbd\xa5g\xb8\x91Z\xe2\xb9\x83\xff\xe4\xab\xae\x91U\x9f\x7fB\x94v6\x8cy4\x86s.\x85d!\x88b!\x8be&\x8fi*\x91m/\x8fk/\x86c)}]${\\&z\\(x\\*vZ(tZ)u[*v^.ya1\x81yR~vO}vL\x81zP\x89\x80U\x91\x86Y\x91\x86X\x91\x85U\x8a|K\x88{G\x87wC\x87v@\x88w?\x8e{C\x93\x80F\x94\x83M\x9e\x8e]\x9c\x95a\x9b\xa3j\xb6\xcb\x8a\xb1\xd2\x8b\xa5\xce\x82\x94\xbdn\xa3\xc6v\xa8\xbfq\xb3\xbfu\xb8\xbbt\xe5\xe6\xa0\xab\xb1k\xb7\xc1z\xae\xbbs\xbf\xd8\x87\xba\xdf\x82\xb5\xc8n\xc1\xa0Y\xbc\x8bI\xba\x97S\xba\x9fX\xa7\x8cG\xa4\x82B\xb3\x82H\xacx?\xaa}D\x8d`\'\x98g/\x9cm5}V\x1byW\x1a\x82^ \x84`"\x88d(\x8di-\x91m3\x90k4\x89f0\x82a,}\\\'\x7f_,\x82b1\x83d5\x83d6\x83d6\x85f:\x87h<\x81}X\x80|V\x81}W\x86\x83Z\x90\x89_\x95\x8eb\x97\x8fa\x98\x8d_\x8d\x81Q\x8c\x7fK\x8b{G\x8ayC\x8cyA\x8f|D\x94\x7fF\x96\x81J\x9d\x87U\x9f\x8f\\\xa4\x9fg\xbb\xc4\x85\xc0\xd4\x8f\xa7\xc1x\x8a\xa4[\x8f\xa3[\x9f\xaae\xb6\xb9v\xbf\xbb|\xeb\xe5\xa9\xa8\xa6i\x9e\x9fc\x8a\x90T\xa9\xb8s\xbc\xd9\x85\xb0\xbch\xb9\x92O\xc2\x8bK\xa0x3\x91p)\xa3\x82;\x92k&\xacv8\x9db(\x86R\x18\x90]$\x9c`,\xa0f4\x7fP\x1c\x86]\'xU\x1dyY\x1e~^%\x82b)\x83b,\x83b,\x80a+~^+~^-\x81c1\x85f7\x88i;\x89j<\x89j>\x89j>\x8ak?}\x7fZ~\x80[\x81\x81[\x85\x86^\x8c\x89`\x8e\x8b`\x8f\x8b\\\x90\x8aZ\x8b\x81P\x8a~J\x8b{G\x8ayC\x8dzB\x8f|D\x93~E\x96\x80G\xa2\x89S\xa8\x91]\xb0\xa1h\xc2\xbe\x81\xcd\xd2\x90\xaf\xbav\x93\x9eZ\x9b\xa2_\xb4\xb4v\x9d\x97]\xb6\xabu\xff\xfc\xc8\xfe\xf4\xc1\xb2\xacz\x8e\x8c[\xbb\xc3\x88\xc2\xd8\x8d\xa9\xad`\xaa|>\xb8z?\x9eq.\x8dh!\x99w-\x8fg\x1f\x93^\x1c\xa3i*\x8f[\x1f\x9ag.\x9b_-\x95[-\x89Y)\x8ab/y[%xb)|f-|f-|c-{b,}c0\x80f3\x86l;\x88m>\x8bpA\x8cqD\x8cqD\x8cqF\x8drG\x8ftI\x82\x7f^\x85\x82_\x88\x84a\x8a\x86`\x8c\x87a\x8d\x86\\\x8c\x83X\x8f\x82U\x89{L\x8byG\x8bwE\x8dvB\x90vC\x92yA\x95zC\x99|D\xa0\x81K\xa5\x87Q\xa5\x90W\xb1\xa2g\xbe\xb8z\xa6\xa2c\x98\x94U\xa9\xa3g\xb8\xadu\xba\xadx\xb5\xa3s\xa9\x97i\xed\xdc\xb0\xff\xfc\xd2\xff\xff\xd8\xf0\xf0\xbe\xca\xd9\x94\xa1\x9eW\x98e,\x9d\\$\x96f&\x88c\x1c\x7f]\x13\x82\\\x13\x8d[\x16\xacv6\x97i+\x98k0\xa5q?\x9fl?\x9bpC|Z,\x7fe4\x80k6\x84o:\x81l7{e3zc1\x7fg7\x84l<\x86n@\x87oA\x89nA\x88mB\x86k@\x87lA\x89nC\x8bpE\x89{^\x89|\\\x8c}^\x91\x81_\x95\x82a\x95\x81\\\x94~W\x93zQ\x8ftI\x91rD\x95tG\x9ezJ\x9fyJ\x9evC\x9dsA\x9evB\x9e{E\x9e\x7fH\xa1\x87N\xa1\x8dR\x9b\x8bO\x95\x86K\x9c\x8dR\xa7\x98_\xaf\x9bh\xc1\xad{\xb8\xa1u\xa6\x8ff\xa3\x8df\x94\x82^\x97\x89f\xb5\xb1\x84\xf7\xff\xc2\x92\x8dK\x99d0\x98U!\x95e\'\x8ci%\x86h\x1f\x89i \x9ao*\x9dq0\x9ax8\x9c|A\xa1yF\x9dwH\x92uI\x8duG\x8bsE\x8evH\x8duG\x8arD\x89qC\x8cqD\x8aoB\x87l?\x8frH\x89lB\x87g>\x87g@\x8bkD\x8dmF\x8ajC\x86f?\x8a{\\\x8c}^\x8f\x7f^\x90\x80^\x92\x80\\\x91}X\x92{R\x92yP\x92uK\x92sE\x97uG\x9f{K\xa5}L\xa3yI\xa0vD\xa0xD\x9cyC\x98wA\x97zB\x9a\x80G\x9a\x81H\x94~D\x94}G\x98\x81M\xa4\x8d[\x9a\x7fR\xa0\x87^\xb5\x9bv\xb2\x9av\x9e\x87g\x8dzZ\x82yP\xf5\xfd\xc2\x90\x8aL\x9ad5\x98V&\x94g,\x8eo.\x89s*\x8du-\x9ay6\x9ax8\x94|<\x91}B\x95yG\x94yJ\x8d|N\x8a}P\x8f~R\x92\x7fT\x91~S\x8cyN\x89vK\x89vK\x89tI\x84oD\x82kA\x7fh>~e=\x7ff=\x81h?\x82i@\x80g>}d;|xS\x7f{U\x83~X\x84|U\x81xO\x81uK\x82sH\x86uI\x8auH\x88rA\x8crA\x94xF\x9c~J\x9d}J\x9c}G\x9d~H\x9e\x7fH\x98yB\x96w@\x9a}E\x9f\x82J\x9c\x7fG\x96xD\x90tB\x95yI\x9f\x82V\xd7\xbc\x91\xf0\xd7\xaf\xbc\xa4\x80\x90yY\x98\x81b\x9b\x8fg\xe8\xf1\xb8\x88\x86K\x97g9\x97Z-\x95n7\x91y;\x8d~;\x91\x82?\x9d\x85G\x9a\x82F\x8d\x81E\x89~F\x8d{I\x8c~O\x89\x85V\x8a\x89[\x91\x8ea\x94\x8fe\x91\x8cb\x8a\x85[\x85\x81T\x83\x7fR\x81zN|uI{tH{tF{rGzrDwoAtl>qi;og9syMw}Q}\x80S{~QwyJvuE{wG\x80zH\x90\x86S\x89}I\x84wB\x89zC\x8f|D\x8f|D\x90{B\x92|C\x9f\x86N\x9b\x81H\x9b}G\xa0\x81J\xa4\x85N\xa2\x83M\x9bzG\x94tC\x90qBz^/\x98}R\xdd\xc4\x9b\xff\xf7\xd1\xff\xfc\xda\xff\xfb\xdc\xff\xf4\xcc\xd3\xe2\xa9|\x82F\x95l>\x99d:\x98xE\x95\x85J\x92\x8aL\x95\x8dO\xa0\x8dU\x9a\x89S\x8e\x87P\x89\x83O\x8c\x7fR\x8e\x83V\x8b\x8a\\\x89\x8fa\x91\x98l\x92\x9bp\x8f\x96l\x86\x8fb\x80\x87[}\x84Xx\x7fSrzKgo@iq@lrBkqAhn>el9bi6bi6\xb6\xc7\x93\xbb\xcc\x98\xc0\xce\x99\xbf\xcc\x97\xbd\xc7\x92\xbd\xc6\x8f\xc2\xc9\x93\xc9\xce\x96\x94\x97^\x8b\x8cR\x81\x81E~|A\x7fy=}w;\x80w<\x84z?\x98\x87O\x9b\x86M\x9f\x84O\x9f\x81K\x9e}H\x9ayD\x96r@\x8fn;\x84d3\x8erB\x8cqD\x8fwK\x9e\x87^\x9b\x83]\x90xT\x8a\x81V\xc1\xd9\x9ds\x83E\x97xJ\x9ftJ\x9d\x87V\x98\x91[\x93\x92Z\x94\x91\\\x9c\x8e]\x99\x8b\\\x91\x8b[\x8c\x88Y\x8f\x83Y\x8f\x83[\x8b\x88]\x85\x8c`\x8a\x97l\x87\x99o\x85\x96j{\x8eav\x87[s\x84Wn\x7fRhzJ\xb4\xc8\x97\xb8\xcc\x99\xbc\xd0\x9d\xbf\xd3\x9e\xbf\xd3\x9e\xbf\xd3\x9e\xc0\xd4\x9f\xc0\xd4\x9ffs=hu?ku@it\x82l=\x9b\x84Z\xa1\x8aa\x9c\x82]\x9f\x98j\xb8\xd8\x99p\x8aK\x9b\x86W\xa4\x83Z\x9e\x91d\x96\x95e\x8c\x93`\x8d\x8f`\x93\x87]\x94\x86_\x8e\x87]\x8c\x85[\x8e\x80Y\x8c~Y\x86~W\x7f\x80X\x80\x88a|\x8bd|\x88bs\x82Yn}Tl{PhwLcsFXh;Wi9Wi9Xj8Wi7Vh4Se1Rd0sm;rl:pj8oi5rh5si4wk7zm8qa-ra-vb-ze0ze0|e1\x84m7\x8dxA\x8b|E\x8e\x81J\x91}H\x8cs=\x89i6\x89e3\x86b.\x7f^)xZ&~d1\x82l:\x88tB\x92}N\x97\x80T\x99\x80W\x91\x8d]\xb4\xdb\x9am\x92O\x99\x8f^\x9f\x86]\x93\x8fb\x88\x8e`\x7f\x88[}\x82Y\x88~[\x8b|[\x87}Z\x85{W\x89uT\x86rO\x7fqL{qMxrRytVyrUsnNpkKplInjGhf@nlEjh?cd:`a5_`4\\]1WY*SU&{j\x83pF\x85rH\x86pIwtG\xb8\xda\x9ed\x8aMx~P{sN\x81{YwrRniIzmM\x81hJ\x80eGyfErhEmjGmhJqbKo`IeaDabBa`Bb]@dZ?cW=dVs_bfCbeFj]Jj[H_^@\\`?]^?^Z=^W=`TweAuc?hf=\xb6\xd3\x9bXy@glCmdCriLlaEf]@m^ArW:rU7jW6a[7Z`>Z_AbWEdUB]X;YX9ZW8[T7\\P6]O5]M4]K3\\H0[G/WE-UE,TD+SE*RG+RG+eS-dR,cQ+dR,eS-eS/cQ-aO+bO.dQ0eU3eU3eU3fW6k[:q^=oX6pV3pV1nV0nW.oX/oY0oY0o\\2n[1m[5n\\6o\\;m];k[:`^7\xb7\xd4\x9cSp8ac;h]=j_CeY?bX=fW:kP5iN0aQ0YU2SZ8SZ;[S@\\O$K="G< F;\x1fD;\x1eB;\x1eC<\x1fD= [Q-ZP,ZP,\\R.^T1_U1^T0\\R.]P.\\O-]N-]N/_N0aP4cR8dS7dS5bS4bR1_P/^P-]O,]O*]O*aT1_R/^Q1^Q1_Q6^S7^P6TT2\xc0\xd8\xa6Ia-VT.`R5]N7YM7]RE$5H(5H*=@+A<)D>&D?"B>!B? ?>\x1f=>\x1c:@\x1c9A\x1c3=\x182<\x17/;\x15-9\x15-9\x15-8\x16.9\x17.9\x19?I\'CN,1<\x1a=I%FR.KW1LX2\x86\x92lDN);C\x1eDH\'ON0IE,E=(NC1QI6IG0GG-==#\xb0\xb0\x94JI-A@"IE(KH)LH+JD*GA)F@*IA.LD1MD3GG-\xbf\xd2\xa58I\x1cKE%RB+I<+LC4HB2C<*G6$N=)A="9B#\x94\xa9\x88$8\x1c9<)85$>9#><#<< 8;\x1c8>\x1c6@\x1d.=\x16&6\x0f0B\x1aj\x7fV7L%)>\x17"3\x0f\'8\x16#4\x14->\x1e\x9e\xab\x8d\xc6\xd3\xb5\xc7\xd6\xb5\xc4\xd4\xb0\xbe\xce\xaa\xbe\xd0\xaa\xaf\xbf\x9a\xd6\xe6\xbf\x8c\x9bt]iE;B!49\x1b<:#GB.MF4GB/EG1DH/=A(\xb5\xb9\x9eJM2AA%CC\'CB&DC\'EA(C>(D=*E>,HA1KB3GG/\xc1\xd3\xa98I\x1cID&O@+E8(E=0@=.=7)F6\'E8%:8\x1f;F(\xa1\xb9\x99,@$7<(41 85"99!69\x1e49\x1b1<\x1a/?\x1a(=\x16 7\r\x83\x9cr\xe5\xff\xd7\xa9\xc4\x9b\xb7\xd0\xa9\xba\xd0\xac\xc2\xd7\xb6\xba\xcd\xaf\xbe\xd1\xb50:!6C)-<\x1f1@!.?\x1b.C\x1cEW1\xaf\xc1\x9b=O)0>\x1a4=\x1e@F*BA,?<+?7*52#=@+#:: <<"><%@;\'?:\'@9)B:-F<0CB-\xc0\xd5\xaa6I\x1bEB#I=\'?3%=7+75)40$?3%9-\x1d00\x189F*\xaf\xc7\xa7/C(36#/)\x1b30\x1f43\x1f24\x1e.4\x1a*8\x17*;\x17";\x13\x1b6\r\x195\x0c\x99\xb5\x8c!=\x15(B\x1d\x1e6\x16$7\x1b\x1c,\x12\x1f/\x15:?+2:#,6\x1d7D(3B!&7\x13=N*\xc9\xdb\xb5"2\r*8\x15?H+EI054".(\x1a80%@:.38$2<$3=$\xbd\xc5\xad4*<;\'\xbe\xd3\xaa0F\x18?<\x1dC8$7-!40%..$/-!7-#.%\x16\')\x133@&\xb4\xcb\xae\'; (+\x1a*$\x160*\x1c3-\x1d0/\x1b*/\x18\'2\x14&7\x15\x1f8\x11\x1a5\x0c$@\x18\xc2\xde\xb6\x160\x0b\x1c4\x12\x12%\t\x1b)\x10\x1a&\x10$-\x1a)\'\x18:9\';=(37 1:\x1d3>\x1e7E"\xb2\xc0\x9d:E#3:\x1b48\x1f87"3-\x1f6,"?2,:1(-2\x1e(4\x1c+7\x1f\xbe\xc8\xaf4="39\x1f48\x1f46\x1e66\x1e77\x1f94!83 7/"8.":-$43\x1f\xba\xd1\xa7)@\x1276\x18=5"3+ .+"*+#--#3) -#\x17(*\x15-9!\xb3\xca\xae\x1e0\x16"#\x13,$\x190#\x1a2(\x1e1+\x1d)+\x16&,\x12#0\x12\x1f3\x10\x1b1\x0b\x0b#\x00\xc8\xde\xba\x10#\x03\'6\x19%/\x17*.\x1d$&\x18*+\x1d7-#7/"4.\x1e.,\x1777\x1f87$*&\r \x04\xc4\xc5\xa6??#:8\x1f=7!;/\x1f2!\x196!\x1cA\'(:($\'+\x1a\x1e.\x14&4\x1b\xc6\xd3\xb9)3\x1a,5\x1a.4\x1a/3\x1a33\x1b0.\x17/*\x173,\x1a9/#>2&;,%.-\x19\xbb\xd2\xa8$>\x11/2\x1381\x1e,(\x1d\'\'\x1f&)",-\'-$\x1d2*\x1f/0 %.\x19\xbc\xd0\xb7#2\x1b*(\x194\'\x1f0\x1b\x184\x1f\x1c2!\x1a(\x1e\x12!\x1e\r\x1e"\x0b\x1c\'\t\x19&\x08\x14!\x03\xc4\xcf\xb1\x16\x1b\x04!#\x0e*$\x18\'\x1d\x14,\x1a\x165"\x1e8 \x1cB.\'@.$&\x19\t.#\x1171\x1973\x18\xcc\xcb\xad,(\r)#\x0b/\'\x125(\x185!\x1a<"!@#%0\x1d\x19#\'\x16\x1a*\x10$2\x19\xc7\xd5\xbb%/\x16(1\x16(.\x14(,\x1311\x19,*\x13+&\x131*\x18=1%A5)>/(/.\x1a\xbe\xd4\xad&@\x1303\x1492\x1f+\'\x1c%%\x1d"\'!*-&) \x191)\x1e./\x1f )\x14\xbf\xd3\xba(7 -+\x1e3$\x1d/\x17\x155\x1b\x1a1\x1e\x1a\'\x1a\x11\x1f\x19\t\x1b\x1d\x07\x19"\x07\x17"\x04\x16!\x03\xbe\xc7\xac\x1d!\n(\'\x152(\x1e*\x1b\x14*\x15\x142\x1a\x1a' SkyQuery_TestTableName = "TestTable_SciScript_R" SkyQuery_TestTableCSV = u"Column1,Column2\n4.5,5.5\n" @@ -148,7 +146,7 @@ def test_CasJobs_executeQuery(self): def test_CasJobs_submitJob(self): jobId = CasJobs.submitJob(sql=CasJobs_TestQuery + " into MyDB." + CasJobs_TestTableName1, context=CasJobs_TestDatabase) - jobDescription = CasJobs.waitForJob(jobId=jobId, verbose=TRUE) + jobDescription = CasJobs.waitForJob(jobId=jobId, verbose=True) df = CasJobs.executeQuery(sql="DROP TABLE " + CasJobs_TestTableName1, context="MyDB", format="csv") self.assertNotEqual(jobId, "") @@ -231,11 +229,9 @@ def test_SkyServer_sqlSearch(self): def test_SkyServer_getJpegImgCutout(self): #image cutout - img = SkyServer.getJpegImgCutout(ra=197.614455642896, dec=18.438168853724, width=50, height=50, scale=0.4, dataRelease=SkyServer_DataRelease,opt="OG",query="SELECT TOP 100 p.objID, p.ra, p.dec, p.r FROM fGetObjFromRectEq(197.6,18.4,197.7,18.5) n, PhotoPrimary p WHERE n.objID=p.objID") - if (sys.version_info > (3, 0)): # python3 - self.assertEqual(SkyServer_ImageCutoutImagePy3, img.tostring()) - else: # python2 - self.assertEqual(SkyServer_ImageCutoutImagePy2, img.tostring()) + img = SkyServer.getJpegImgCutout(ra=197.614455642896, dec=18.438168853724, width=512, height=512, scale=0.4, dataRelease=SkyServer_DataRelease,opt="OG",query="SELECT TOP 100 p.objID, p.ra, p.dec, p.r FROM fGetObjFromRectEq(197.6,18.4,197.7,18.5) n, PhotoPrimary p WHERE n.objID=p.objID") + im = skimage.io.imread("./TestGalaxy.jpeg") + self.assertEqual(img.tobytes(), im.tobytes()) def test_SkyServer_radialSearch(self): # radial search @@ -312,14 +308,14 @@ def test_SciDrive_upload_download_delete(self): stringio = SciDrive.download(path=SciDrive_Directory + "/" + SciDrive_FileName, format="StringIO") fileContent = stringio.read() responseDelete = SciDrive.delete(SciDrive_Directory) - self.assertEqual(responseUpload["path"], "/" + SciDrive_Directory + "/" + SciDrive_FileName) + self.assertEqual(responseUpload["path"], SciDrive_Directory + "/" + SciDrive_FileName) self.assertEqual(fileContent, SciDrive_FileContent) self.assertEqual(responseDelete, True) responseUpload = SciDrive.upload(path=SciDrive_Directory + "/" + SciDrive_FileName, data=SciDrive_FileContent) fileContent = SciDrive.download(path=SciDrive_Directory + "/" + SciDrive_FileName, format="text") responseDelete = SciDrive.delete(SciDrive_Directory) - self.assertEqual(responseUpload["path"], "/" + SciDrive_Directory + "/" + SciDrive_FileName) + self.assertEqual(responseUpload["path"], SciDrive_Directory + "/" + SciDrive_FileName) self.assertEqual(fileContent, SciDrive_FileContent) self.assertEqual(responseDelete, True) @@ -402,12 +398,6 @@ def test_SkyQuery_getDatasetInfo(self): def test_SkyQuery_listDatasetTables(self): tables = SkyQuery.listDatasetTables("MyDB") - - - - - - if __name__ == '__main__': #unittest.main() unittest.TestLoader.sortTestMethodsUsing = lambda x, y: cmp(x,y); @@ -418,4 +408,5 @@ def test_SkyQuery_listDatasetTables(self): suite = testLoader.loadTestsFromTestCase(TestCasJobs); unittest.TextTestRunner(verbosity=2).run(suite) suite = testLoader.loadTestsFromTestCase(TestSkyServer); unittest.TextTestRunner(verbosity=2).run(suite) suite = testLoader.loadTestsFromTestCase(TestSciDrive); unittest.TextTestRunner(verbosity=2).run(suite) - suite = testLoader.loadTestsFromTestCase(TestSkyQuery); unittest.TextTestRunner(verbosity=2).run(suite) \ No newline at end of file + suite = testLoader.loadTestsFromTestCase(TestSkyQuery); unittest.TextTestRunner(verbosity=2).run(suite) + From d51a3779e5ca3e2d56c197f4baecf17ca4fd932a Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Sun, 16 Apr 2017 20:01:43 -0400 Subject: [PATCH 07/79] added unit tests for computeJobs --- Tests/UnitTests.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/Tests/UnitTests.py b/Tests/UnitTests.py index 76aa138..abe421d 100644 --- a/Tests/UnitTests.py +++ b/Tests/UnitTests.py @@ -398,6 +398,48 @@ def test_SkyQuery_getDatasetInfo(self): def test_SkyQuery_listDatasetTables(self): tables = SkyQuery.listDatasetTables("MyDB") + + + + +class TestComputeJobs(unittest.TestCase): + + token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword); + + def setUp(self): + pass + + # ******************************************************************************************************* + # ComputeJobs section + + def test_ComputeJobs_getComputeDomains(self): + pass + + def test_ComputeJobs_submitNotebookJob(self): + pass + + def test_ComputeJobs_submitShellCommandJob(self): + pass + + def test_ComputeJobs_getJobDescription(self): + pass + + def test_ComputeJobs_getJobStatus(self): + pass + + def test_ComputeJobs_getJobsList(self): + pass + + def test_ComputeJobs_getJobDirectory(self): + pass + + def test_ComputeJobs_cancelJob(self): + pass + + def test_ComputeJobs_waitForJob(self): + pass + + if __name__ == '__main__': #unittest.main() unittest.TestLoader.sortTestMethodsUsing = lambda x, y: cmp(x,y); From 93ef512472052db334c6389607ebbe2987b5f1c5 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Mon, 17 Apr 2017 10:32:23 -0400 Subject: [PATCH 08/79] Added skeleton for FileService unit tests --- Tests/UnitTests.py | 22 ++++++++++++++++++++++ py2/SciServer/{Files.py => FileService.py} | 0 2 files changed, 22 insertions(+) rename py2/SciServer/{Files.py => FileService.py} (100%) diff --git a/Tests/UnitTests.py b/Tests/UnitTests.py index abe421d..f3ebf65 100644 --- a/Tests/UnitTests.py +++ b/Tests/UnitTests.py @@ -440,6 +440,28 @@ def test_ComputeJobs_waitForJob(self): pass +class TestFileService(unittest.TestCase): + + token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword); + + def setUp(self): + pass + + # ******************************************************************************************************* + # FileService section + + def test_FileService_createDir(self): + pass + + def test_FileService_delete(self): + pass + + def test_FileService_upload(self): + pass + + def test_FileService_download(self): + pass + if __name__ == '__main__': #unittest.main() unittest.TestLoader.sortTestMethodsUsing = lambda x, y: cmp(x,y); diff --git a/py2/SciServer/Files.py b/py2/SciServer/FileService.py similarity index 100% rename from py2/SciServer/Files.py rename to py2/SciServer/FileService.py From e9b2d2441f76d96346c66c274fdc9ce42bbb9bae Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Mon, 24 Apr 2017 08:23:58 -0400 Subject: [PATCH 09/79] Update ComputeJobs names and code --- py2/SciServer/ComputeJobs.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/py2/SciServer/ComputeJobs.py b/py2/SciServer/ComputeJobs.py index f7a838f..25d63dc 100644 --- a/py2/SciServer/ComputeJobs.py +++ b/py2/SciServer/ComputeJobs.py @@ -22,16 +22,19 @@ def getJobDirectory(jobId = None): return jobDirectory else: - raise OSError("Cannot find jobs directory for jobId=" + jobId + ".") + raise OSError("Cannot find jobs directory for jobId=" + str(jobId) + ".") -def getComputeDomains(): +def getComputeDomains(batch=True, interactive=False): """ Gets a list of all registered compute domains that the user has access to. """ token = Authentication.getToken() if token is not None and token != "": - url = Config.JobmApiURL + "/computedomains" + + queryString = ("batch=true&" if bool(batch) == True else "batch=false&"); + queryString = queryString + ("interactive=true" if bool(interactive) == True else "interactive=false"); + url = Config.JobmApiURL + "/computedomains?" + queryString headers = {'X-Auth-Token': token, "Content-Type": "application/json"} res = requests.get(url, headers=headers, stream=True) @@ -113,7 +116,7 @@ def getJobStatus(jobId): raise Exception("Invalid integer value given to job status.") -def submitNotebookJob(computeDomainName, notebookPath, dockerImage=[], volumes=[], parameters="", jobAlias = ""): +def submitNotebookJob(domainApiEndpoint, notebookPath, dockerImage="", volumes=[None], parameters="", jobAlias = ""): """ Submits a Jupyter Notebook for execution as an asynchronous job, :param computeDomainName: Name of compute domain (string). E.g: http://compute.sciserver.org/dashboard/api/container/ @@ -132,7 +135,7 @@ def submitNotebookJob(computeDomainName, notebookPath, dockerImage=[], volumes=[ "command": parameters, "scriptURI": notebookPath, "submitterDID": jobAlias, - "dockerComputeEndpoint": computeDomainName, + "dockerComputeEndpoint": domainApiEndpoint, "dockerImageName": dockerImage, "volumeContainers": volumes } @@ -150,7 +153,7 @@ def submitNotebookJob(computeDomainName, notebookPath, dockerImage=[], volumes=[ raise Exception("User token is not defined. First log into SciServer.") -def submitShellCommandJob(computeDomainName, shellCommand, dockerImage=[], volumes=[], jobAlias = ""): +def submitShellCommandJob(domainApiEndpoint, shellCommand, dockerImage=[], volumes=[], jobAlias = ""): """ Submits a shell command for execution as an asynchronous job, :param computeDomainName: Name of compute domain (string). E.g: http://compute.sciserver.org/dashboard/api/container/ @@ -168,7 +171,7 @@ def submitShellCommandJob(computeDomainName, shellCommand, dockerImage=[], volum dockerJobModel = { "command": shellCommand, "submitterDID": jobAlias, - "dockerComputeEndpoint": computeDomainName, + "dockerComputeEndpoint": domainApiEndpoint, "dockerImageName": dockerImage, "volumeContainers": volumes } @@ -191,7 +194,7 @@ def cancelJob(jobId): token = Authentication.getToken() if token is not None and token != "": - url = Config.JobmApiURL + "/jobs/" + jobId + "/cancel" + url = Config.JobmApiURL + "/jobs/" + str(jobId) + "/cancel" headers = {'X-Auth-Token': token, "Content-Type": "application/json"} res = requests.post(url, headers=headers, stream=True) @@ -216,8 +219,8 @@ def waitForJob(jobId, verbose=True, pollTime = 5): if verbose: #print back, print waitingStr, - jobDesc = getJobStatus(jobId) - jobStatus = int(jobDesc["status"]) + jobDesc = getJobDescription(jobId) + jobStatus = jobDesc["status"] if jobStatus >= 32: complete = True if verbose: From 2792e95c4e125e346f918d68f6a7b5c13f2d9e78 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Mon, 24 Apr 2017 08:26:09 -0400 Subject: [PATCH 10/79] Added DirList method in FileService --- py2/SciServer/FileService.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/py2/SciServer/FileService.py b/py2/SciServer/FileService.py index 8392107..9269f05 100644 --- a/py2/SciServer/FileService.py +++ b/py2/SciServer/FileService.py @@ -32,6 +32,34 @@ def createDir(path): raise Exception("User token is not defined. First log into SciServer.") +def getDirList(path, options="X", level=1): + """ + Lists the contents of a directory. + :param path: path to the directory. E.g: path="/persistent/newDirectory" + :return: True if the directory creation was successful. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileSystem API returns an error. + """ + token = Authentication.getToken() + if token is not None and token != "": + + if ~path.startswith("/"): + path = "/" + path; + + url = Config.FileServiceURL + "/tree" + path + "?options=" + str(options) + "&level=" + str(level) + + headers = {'X-Auth-Token': token} + res = requests.get(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + return res.content.decode(); + else: + raise Exception("Error when creating new directory " + path + ".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + + + def delete(path): """ Delete a directory or file in the remote File System. From 2a41d737a4c9961636abcad64ad83ddb4ae3a33f Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Mon, 24 Apr 2017 08:27:26 -0400 Subject: [PATCH 11/79] Updated FIleService and ComputeJobs unit tests --- Tests/UnitTests.py | 207 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 171 insertions(+), 36 deletions(-) diff --git a/Tests/UnitTests.py b/Tests/UnitTests.py index f3ebf65..40640dd 100644 --- a/Tests/UnitTests.py +++ b/Tests/UnitTests.py @@ -1,5 +1,5 @@ #!/usr/bin/python -from SciServer import Authentication, LoginPortal, Config, CasJobs, SkyServer, SkyQuery, SciDrive +from SciServer import Authentication, LoginPortal, Config, CasJobs, SkyServer, SkyQuery, SciDrive, FileService, ComputeJobs import unittest2 as unittest import os; import pandas; @@ -39,12 +39,13 @@ SciDrive_FileName = "TestFile.csv" SciDrive_FileContent = "Column1,Column2\n4.5,5.5\n" +FileService_NewDirName = "UnitTestDir" +FileService_NewFileName = "MyNewFile.txt" +FileService_newFileContent = "#ID,Column1,Column2\n1,4.5,5.5\n" +FileService_RemoteFileName = "persistent/" + FileService_NewDirName + "/" + FileService_NewFileName +FileService_RemoteDirName = "persistent/" + FileService_NewDirName -#class TestVariables: -# SkyQuery_jobId = None; -# def __init__(self): -# pass - +ComputeJobs_ = "" class TestAuthentication(unittest.TestCase): @@ -239,6 +240,9 @@ def test_SkyServer_radialSearch(self): self.maxDiff = None; self.assertEqual(SkyServer_RadialSearchResultCSV, df.to_csv(index=False, float_format="%.6f")) + + + def test_SkyServer_rectangularSearch(self): #rectangular search df = SkyServer.rectangularSearch(min_ra=258.3, max_ra=258.31, min_dec=64,max_dec=64.01, dataRelease=SkyServer_DataRelease) @@ -406,38 +410,116 @@ class TestComputeJobs(unittest.TestCase): token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword); + def setUp(self): pass # ******************************************************************************************************* # ComputeJobs section + + + + ComputeJobs_NotebookName = "TestNotebook.ipynb" + ComputeJobs_LocalNotebookPath = "./" + ComputeJobs_NotebookName + ComputeJobs_RemoteNotebookPath = "/home/idies/workspace/persistent/" + ComputeJobs_NotebookName + ComputeJobs_JobAlias = "MyJobAlias" + ComputeJobs_Parameters = "param1" + ComputeJobs_ShellCommand = "ls"; + ComputeJobs_ImageNameLike = "python" #must be all lowercase + def test_ComputeJobs_getComputeDomains(self): - pass + domains = ComputeJobs.getComputeDomains(batch=True, interactive=False) + self.assertTrue(domains[0]["apiEndpoint"] != "" and domains[0]["apiEndpoint"] is not None) - def test_ComputeJobs_submitNotebookJob(self): - pass - def test_ComputeJobs_submitShellCommandJob(self): - pass - def test_ComputeJobs_getJobDescription(self): - pass - def test_ComputeJobs_getJobStatus(self): - pass - def test_ComputeJobs_getJobsList(self): - pass + def test_ComputeJobs_submitNotebookJob_getJobsList_getJobDescription_waitForJob_getJobStatus_submitShellCommandJob_cancelJob(self): + + try: + isUploaded = FileService.upload(path=ComputeJobs_RemoteNotebookPath,localFilePath=ComputeJobs_LocalNotebookPath) + except: + pass; + + try: + + domains = ComputeJobs.getComputeDomains(batch=True, interactive=False) + self.assertTrue( domains[0]['id'] > 0 ) + + domainIndex = None + imageIndex = None + try: + for i in range(len(domains)): + images = domains[i]['images'] + for j in range(len(images)): + if ComputeJobs_ImageNameLike in str(images[j]['name']).lower() or ComputeJobs_ImageNameLike in str(images[j]['description']).lower() : + domainIndex = i + imageIndex = j + raise Exception(""); + except: + pass; + + domainIndex = 1 + imageIndex = 0 + + if domainIndex is None or imageIndex is None: + raise Exception("Cannot find '" + ComputeJobs_ImageNameLike +"' image in any Compute Domain.") + + domainApiEndpoint = domains[domainIndex]['apiEndpoint'] + dockerImage = domains[domainIndex]['images'][imageIndex]['name'] + + volumes = []; + if len(domains[domainIndex]['volumes']) > 0: + volumes = [ { "name" : domains[domainIndex]['volumes'][0]['name'] } ] + + notebookPath = ComputeJobs_RemoteNotebookPath + parameters = ComputeJobs_Parameters + jobAlias = ComputeJobs_JobAlias + + job = ComputeJobs.submitNotebookJob(domainApiEndpoint=domainApiEndpoint, notebookPath=notebookPath, dockerImage=dockerImage, volumes=volumes, parameters=parameters, jobAlias=jobAlias) + self.assertTrue(job['scriptURI'] == ComputeJobs_RemoteNotebookPath) + self.assertTrue(job['dockerImageName'] == dockerImage) + self.assertTrue(job['dockerComputeEndpoint'] in domainApiEndpoint ) + self.assertTrue(job['submitterDID'] == jobAlias) + self.assertTrue(job['command'] == parameters) + self.assertTrue(job['status'] >= 1) + + jobList = ComputeJobs.getJobsList(); + jobId = jobList[0]['id']; + + job = ComputeJobs.waitForJob(jobId=jobId, verbose=True, pollTime=2) + self.assertTrue(jobList[0] == job) + + jobDesc = ComputeJobs.getJobDescription(jobId) + self.assertTrue(jobDesc == job) + + status = ComputeJobs.getJobStatus(jobId=jobId) + self.assertTrue(status == "SUCCESS") + + job = ComputeJobs.submitShellCommandJob(domainApiEndpoint=domainApiEndpoint, shellCommand=ComputeJobs_ShellCommand, dockerImage=dockerImage, volumes=volumes, jobAlias=jobAlias) + self.assertTrue(job['dockerImageName'] == dockerImage) + self.assertTrue(job['dockerComputeEndpoint'] in domainApiEndpoint ) + self.assertTrue(job['submitterDID'] == jobAlias) + self.assertTrue(job['command'] == ComputeJobs_ShellCommand) + self.assertTrue(job['status'] >= 1) + + job = ComputeJobs.submitNotebookJob(domainApiEndpoint=domainApiEndpoint, notebookPath=notebookPath, dockerImage=dockerImage, volumes=volumes, parameters=parameters, jobAlias=jobAlias) + isCanceled = ComputeJobs.cancelJob(job['jobId']) + self.assertTrue(isCanceled) + status = ComputeJobs.getJobStatus(jobId=jobId) + self.assertTrue(status == "CANCELED") + + finally: + FileService.delete(path=ComputeJobs_RemoteNotebookPath) + + def test_ComputeJobs_getJobDirectory(self): - pass + pass; - def test_ComputeJobs_cancelJob(self): - pass - def test_ComputeJobs_waitForJob(self): - pass class TestFileService(unittest.TestCase): @@ -450,27 +532,80 @@ def setUp(self): # ******************************************************************************************************* # FileService section - def test_FileService_createDir(self): - pass + def test_FileService_createDir_getDirList_delete(self): - def test_FileService_delete(self): - pass + try: + wasDirDeleted = FileService.delete(FileService_RemoteDirName) + except: + pass + + try: + wasDirCreated = FileService.createDir(FileService_RemoteDirName); + self.assertTrue(wasDirCreated) + + dirList = FileService.getDirList(FileService_RemoteDirName) + self.assertTrue(dirList["path"].__contains__(FileService_NewDirName)) + + wasDirDeleted = FileService.delete(FileService_RemoteDirName) + self.assertTrue(wasDirDeleted) + + finally: + try: + wasDirDeleted = FileService.delete(FileService_RemoteDirName) + except: + pass; + + + + def test_FileService_upload_getDirList_download_delete(self): + try: + + if (sys.version_info > (3, 0)): #python3 + file = open(FileService_NewFileName, "w") + else: #python2 + file = open(FileService_NewFileName, "wb") + + file.write(FileService_newFileContent) + file.close() + + isUploaded = FileService.upload(path=FileService_RemoteFileName, localFilePath=FileService_NewFileName) + dirList = FileService.getDirList(FileService_RemoteDirName) + stringio = FileService.download(path=FileService_RemoteFileName, format="StringIO") + fileContent = stringio.read() + isDeleted = FileService.delete(FileService_RemoteFileName) + self.assertTrue(isUploaded) + self.assertTrue( dirList.__contains__(FileService_NewFileName)) + self.assertEqual(fileContent, FileService_newFileContent) + self.assertTrue(isDeleted) + + isUploaded = FileService.upload(path=FileService_RemoteFileName, data=FileService_newFileContent) + dirList = FileService.getDirList(FileService_RemoteDirName) + fileContent = FileService.download(path=FileService_RemoteFileName, format="text") + isDeleted = FileService.delete(FileService_RemoteFileName) + self.assertTrue(isUploaded) + self.assertTrue( dirList.__contains__(FileService_NewFileName)) + self.assertEqual(fileContent, FileService_newFileContent) + self.assertTrue(isDeleted) + + finally: + try: + os.remove(FileService_newFileContent) + except: + pass; - def test_FileService_upload(self): - pass - def test_FileService_download(self): - pass if __name__ == '__main__': #unittest.main() unittest.TestLoader.sortTestMethodsUsing = lambda x, y: cmp(x,y); testLoader = unittest.TestLoader() testLoader.sortTestMethodsUsing = lambda x, y: 0; - suite = testLoader.loadTestsFromTestCase(TestAuthentication); unittest.TextTestRunner(verbosity=2).run(suite) - suite = testLoader.loadTestsFromTestCase(TestLoginPortal); unittest.TextTestRunner(verbosity=2).run(suite) - suite = testLoader.loadTestsFromTestCase(TestCasJobs); unittest.TextTestRunner(verbosity=2).run(suite) - suite = testLoader.loadTestsFromTestCase(TestSkyServer); unittest.TextTestRunner(verbosity=2).run(suite) - suite = testLoader.loadTestsFromTestCase(TestSciDrive); unittest.TextTestRunner(verbosity=2).run(suite) - suite = testLoader.loadTestsFromTestCase(TestSkyQuery); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestAuthentication); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestLoginPortal); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestCasJobs); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestSkyServer); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestSciDrive); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestSkyQuery); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestComputeJobs); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestFileService); unittest.TextTestRunner(verbosity=2).run(suite) From 96e508745150eaa63b7cf1151584eb7ff869e658 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Mon, 24 Apr 2017 08:27:58 -0400 Subject: [PATCH 12/79] Added test notebook --- Tests/TestNotebook.ipynb | 156 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 Tests/TestNotebook.ipynb diff --git a/Tests/TestNotebook.ipynb b/Tests/TestNotebook.ipynb new file mode 100644 index 0000000..84d9835 --- /dev/null +++ b/Tests/TestNotebook.ipynb @@ -0,0 +1,156 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import time;\n", + "import random;\n", + "import os;\n", + "from datetime import datetime\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print(str(datetime.now()))\n", + "\n", + "# sleep times.\n", + "sleepTime1 = 5;#5sec\n", + "sleepTime2 = 300;#5min\n", + "sleepTime3 = 3900;#1h5min, thought for testing the case of a job timeout of 1hr.\n", + "#rand = random.random();\n", + "rand = 0;#force fixed sleepTime \n", + "print(\"rand=\" + str(rand))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "now = str(datetime.now());\n", + "print(now + \" Starting Notebook execution...\")\n", + "os.system(\"echo '\" + now + \" Starting Notebook execution...'\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "if rand < 0.3:\n", + " time.sleep(sleepTime1);\n", + "elif rand >= 0.3 and rand < 0.6:\n", + " time.sleep(sleepTime2);\n", + "else:\n", + " time.sleep(sleepTime3);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "now = str(datetime.now());\n", + "print(now + \" Now writing HelloWorld.txt file into job directory.\");\n", + "os.system(\"echo '\" + now + \" Now writing HelloWorld.txt file into job directory.'\");\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "now = str(datetime.now());\n", + "parameters = None;\n", + "with open('parameters.txt') as f:\n", + " parameters = f.read().rstrip('\\n')\n", + "\n", + "if parameters is None:\n", + " raise Exception('Could not read /home/idies/jobs.path') \n", + " \n", + "jobsPath = None\n", + "with open('/home/idies/jobs.path') as f:\n", + " jobsPath = f.read().rstrip('\\n')\n", + "\n", + "if jobsPath == \"\" or jobsPath == None: \n", + " raise Exception(now + ' Could not read /home/idies/jobs.path') \n", + " \n", + "print(now + \" Was able to read /home/idies/jobs.path : \" + jobsPath);\n", + "os.system(\"echo '\" + now + \" Was able to read /home/idies/jobs.path : \" + jobsPath +\"'\");\n", + " \n", + "target = open(jobsPath + \"HelloWorld.txt\", 'w')\n", + "target.truncate()\n", + "line1 =\"Hello world!\"\n", + "target.write(line1)\n", + "target.write(\"\\n\")\n", + "target.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "now = str(datetime.now());\n", + "rand2 = random.random();\n", + "#rand2 = 0.1 # force one result type: python exception or not.\n", + "\n", + "if rand2 < 0.5:\n", + " print(now + \" Ending TestNotebook execution with a python exception (dividing by zero).\");\n", + " os.system(\"echo '\" + now + \" Ending TestNotebook execution with a python exception (dividing by zero).'\");\n", + " a = 1/0;\n", + " print(a);\n", + "else:\n", + " print(now + \" Ending TestNotebook execution with no python exception.\");\n", + " os.system(\"echo '\" + now + \" Ending TestNotebook execution with no python exception.'\");" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.1" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From ae1838c02fc2c9756f9bbc8b7696040ee11f16b3 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Sun, 21 May 2017 15:17:01 -0400 Subject: [PATCH 13/79] added taskName to url --- py2/SciServer/Authentication.py | 17 ++++- py2/SciServer/CasJobs.py | 66 ++++++++++++----- py2/SciServer/SciDrive.py | 49 +++++++++++-- py2/SciServer/SkyQuery.py | 121 ++++++++++++++++++++++++++++---- py3/SciServer/Authentication.py | 17 ++++- py3/SciServer/CasJobs.py | 66 ++++++++++++----- py3/SciServer/SciDrive.py | 49 +++++++++++-- py3/SciServer/SkyQuery.py | 121 ++++++++++++++++++++++++++++---- 8 files changed, 428 insertions(+), 78 deletions(-) diff --git a/py2/SciServer/Authentication.py b/py2/SciServer/Authentication.py index 4610b49..41511a8 100644 --- a/py2/SciServer/Authentication.py +++ b/py2/SciServer/Authentication.py @@ -38,10 +38,17 @@ def getKeystoneUserWithToken(token): .. seealso:: Authentication.getToken, Authentication.login, Authentication.setToken. """ + taskName = "" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Authentication.getKeystoneUserWithToken" + else: + taskName = "SciScript-Python.Authentication.getKeystoneUserWithToken" + loginURL = Config.AuthenticationURL if ~loginURL.endswith("/"): loginURL = loginURL + "/" - loginURL = loginURL + token + + loginURL = loginURL + token + "?TaskName=" + taskName; getResponse = requests.get(loginURL) if getResponse.status_code != 200: @@ -72,7 +79,13 @@ def login(UserName, Password): .. seealso:: Authentication.getKeystoneUserWithToken, Authentication.getToken, Authentication.setToken, Authentication.token. """ - loginURL = Config.AuthenticationURL + taskName = "" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Authentication.login" + else: + taskName = "SciScript-Python.Authentication.login" + + loginURL = Config.AuthenticationURL + "?TaskName=" + taskName authJson = {"auth":{"identity":{"password":{"user":{"name":UserName,"password":Password}}}}} diff --git a/py2/SciServer/CasJobs.py b/py2/SciServer/CasJobs.py index 6c4b66c..966ee06 100644 --- a/py2/SciServer/CasJobs.py +++ b/py2/SciServer/CasJobs.py @@ -24,7 +24,14 @@ def getSchemaName(): if token is not None and token != "": keystoneUserId = Authentication.getKeystoneUserWithToken(token).id - usersUrl = Config.CasJobsRESTUri + "/users/" + keystoneUserId + + taskName = "" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.getSchemaName" + else: + taskName = "SciScript-Python.CasJobs.getSchemaName" + + usersUrl = Config.CasJobsRESTUri + "/users/" + keystoneUserId + "?TaskName=" + taskName headers={'X-Auth-Token': token,'Content-Type': 'application/json'} getResponse = requests.get(usersUrl,headers=headers) if getResponse.status_code != 200: @@ -51,7 +58,13 @@ def getTables(context="MyDB"): token = Authentication.getToken() if token is not None and token != "": - TablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables" + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.getTables" + else: + taskName = "SciScript-Python.CasJobs.getTables" + + TablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables" + "?TaskName=" + taskName headers={'X-Auth-Token': token,'Content-Type': 'application/json'} @@ -100,15 +113,15 @@ def executeQuery(sql, context="MyDB", format="pandas"): else: raise Exception("Error when executing query. Illegal format parameter specification: " + str(format)); - QueryUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/query" - - TaskName = ""; + taskName = ""; if Config.isSciServerComputeEnvironment(): - TaskName = "Compute.SciScript-Python.CasJobs.executeQuery" + taskName = "Compute.SciScript-Python.CasJobs.executeQuery" else: - TaskName = "SciScript-Python.CasJobs.executeQuery" + taskName = "SciScript-Python.CasJobs.executeQuery" + + QueryUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/query" + "?TaskName=" + taskName - query = {"Query": sql, "TaskName": TaskName} + query = {"Query": sql, "TaskName": taskName} data = json.dumps(query).encode() @@ -154,15 +167,15 @@ def submitJob(sql, context="MyDB"): token = Authentication.getToken() if token is not None and token != "": - QueryUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/jobs" - - TaskName = ""; + taskName = ""; if Config.isSciServerComputeEnvironment(): - TaskName = "Compute.SciScript-Python.CasJobs.submitJob" + taskName = "Compute.SciScript-Python.CasJobs.submitJob" else: - TaskName = "SciScript-Python.CasJobs.submitJob" + taskName = "SciScript-Python.CasJobs.submitJob" + + QueryUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/jobs" + "?TaskName=" + taskName - query = {"Query": sql, "TaskName": TaskName} + query = {"Query": sql, "TaskName": taskName} data = json.dumps(query).encode() @@ -193,7 +206,13 @@ def getJobStatus(jobId): token = Authentication.getToken() if token is not None and token != "": - QueryUrl = Config.CasJobsRESTUri + "/jobs/" + str(jobId) + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.getJobStatus" + else: + taskName = "SciScript-Python.CasJobs.getJobStatus" + + QueryUrl = Config.CasJobsRESTUri + "/jobs/" + str(jobId) + "?TaskName=" + taskName headers={'X-Auth-Token': token,'Content-Type': 'application/json'} @@ -220,7 +239,13 @@ def cancelJob(jobId): token = Authentication.getToken() if token is not None and token != "": - QueryUrl = Config.CasJobsRESTUri + "/jobs/" + str(jobId) + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.cancelJob" + else: + taskName = "SciScript-Python.CasJobs.cancelJob" + + QueryUrl = Config.CasJobsRESTUri + "/jobs/" + str(jobId) + "?TaskName=" + taskName headers={'X-Auth-Token': token,'Content-Type': 'application/json'} @@ -387,7 +412,14 @@ def uploadCSVDataToTable(csvData, tableName, context="MyDB"): #if (Config.executeMode == "debug"): # print "Uploading ", sys.getsizeof(CVSdata), "bytes..." - tablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables/" + tableName + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.uploadCSVDataToTable" + else: + taskName = "SciScript-Python.CasJobs.uploadCSVDataToTable" + + tablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables/" + tableName + "?TaskName=" + taskName headers={} headers['X-Auth-Token']= token diff --git a/py2/SciServer/SciDrive.py b/py2/SciServer/SciDrive.py index b3df2db..4f1bde2 100644 --- a/py2/SciServer/SciDrive.py +++ b/py2/SciServer/SciDrive.py @@ -25,7 +25,13 @@ def createContainer(path): 'uri="vos://' + Config.SciDriveHost + '!vospace/' + path + '">' '' '') - url = Config.SciDriveHost + '/vospace-2.0/nodes/' + path + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.createContainer" + else: + taskName = "SciScript-Python.SciDrive.createContainer" + + url = Config.SciDriveHost + '/vospace-2.0/nodes/' + path + "?TaskName=" + taskName; data = str.encode(containerBody) headers = {'X-Auth-Token': token, 'Content-Type': 'application/xml'} res = requests.put(url, data=data, headers=headers) @@ -52,7 +58,14 @@ def upload(path, data="", localFilePath=""): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SciDriveHost + '/vospace-2.0/1/files_put/dropbox/' + path + + taskName = "" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.upload" + else: + taskName = "SciScript-Python.SciDrive.upload" + + url = Config.SciDriveHost + '/vospace-2.0/1/files_put/dropbox/' + path + "?TaskName=" + taskName; headers = {'X-Auth-Token': token} if(localFilePath != ""): with open(localFilePath, "rb") as file: @@ -85,7 +98,13 @@ def publicUrl(path): token = Authentication.getToken() if token is not None and token != "": - url = Config.SciDriveHost + '/vospace-2.0/1/media/sandbox/' + str(path) + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.publicUrl" + else: + taskName = "SciScript-Python.SciDrive.publicUrl" + + url = Config.SciDriveHost + '/vospace-2.0/1/media/sandbox/' + str(path) + "?TaskName=" + taskName headers = {'X-Auth-Token': token} res = requests.get(url, headers=headers) if res.status_code != 200: @@ -113,7 +132,13 @@ def directoryList(path=""): token = Authentication.getToken() if token is not None and token != "": - url = Config.SciDriveHost + "/vospace-2.0/1/metadata/sandbox/" + str(path) + "?list=True&path=" + str(path) + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.directoryList" + else: + taskName = "SciScript-Python.SciDrive.directoryList" + + url = Config.SciDriveHost + "/vospace-2.0/1/metadata/sandbox/" + str(path) + "?list=True&path=" + str(path) + "&TaskName=" + taskName; headers = {'X-Auth-Token': token} res = requests.get(url, headers=headers) if res.status_code != 200: @@ -143,7 +168,13 @@ def download(path, format="text", localFilePath=""): token = Authentication.getToken() if token is not None and token != "": - fileUrl = publicUrl(path) + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.download" + else: + taskName = "SciScript-Python.SciDrive.download" + + fileUrl = publicUrl(path) + "?TaskName=" + taskName res = requests.get(fileUrl, stream=True) if res.status_code != 200: raise Exception("Error when downloading SciDrive file " + str(path) + ".\nHttp Response from SciDrive API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); @@ -194,7 +225,13 @@ def delete(path): 'uri="vos://' + Config.SciDriveHost + '!vospace/' + path + '">' '' '') - url = Config.SciDriveHost + '/vospace-2.0/nodes/' + path + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.delete" + else: + taskName = "SciScript-Python.SciDrive.delete" + + url = Config.SciDriveHost + '/vospace-2.0/nodes/' + path + "?TaskName=" + taskName; data = str.encode(containerBody) headers = {'X-Auth-Token': token, 'Content-Type': 'application/xml'} res = requests.delete(url, data=data, headers=headers) diff --git a/py2/SciServer/SkyQuery.py b/py2/SciServer/SkyQuery.py index e2991ae..5cd5f54 100644 --- a/py2/SciServer/SkyQuery.py +++ b/py2/SciServer/SkyQuery.py @@ -27,7 +27,14 @@ def getJobStatus(jobId): """ token = Authentication.getToken() if token is not None and token != "": - statusURL = Config.SkyQueryUrl + '/Jobs.svc/jobs/' + str(jobId) + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.getJobStatus" + else: + taskName = "SciScript-Python.SkyQuery.getJobStatus" + + statusURL = Config.SkyQueryUrl + '/Jobs.svc/jobs/' + str(jobId) + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -57,7 +64,13 @@ def cancelJob(jobId): token = Authentication.getToken() if token is not None and token != "": - statusURL = Config.SkyQueryUrl + '/Jobs.svc/jobs/' + jobId + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.cancelJob" + else: + taskName = "SciScript-Python.SkyQuery.cancelJob" + + statusURL = Config.SkyQueryUrl + '/Jobs.svc/jobs/' + jobId + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -93,7 +106,14 @@ def listQueues(): """ token = Authentication.getToken() if token is not None and token != "": - jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues' + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.listQueues" + else: + taskName = "SciScript-Python.SkyQuery.listQueues" + + jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -123,7 +143,13 @@ def getQueueInfo(queue): token = Authentication.getToken() if token is not None and token != "": - jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.getQueueInfo" + else: + taskName = "SciScript-Python.SkyQuery.getQueueInfo" + + jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -154,7 +180,13 @@ def submitJob(query, queue='quick'): token = Authentication.getToken() if token is not None and token != "": - jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + '/jobs' + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.submitJob" + else: + taskName = "SciScript-Python.SkyQuery.submitJob" + + jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + '/jobs' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -226,7 +258,14 @@ def listJobs(queue="quick"): """ token = Authentication.getToken() if token is not None and token != "": - jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + '/jobs?' + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.listJobs" + else: + taskName = "SciScript-Python.SkyQuery.listJobs" + + jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + '/jobs' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -261,7 +300,13 @@ def listAllDatasets(): token = Authentication.getToken() if token is not None and token != "": - schemaURL = Config.SkyQueryUrl + '/Schema.svc/datasets' + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.listAllDatasets" + else: + taskName = "SciScript-Python.SkyQuery.listAllDatasets" + + schemaURL = Config.SkyQueryUrl + '/Schema.svc/datasets' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -290,7 +335,14 @@ def getDatasetInfo(datasetName="MyDB"): """ token = Authentication.getToken() if token is not None and token != "": - schemaURL = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.getDatasetInfo" + else: + taskName = "SciScript-Python.SkyQuery.getDatasetInfo" + + schemaURL = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -318,7 +370,14 @@ def listDatasetTables(datasetName="MyDB"): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables' + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.listDatasetTables" + else: + taskName = "SciScript-Python.SkyQuery.listDatasetTables" + + url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -348,7 +407,14 @@ def getTableInfo(tableName, datasetName="MyDB"): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables/' + tableName + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.getTableInfo" + else: + taskName = "SciScript-Python.SkyQuery.getTableInfo" + + url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables/' + tableName + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -377,7 +443,14 @@ def listTableColumns(tableName, datasetName="MyDB"): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables/' + tableName + '/columns' + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.listTableColumns" + else: + taskName = "SciScript-Python.SkyQuery.listTableColumns" + + url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables/' + tableName + '/columns' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -412,7 +485,13 @@ def getTable(tableName, datasetName="MyDB", top = None): token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.getTable" + else: + taskName = "SciScript-Python.SkyQuery.getTable" + + url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + "?TaskName=" + taskName if top != None and top != "": url = url + '?top=' + str(top) @@ -443,7 +522,14 @@ def dropTable(tableName, datasetName="MyDB"): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.dropTable" + else: + taskName = "SciScript-Python.SkyQuery.dropTable" + + url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -474,7 +560,14 @@ def uploadTable(uploadData, tableName, datasetName="MyDB", format="csv"): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.uploadTable" + else: + taskName = "SciScript-Python.SkyQuery.uploadTable" + + url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + "?TaskName=" + taskName ctype = "" if format == "csv": ctype = 'text/csv' diff --git a/py3/SciServer/Authentication.py b/py3/SciServer/Authentication.py index 4610b49..e15fadb 100644 --- a/py3/SciServer/Authentication.py +++ b/py3/SciServer/Authentication.py @@ -38,9 +38,16 @@ def getKeystoneUserWithToken(token): .. seealso:: Authentication.getToken, Authentication.login, Authentication.setToken. """ - loginURL = Config.AuthenticationURL + taskName = "" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Authentication.getKeystoneUserWithToken" + else: + taskName = "SciScript-Python.Authentication.getKeystoneUserWithToken" + + loginURL = Config.AuthenticationURL + "?TaskName=" + taskName; if ~loginURL.endswith("/"): loginURL = loginURL + "/" + loginURL = loginURL + token getResponse = requests.get(loginURL) @@ -72,7 +79,13 @@ def login(UserName, Password): .. seealso:: Authentication.getKeystoneUserWithToken, Authentication.getToken, Authentication.setToken, Authentication.token. """ - loginURL = Config.AuthenticationURL + taskName = "" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Authentication.login" + else: + taskName = "SciScript-Python.Authentication.login" + + loginURL = Config.AuthenticationURL + "?TaskName=" + taskName authJson = {"auth":{"identity":{"password":{"user":{"name":UserName,"password":Password}}}}} diff --git a/py3/SciServer/CasJobs.py b/py3/SciServer/CasJobs.py index 6d28ff3..f6d6f42 100644 --- a/py3/SciServer/CasJobs.py +++ b/py3/SciServer/CasJobs.py @@ -24,7 +24,14 @@ def getSchemaName(): if token is not None and token != "": keystoneUserId = Authentication.getKeystoneUserWithToken(token).id - usersUrl = Config.CasJobsRESTUri + "/users/" + keystoneUserId + + taskName = "" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.getSchemaName" + else: + taskName = "SciScript-Python.CasJobs.getSchemaName" + + usersUrl = Config.CasJobsRESTUri + "/users/" + keystoneUserId + "?TaskName=" + taskName headers={'X-Auth-Token': token,'Content-Type': 'application/json'} getResponse = requests.get(usersUrl,headers=headers) if getResponse.status_code != 200: @@ -51,7 +58,13 @@ def getTables(context="MyDB"): token = Authentication.getToken() if token is not None and token != "": - TablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables" + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.getTables" + else: + taskName = "SciScript-Python.CasJobs.getTables" + + TablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables" + "?TaskName=" + taskName headers={'X-Auth-Token': token,'Content-Type': 'application/json'} @@ -100,15 +113,15 @@ def executeQuery(sql, context="MyDB", format="pandas"): else: raise Exception("Error when executing query. Illegal format parameter specification: " + str(format)); - QueryUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/query" - - TaskName = ""; + taskName = ""; if Config.isSciServerComputeEnvironment(): - TaskName = "Compute.SciScript-Python.CasJobs.executeQuery" + taskName = "Compute.SciScript-Python.CasJobs.executeQuery" else: - TaskName = "SciScript-Python.CasJobs.executeQuery" + taskName = "SciScript-Python.CasJobs.executeQuery" + + QueryUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/query" + "?TaskName=" + taskName - query = {"Query": sql, "TaskName": TaskName} + query = {"Query": sql, "TaskName": taskName} data = json.dumps(query).encode() @@ -154,15 +167,15 @@ def submitJob(sql, context="MyDB"): token = Authentication.getToken() if token is not None and token != "": - QueryUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/jobs" - - TaskName = ""; + taskName = ""; if Config.isSciServerComputeEnvironment(): - TaskName = "Compute.SciScript-Python.CasJobs.submitJob" + taskName = "Compute.SciScript-Python.CasJobs.submitJob" else: - TaskName = "SciScript-Python.CasJobs.submitJob" + taskName = "SciScript-Python.CasJobs.submitJob" + + QueryUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/jobs" + "?TaskName=" + taskName - query = {"Query": sql, "TaskName": TaskName} + query = {"Query": sql, "TaskName": taskName} data = json.dumps(query).encode() @@ -193,7 +206,13 @@ def getJobStatus(jobId): token = Authentication.getToken() if token is not None and token != "": - QueryUrl = Config.CasJobsRESTUri + "/jobs/" + str(jobId) + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.getJobStatus" + else: + taskName = "SciScript-Python.CasJobs.getJobStatus" + + QueryUrl = Config.CasJobsRESTUri + "/jobs/" + str(jobId) + "?TaskName=" + taskName headers={'X-Auth-Token': token,'Content-Type': 'application/json'} @@ -220,7 +239,13 @@ def cancelJob(jobId): token = Authentication.getToken() if token is not None and token != "": - QueryUrl = Config.CasJobsRESTUri + "/jobs/" + str(jobId) + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.cancelJob" + else: + taskName = "SciScript-Python.CasJobs.cancelJob" + + QueryUrl = Config.CasJobsRESTUri + "/jobs/" + str(jobId) + "?TaskName=" + taskName headers={'X-Auth-Token': token,'Content-Type': 'application/json'} @@ -387,7 +412,14 @@ def uploadCSVDataToTable(csvData, tableName, context="MyDB"): #if (Config.executeMode == "debug"): # print "Uploading ", sys.getsizeof(CVSdata), "bytes..." - tablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables/" + tableName + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.uploadCSVDataToTable" + else: + taskName = "SciScript-Python.CasJobs.uploadCSVDataToTable" + + tablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables/" + tableName + "?TaskName=" + taskName headers={} headers['X-Auth-Token']= token diff --git a/py3/SciServer/SciDrive.py b/py3/SciServer/SciDrive.py index b3df2db..4f1bde2 100644 --- a/py3/SciServer/SciDrive.py +++ b/py3/SciServer/SciDrive.py @@ -25,7 +25,13 @@ def createContainer(path): 'uri="vos://' + Config.SciDriveHost + '!vospace/' + path + '">' '' '') - url = Config.SciDriveHost + '/vospace-2.0/nodes/' + path + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.createContainer" + else: + taskName = "SciScript-Python.SciDrive.createContainer" + + url = Config.SciDriveHost + '/vospace-2.0/nodes/' + path + "?TaskName=" + taskName; data = str.encode(containerBody) headers = {'X-Auth-Token': token, 'Content-Type': 'application/xml'} res = requests.put(url, data=data, headers=headers) @@ -52,7 +58,14 @@ def upload(path, data="", localFilePath=""): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SciDriveHost + '/vospace-2.0/1/files_put/dropbox/' + path + + taskName = "" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.upload" + else: + taskName = "SciScript-Python.SciDrive.upload" + + url = Config.SciDriveHost + '/vospace-2.0/1/files_put/dropbox/' + path + "?TaskName=" + taskName; headers = {'X-Auth-Token': token} if(localFilePath != ""): with open(localFilePath, "rb") as file: @@ -85,7 +98,13 @@ def publicUrl(path): token = Authentication.getToken() if token is not None and token != "": - url = Config.SciDriveHost + '/vospace-2.0/1/media/sandbox/' + str(path) + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.publicUrl" + else: + taskName = "SciScript-Python.SciDrive.publicUrl" + + url = Config.SciDriveHost + '/vospace-2.0/1/media/sandbox/' + str(path) + "?TaskName=" + taskName headers = {'X-Auth-Token': token} res = requests.get(url, headers=headers) if res.status_code != 200: @@ -113,7 +132,13 @@ def directoryList(path=""): token = Authentication.getToken() if token is not None and token != "": - url = Config.SciDriveHost + "/vospace-2.0/1/metadata/sandbox/" + str(path) + "?list=True&path=" + str(path) + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.directoryList" + else: + taskName = "SciScript-Python.SciDrive.directoryList" + + url = Config.SciDriveHost + "/vospace-2.0/1/metadata/sandbox/" + str(path) + "?list=True&path=" + str(path) + "&TaskName=" + taskName; headers = {'X-Auth-Token': token} res = requests.get(url, headers=headers) if res.status_code != 200: @@ -143,7 +168,13 @@ def download(path, format="text", localFilePath=""): token = Authentication.getToken() if token is not None and token != "": - fileUrl = publicUrl(path) + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.download" + else: + taskName = "SciScript-Python.SciDrive.download" + + fileUrl = publicUrl(path) + "?TaskName=" + taskName res = requests.get(fileUrl, stream=True) if res.status_code != 200: raise Exception("Error when downloading SciDrive file " + str(path) + ".\nHttp Response from SciDrive API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); @@ -194,7 +225,13 @@ def delete(path): 'uri="vos://' + Config.SciDriveHost + '!vospace/' + path + '">' '' '') - url = Config.SciDriveHost + '/vospace-2.0/nodes/' + path + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.delete" + else: + taskName = "SciScript-Python.SciDrive.delete" + + url = Config.SciDriveHost + '/vospace-2.0/nodes/' + path + "?TaskName=" + taskName; data = str.encode(containerBody) headers = {'X-Auth-Token': token, 'Content-Type': 'application/xml'} res = requests.delete(url, data=data, headers=headers) diff --git a/py3/SciServer/SkyQuery.py b/py3/SciServer/SkyQuery.py index 35ecee1..9b7b06a 100644 --- a/py3/SciServer/SkyQuery.py +++ b/py3/SciServer/SkyQuery.py @@ -27,7 +27,14 @@ def getJobStatus(jobId): """ token = Authentication.getToken() if token is not None and token != "": - statusURL = Config.SkyQueryUrl + '/Jobs.svc/jobs/' + str(jobId) + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.getJobStatus" + else: + taskName = "SciScript-Python.SkyQuery.getJobStatus" + + statusURL = Config.SkyQueryUrl + '/Jobs.svc/jobs/' + str(jobId) + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -57,7 +64,13 @@ def cancelJob(jobId): token = Authentication.getToken() if token is not None and token != "": - statusURL = Config.SkyQueryUrl + '/Jobs.svc/jobs/' + jobId + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.cancelJob" + else: + taskName = "SciScript-Python.SkyQuery.cancelJob" + + statusURL = Config.SkyQueryUrl + '/Jobs.svc/jobs/' + jobId + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -93,7 +106,14 @@ def listQueues(): """ token = Authentication.getToken() if token is not None and token != "": - jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues' + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.listQueues" + else: + taskName = "SciScript-Python.SkyQuery.listQueues" + + jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -123,7 +143,13 @@ def getQueueInfo(queue): token = Authentication.getToken() if token is not None and token != "": - jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.getQueueInfo" + else: + taskName = "SciScript-Python.SkyQuery.getQueueInfo" + + jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -154,7 +180,13 @@ def submitJob(query, queue='quick'): token = Authentication.getToken() if token is not None and token != "": - jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + '/jobs' + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.submitJob" + else: + taskName = "SciScript-Python.SkyQuery.submitJob" + + jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + '/jobs' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -226,7 +258,14 @@ def listJobs(queue="quick"): """ token = Authentication.getToken() if token is not None and token != "": - jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + '/jobs?' + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.listJobs" + else: + taskName = "SciScript-Python.SkyQuery.listJobs" + + jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + '/jobs' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -261,7 +300,13 @@ def listAllDatasets(): token = Authentication.getToken() if token is not None and token != "": - schemaURL = Config.SkyQueryUrl + '/Schema.svc/datasets' + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.listAllDatasets" + else: + taskName = "SciScript-Python.SkyQuery.listAllDatasets" + + schemaURL = Config.SkyQueryUrl + '/Schema.svc/datasets' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -290,7 +335,14 @@ def getDatasetInfo(datasetName="MyDB"): """ token = Authentication.getToken() if token is not None and token != "": - schemaURL = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.getDatasetInfo" + else: + taskName = "SciScript-Python.SkyQuery.getDatasetInfo" + + schemaURL = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -318,7 +370,14 @@ def listDatasetTables(datasetName="MyDB"): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables' + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.listDatasetTables" + else: + taskName = "SciScript-Python.SkyQuery.listDatasetTables" + + url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -348,7 +407,14 @@ def getTableInfo(tableName, datasetName="MyDB"): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables/' + tableName + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.getTableInfo" + else: + taskName = "SciScript-Python.SkyQuery.getTableInfo" + + url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables/' + tableName + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -377,7 +443,14 @@ def listTableColumns(tableName, datasetName="MyDB"): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables/' + tableName + '/columns' + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.listTableColumns" + else: + taskName = "SciScript-Python.SkyQuery.listTableColumns" + + url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables/' + tableName + '/columns' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -412,7 +485,13 @@ def getTable(tableName, datasetName="MyDB", top = None): token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.getTable" + else: + taskName = "SciScript-Python.SkyQuery.getTable" + + url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + "?TaskName=" + taskName if top != None and top != "": url = url + '?top=' + str(top) @@ -443,7 +522,14 @@ def dropTable(tableName, datasetName="MyDB"): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.dropTable" + else: + taskName = "SciScript-Python.SkyQuery.dropTable" + + url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -474,7 +560,14 @@ def uploadTable(uploadData, tableName, datasetName="MyDB", format="csv"): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.uploadTable" + else: + taskName = "SciScript-Python.SkyQuery.uploadTable" + + url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + "?TaskName=" + taskName ctype = "" if format == "csv": ctype = 'text/csv' From 4398a1abf2060c923e167288d773476295a25fab Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Wed, 5 Jul 2017 14:35:30 -0400 Subject: [PATCH 14/79] Fixed token refresh --- docs_sphinx/conf.py | 4 +- py2/SciServer/Authentication.py | 86 +++++++++++++++++++++------------ py2/SciServer/Config.py | 2 +- py2/SciServer/__init__.py | 2 +- py2/setup.py | 2 +- py3/SciServer/Authentication.py | 86 +++++++++++++++++++++------------ py3/SciServer/Config.py | 2 +- py3/SciServer/__init__.py | 2 +- py3/setup.py | 2 +- 9 files changed, 116 insertions(+), 72 deletions(-) diff --git a/docs_sphinx/conf.py b/docs_sphinx/conf.py index 0d13936..7ddf897 100644 --- a/docs_sphinx/conf.py +++ b/docs_sphinx/conf.py @@ -55,9 +55,9 @@ # built documents. # # The short X.Y version. -version = '1.10.0' +version = '1.10.2' # The full version, including alpha/beta/rc tags. -release = '1.10.0' +release = '1.10.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/py2/SciServer/Authentication.py b/py2/SciServer/Authentication.py index 4610b49..1b81498 100644 --- a/py2/SciServer/Authentication.py +++ b/py2/SciServer/Authentication.py @@ -101,29 +101,49 @@ def getToken(): """ try: - if token.value is not None: - return token.value + + if Config.isSciServerComputeEnvironment(): + tokenFile = Config.KeystoneTokenPath; # '/home/idies/keystone.token' + if os.path.isfile(tokenFile): + with open(tokenFile, 'r') as f: + _token = f.read().rstrip('\n') + if _token is not None and _token != "": + token.value = _token; + + found = False + ident = identArgIdentifier() + for arg in sys.argv: + if (arg.startswith(ident)): + sys.argv.remove(arg) + sys.argv.append(ident + _token) + found = True + if not found: + sys.argv.append(ident + _token) + + return _token + else: + warnings.warn("In Authentication.getToken: Cannot find token in system token file " + str(Config.KeystoneTokenPath) + ".", Warning, stacklevel=2) + return None; + else: + warnings.warn("In Authentication.getToken: Cannot find system token file " + str(Config.KeystoneTokenPath) + ".", Warning, stacklevel=2) + return None; else: - _token = None - ident = identArgIdentifier() - for arg in sys.argv: - if (arg.startswith(ident)): - _token = arg[len(ident):] - - if _token is not None and _token != "": - setToken(_token) - return _token + if token.value is not None: + return token.value else: - tokenFile = Config.KeystoneTokenPath; # '/home/idies/keystone.token' - if os.path.isfile(tokenFile) : - with open(tokenFile, 'r') as f: - _token = f.read().rstrip('\n') - if _token is not None and _token != "": - setToken(_token) - return _token - else: - warnings.warn("In Authentication.getToken: Keystone token is not defined. Returning a None token value. Keystone token is not stored in the command line argument --ident, nor in the system token file " + str(Config.KeystoneTokenPath) + ".", Warning, stacklevel=2) - return None; + _token = None + ident = identArgIdentifier() + for arg in sys.argv: + if (arg.startswith(ident)): + _token = arg[len(ident):] + + if _token is not None and _token != "": + token.value = _token + return _token + else: + warnings.warn("In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.", Warning, stacklevel=2) + return None; + except Exception as e: raise e; @@ -142,18 +162,20 @@ def setToken(_token): if _token == "": warnings.warn("Authentication token is being set as an empty string.", Warning, stacklevel=2) - token.value = _token; - - found = False - ident = identArgIdentifier() - for arg in sys.argv: - if (arg.startswith(ident)): - sys.argv.remove(arg) + if Config.isSciServerComputeEnvironment(): + warnings.warn("Authentication token cannot be set to arbitary value when inside SciServer-Compute environment.", Warning, stacklevel=2) + else: + token.value = _token; + + found = False + ident = identArgIdentifier() + for arg in sys.argv: + if (arg.startswith(ident)): + sys.argv.remove(arg) + sys.argv.append(ident + _token) + found = True + if not found: sys.argv.append(ident + _token) - found = True - if not found: - sys.argv.append(ident + _token) - def identArgIdentifier(): diff --git a/py2/SciServer/Config.py b/py2/SciServer/Config.py index 36981ec..e8b5579 100644 --- a/py2/SciServer/Config.py +++ b/py2/SciServer/Config.py @@ -26,7 +26,7 @@ SkyServerWSurl = "https://skyserver.sdss.org" DataRelease = "DR13" KeystoneTokenPath = "/home/idies/keystone.token" #this path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment -version = "sciserver-v1.10.0" #sciserver release version +version = "sciserver-v1.10.2" #sciserver release version diff --git a/py2/SciServer/__init__.py b/py2/SciServer/__init__.py index a7f4476..f31b9b0 100644 --- a/py2/SciServer/__init__.py +++ b/py2/SciServer/__init__.py @@ -31,7 +31,7 @@ **Authors**: Gerard Lemson , Manuchehr Taghizadeh-Popp -**Version**: sciserver-v1.10.0 +**Version**: sciserver-v1.10.2 """ diff --git a/py2/setup.py b/py2/setup.py index 33d3959..424046a 100644 --- a/py2/setup.py +++ b/py2/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages setup( name = "SciServer", - version = "1.10.0", + version = "1.10.2", packages = find_packages(), ) \ No newline at end of file diff --git a/py3/SciServer/Authentication.py b/py3/SciServer/Authentication.py index 4610b49..1b81498 100644 --- a/py3/SciServer/Authentication.py +++ b/py3/SciServer/Authentication.py @@ -101,29 +101,49 @@ def getToken(): """ try: - if token.value is not None: - return token.value + + if Config.isSciServerComputeEnvironment(): + tokenFile = Config.KeystoneTokenPath; # '/home/idies/keystone.token' + if os.path.isfile(tokenFile): + with open(tokenFile, 'r') as f: + _token = f.read().rstrip('\n') + if _token is not None and _token != "": + token.value = _token; + + found = False + ident = identArgIdentifier() + for arg in sys.argv: + if (arg.startswith(ident)): + sys.argv.remove(arg) + sys.argv.append(ident + _token) + found = True + if not found: + sys.argv.append(ident + _token) + + return _token + else: + warnings.warn("In Authentication.getToken: Cannot find token in system token file " + str(Config.KeystoneTokenPath) + ".", Warning, stacklevel=2) + return None; + else: + warnings.warn("In Authentication.getToken: Cannot find system token file " + str(Config.KeystoneTokenPath) + ".", Warning, stacklevel=2) + return None; else: - _token = None - ident = identArgIdentifier() - for arg in sys.argv: - if (arg.startswith(ident)): - _token = arg[len(ident):] - - if _token is not None and _token != "": - setToken(_token) - return _token + if token.value is not None: + return token.value else: - tokenFile = Config.KeystoneTokenPath; # '/home/idies/keystone.token' - if os.path.isfile(tokenFile) : - with open(tokenFile, 'r') as f: - _token = f.read().rstrip('\n') - if _token is not None and _token != "": - setToken(_token) - return _token - else: - warnings.warn("In Authentication.getToken: Keystone token is not defined. Returning a None token value. Keystone token is not stored in the command line argument --ident, nor in the system token file " + str(Config.KeystoneTokenPath) + ".", Warning, stacklevel=2) - return None; + _token = None + ident = identArgIdentifier() + for arg in sys.argv: + if (arg.startswith(ident)): + _token = arg[len(ident):] + + if _token is not None and _token != "": + token.value = _token + return _token + else: + warnings.warn("In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.", Warning, stacklevel=2) + return None; + except Exception as e: raise e; @@ -142,18 +162,20 @@ def setToken(_token): if _token == "": warnings.warn("Authentication token is being set as an empty string.", Warning, stacklevel=2) - token.value = _token; - - found = False - ident = identArgIdentifier() - for arg in sys.argv: - if (arg.startswith(ident)): - sys.argv.remove(arg) + if Config.isSciServerComputeEnvironment(): + warnings.warn("Authentication token cannot be set to arbitary value when inside SciServer-Compute environment.", Warning, stacklevel=2) + else: + token.value = _token; + + found = False + ident = identArgIdentifier() + for arg in sys.argv: + if (arg.startswith(ident)): + sys.argv.remove(arg) + sys.argv.append(ident + _token) + found = True + if not found: sys.argv.append(ident + _token) - found = True - if not found: - sys.argv.append(ident + _token) - def identArgIdentifier(): diff --git a/py3/SciServer/Config.py b/py3/SciServer/Config.py index 36981ec..e8b5579 100644 --- a/py3/SciServer/Config.py +++ b/py3/SciServer/Config.py @@ -26,7 +26,7 @@ SkyServerWSurl = "https://skyserver.sdss.org" DataRelease = "DR13" KeystoneTokenPath = "/home/idies/keystone.token" #this path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment -version = "sciserver-v1.10.0" #sciserver release version +version = "sciserver-v1.10.2" #sciserver release version diff --git a/py3/SciServer/__init__.py b/py3/SciServer/__init__.py index a7f4476..f31b9b0 100644 --- a/py3/SciServer/__init__.py +++ b/py3/SciServer/__init__.py @@ -31,7 +31,7 @@ **Authors**: Gerard Lemson , Manuchehr Taghizadeh-Popp -**Version**: sciserver-v1.10.0 +**Version**: sciserver-v1.10.2 """ diff --git a/py3/setup.py b/py3/setup.py index 33d3959..424046a 100644 --- a/py3/setup.py +++ b/py3/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages setup( name = "SciServer", - version = "1.10.0", + version = "1.10.2", packages = find_packages(), ) \ No newline at end of file From 64c993535c7276b359f5a02b3e1b2c2fcdb271dc Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Tue, 15 Aug 2017 18:27:34 -0400 Subject: [PATCH 15/79] fixed taskname --- py2/SciServer/CasJobs.py | 57 +++++++++++++++++++++++++++++---- py2/SciServer/SciDrive.py | 25 +++++++++++---- py3/SciServer/Authentication.py | 6 ++-- py3/SciServer/CasJobs.py | 55 +++++++++++++++++++++++++++---- py3/SciServer/SciDrive.py | 25 +++++++++++---- 5 files changed, 140 insertions(+), 28 deletions(-) diff --git a/py2/SciServer/CasJobs.py b/py2/SciServer/CasJobs.py index 966ee06..797d366 100644 --- a/py2/SciServer/CasJobs.py +++ b/py2/SciServer/CasJobs.py @@ -10,6 +10,16 @@ from SciServer import Authentication, Config +class Task: + """ + The class TaskName stores the name of the task that executes the API call. + """ + name = None + + +task = Task(); + + def getSchemaName(): """ Returns the WebServiceID that identifies the schema for a user in MyScratch database with CasJobs. @@ -114,10 +124,14 @@ def executeQuery(sql, context="MyDB", format="pandas"): raise Exception("Error when executing query. Illegal format parameter specification: " + str(format)); taskName = ""; - if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.CasJobs.executeQuery" + if task.name is not None: + taskName = task.name; + task.name = None; else: - taskName = "SciScript-Python.CasJobs.executeQuery" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.executeQuery" + else: + taskName = "SciScript-Python.CasJobs.executeQuery" QueryUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/query" + "?TaskName=" + taskName @@ -298,7 +312,7 @@ def waitForJob(jobId, verbose=True): raise e; -def writeFitsFileFromQuery(fileName, queryString, context="MyDB"): +def writeFitsFileFromQuery(fileName, queryString, context="MyDB"): """ Executes a quick CasJobs query and writes the result to a local Fits file (http://www.stsci.edu/institute/software_hardware/pyfits). @@ -312,6 +326,13 @@ def writeFitsFileFromQuery(fileName, queryString, context="MyDB"): .. seealso:: CasJobs.submitJob, CasJobs.getJobStatus, CasJobs.executeQuery, CasJobs.getPandasDataFrameFromQuery, CasJobs.getNumpyArrayFromQuery """ try: + + + if Config.isSciServerComputeEnvironment(): + task.name = "Compute.SciScript-Python.CasJobs.writeFitsFileFromQuery" + else: + task.name = "SciScript-Python.CasJobs.writeFitsFileFromQuery" + bytesio = executeQuery(queryString, context=context, format="fits") theFile = open(fileName, "w+b") @@ -337,6 +358,12 @@ def getPandasDataFrameFromQuery(queryString, context="MyDB"): .. seealso:: CasJobs.submitJob, CasJobs.getJobStatus, CasJobs.executeQuery, CasJobs.writeFitsFileFromQuery, CasJobs.getNumpyArrayFromQuery """ try: + + if Config.isSciServerComputeEnvironment(): + task.name = "Compute.SciScript-Python.CasJobs.getPandasDataFrameFromQuery" + else: + task.name = "SciScript-Python.CasJobs.getPandasDataFrameFromQuery" + cvsResponse = executeQuery(queryString, context=context,format="readable") #if the index column is not specified then it will add it's own column which causes problems when uploading the transformed data @@ -362,7 +389,13 @@ def getNumpyArrayFromQuery(queryString, context="MyDB"): """ try: + if Config.isSciServerComputeEnvironment(): + task.name = "Compute.SciScript-Python.CasJobs.getNumpyArrayFromQuery" + else: + task.name = "SciScript-Python.CasJobs.getNumpyArrayFromQuery" + dataFrame = getPandasDataFrameFromQuery(queryString, context) + return dataFrame.as_matrix() except Exception as e: @@ -384,6 +417,12 @@ def uploadPandasDataFrameToTable(dataFrame, tableName, context="MyDB"): .. seealso:: CasJobs.uploadCSVDataToTable """ try: + + if Config.isSciServerComputeEnvironment(): + task.name = "Compute.SciScript-Python.CasJobs.uploadPandasDataFrameToTable" + else: + task.name = "SciScript-Python.CasJobs.uploadPandasDataFrameToTable" + if dataFrame.index.name is not None and dataFrame.index.name != "": sio = dataFrame.to_csv().encode("utf8") else: @@ -414,10 +453,14 @@ def uploadCSVDataToTable(csvData, tableName, context="MyDB"): # print "Uploading ", sys.getsizeof(CVSdata), "bytes..." taskName = ""; - if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.CasJobs.uploadCSVDataToTable" + if task.name is not None: + taskName = task.name; + task.name = None; else: - taskName = "SciScript-Python.CasJobs.uploadCSVDataToTable" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.uploadCSVDataToTable" + else: + taskName = "SciScript-Python.CasJobs.uploadCSVDataToTable" tablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables/" + tableName + "?TaskName=" + taskName diff --git a/py2/SciServer/SciDrive.py b/py2/SciServer/SciDrive.py index 4f1bde2..523e02d 100644 --- a/py2/SciServer/SciDrive.py +++ b/py2/SciServer/SciDrive.py @@ -7,6 +7,15 @@ from SciServer import Config, Authentication +class Task: + """ + The class TaskName stores the name of the task that executes the API call. + """ + name = None + + +task = Task(); + def createContainer(path): """ Creates a container (directory) in SciDrive @@ -99,10 +108,14 @@ def publicUrl(path): if token is not None and token != "": taskName = ""; - if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.SciDrive.publicUrl" + if task.name is not None: + taskName = task.name; + task.name = None; else: - taskName = "SciScript-Python.SciDrive.publicUrl" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.publicUrl" + else: + taskName = "SciScript-Python.SciDrive.publicUrl" url = Config.SciDriveHost + '/vospace-2.0/1/media/sandbox/' + str(path) + "?TaskName=" + taskName headers = {'X-Auth-Token': token} @@ -170,11 +183,11 @@ def download(path, format="text", localFilePath=""): taskName = ""; if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.SciDrive.download" + task.name = "Compute.SciScript-Python.SciDrive.download" else: - taskName = "SciScript-Python.SciDrive.download" + task.name = "SciScript-Python.SciDrive.download" - fileUrl = publicUrl(path) + "?TaskName=" + taskName + fileUrl = publicUrl(path) res = requests.get(fileUrl, stream=True) if res.status_code != 200: raise Exception("Error when downloading SciDrive file " + str(path) + ".\nHttp Response from SciDrive API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); diff --git a/py3/SciServer/Authentication.py b/py3/SciServer/Authentication.py index e15fadb..9c6622d 100644 --- a/py3/SciServer/Authentication.py +++ b/py3/SciServer/Authentication.py @@ -43,12 +43,12 @@ def getKeystoneUserWithToken(token): taskName = "Compute.SciScript-Python.Authentication.getKeystoneUserWithToken" else: taskName = "SciScript-Python.Authentication.getKeystoneUserWithToken" - - loginURL = Config.AuthenticationURL + "?TaskName=" + taskName; + + loginURL = Config.AuthenticationURL if ~loginURL.endswith("/"): loginURL = loginURL + "/" - loginURL = loginURL + token + loginURL = loginURL + token + "?TaskName=" + taskName; getResponse = requests.get(loginURL) if getResponse.status_code != 200: diff --git a/py3/SciServer/CasJobs.py b/py3/SciServer/CasJobs.py index f6d6f42..d02dc71 100644 --- a/py3/SciServer/CasJobs.py +++ b/py3/SciServer/CasJobs.py @@ -10,6 +10,16 @@ from SciServer import Authentication, Config +class Task: + """ + The class TaskName stores the name of the task that executes the API call. + """ + name = None + + +task = Task(); + + def getSchemaName(): """ Returns the WebServiceID that identifies the schema for a user in MyScratch database with CasJobs. @@ -114,10 +124,14 @@ def executeQuery(sql, context="MyDB", format="pandas"): raise Exception("Error when executing query. Illegal format parameter specification: " + str(format)); taskName = ""; - if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.CasJobs.executeQuery" + if task.name is not None: + taskName = task.name; + task.name = None; else: - taskName = "SciScript-Python.CasJobs.executeQuery" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.executeQuery" + else: + taskName = "SciScript-Python.CasJobs.executeQuery" QueryUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/query" + "?TaskName=" + taskName @@ -312,6 +326,13 @@ def writeFitsFileFromQuery(fileName, queryString, context="MyDB"): .. seealso:: CasJobs.submitJob, CasJobs.getJobStatus, CasJobs.executeQuery, CasJobs.getPandasDataFrameFromQuery, CasJobs.getNumpyArrayFromQuery """ try: + + + if Config.isSciServerComputeEnvironment(): + task.name = "Compute.SciScript-Python.CasJobs.writeFitsFileFromQuery" + else: + task.name = "SciScript-Python.CasJobs.writeFitsFileFromQuery" + bytesio = executeQuery(queryString, context=context, format="fits") theFile = open(fileName, "w+b") @@ -337,6 +358,12 @@ def getPandasDataFrameFromQuery(queryString, context="MyDB"): .. seealso:: CasJobs.submitJob, CasJobs.getJobStatus, CasJobs.executeQuery, CasJobs.writeFitsFileFromQuery, CasJobs.getNumpyArrayFromQuery """ try: + + if Config.isSciServerComputeEnvironment(): + task.name = "Compute.SciScript-Python.CasJobs.getPandasDataFrameFromQuery" + else: + task.name = "SciScript-Python.CasJobs.getPandasDataFrameFromQuery" + cvsResponse = executeQuery(queryString, context=context,format="readable") #if the index column is not specified then it will add it's own column which causes problems when uploading the transformed data @@ -362,7 +389,13 @@ def getNumpyArrayFromQuery(queryString, context="MyDB"): """ try: + if Config.isSciServerComputeEnvironment(): + task.name = "Compute.SciScript-Python.CasJobs.getNumpyArrayFromQuery" + else: + task.name = "SciScript-Python.CasJobs.getNumpyArrayFromQuery" + dataFrame = getPandasDataFrameFromQuery(queryString, context) + return dataFrame.as_matrix() except Exception as e: @@ -384,6 +417,12 @@ def uploadPandasDataFrameToTable(dataFrame, tableName, context="MyDB"): .. seealso:: CasJobs.uploadCSVDataToTable """ try: + + if Config.isSciServerComputeEnvironment(): + task.name = "Compute.SciScript-Python.CasJobs.uploadPandasDataFrameToTable" + else: + task.name = "SciScript-Python.CasJobs.uploadPandasDataFrameToTable" + if dataFrame.index.name is not None and dataFrame.index.name != "": sio = dataFrame.to_csv().encode("utf8") else: @@ -414,10 +453,14 @@ def uploadCSVDataToTable(csvData, tableName, context="MyDB"): # print "Uploading ", sys.getsizeof(CVSdata), "bytes..." taskName = ""; - if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.CasJobs.uploadCSVDataToTable" + if task.name is not None: + taskName = task.name; + task.name = None; else: - taskName = "SciScript-Python.CasJobs.uploadCSVDataToTable" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.uploadCSVDataToTable" + else: + taskName = "SciScript-Python.CasJobs.uploadCSVDataToTable" tablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables/" + tableName + "?TaskName=" + taskName diff --git a/py3/SciServer/SciDrive.py b/py3/SciServer/SciDrive.py index 4f1bde2..523e02d 100644 --- a/py3/SciServer/SciDrive.py +++ b/py3/SciServer/SciDrive.py @@ -7,6 +7,15 @@ from SciServer import Config, Authentication +class Task: + """ + The class TaskName stores the name of the task that executes the API call. + """ + name = None + + +task = Task(); + def createContainer(path): """ Creates a container (directory) in SciDrive @@ -99,10 +108,14 @@ def publicUrl(path): if token is not None and token != "": taskName = ""; - if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.SciDrive.publicUrl" + if task.name is not None: + taskName = task.name; + task.name = None; else: - taskName = "SciScript-Python.SciDrive.publicUrl" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.publicUrl" + else: + taskName = "SciScript-Python.SciDrive.publicUrl" url = Config.SciDriveHost + '/vospace-2.0/1/media/sandbox/' + str(path) + "?TaskName=" + taskName headers = {'X-Auth-Token': token} @@ -170,11 +183,11 @@ def download(path, format="text", localFilePath=""): taskName = ""; if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.SciDrive.download" + task.name = "Compute.SciScript-Python.SciDrive.download" else: - taskName = "SciScript-Python.SciDrive.download" + task.name = "SciScript-Python.SciDrive.download" - fileUrl = publicUrl(path) + "?TaskName=" + taskName + fileUrl = publicUrl(path) res = requests.get(fileUrl, stream=True) if res.status_code != 200: raise Exception("Error when downloading SciDrive file " + str(path) + ".\nHttp Response from SciDrive API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); From dda1666c591f2857af97a1dafbfd256f3f0795e3 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Tue, 15 Aug 2017 18:33:17 -0400 Subject: [PATCH 16/79] update getToken() --- py2/SciServer/Authentication.py | 62 ++++++++++++++++++++++----------- py3/SciServer/Authentication.py | 62 ++++++++++++++++++++++----------- 2 files changed, 82 insertions(+), 42 deletions(-) diff --git a/py2/SciServer/Authentication.py b/py2/SciServer/Authentication.py index 41511a8..e95cda4 100644 --- a/py2/SciServer/Authentication.py +++ b/py2/SciServer/Authentication.py @@ -114,29 +114,49 @@ def getToken(): """ try: - if token.value is not None: - return token.value + + if Config.isSciServerComputeEnvironment(): + tokenFile = Config.KeystoneTokenPath; # '/home/idies/keystone.token' + if os.path.isfile(tokenFile): + with open(tokenFile, 'r') as f: + _token = f.read().rstrip('\n') + if _token is not None and _token != "": + token.value = _token; + + found = False + ident = identArgIdentifier() + for arg in sys.argv: + if (arg.startswith(ident)): + sys.argv.remove(arg) + sys.argv.append(ident + _token) + found = True + if not found: + sys.argv.append(ident + _token) + + return _token + else: + warnings.warn("In Authentication.getToken: Cannot find token in system token file " + str(Config.KeystoneTokenPath) + ".", Warning, stacklevel=2) + return None; + else: + warnings.warn("In Authentication.getToken: Cannot find system token file " + str(Config.KeystoneTokenPath) + ".", Warning, stacklevel=2) + return None; else: - _token = None - ident = identArgIdentifier() - for arg in sys.argv: - if (arg.startswith(ident)): - _token = arg[len(ident):] - - if _token is not None and _token != "": - setToken(_token) - return _token + if token.value is not None: + return token.value else: - tokenFile = Config.KeystoneTokenPath; # '/home/idies/keystone.token' - if os.path.isfile(tokenFile) : - with open(tokenFile, 'r') as f: - _token = f.read().rstrip('\n') - if _token is not None and _token != "": - setToken(_token) - return _token - else: - warnings.warn("In Authentication.getToken: Keystone token is not defined. Returning a None token value. Keystone token is not stored in the command line argument --ident, nor in the system token file " + str(Config.KeystoneTokenPath) + ".", Warning, stacklevel=2) - return None; + _token = None + ident = identArgIdentifier() + for arg in sys.argv: + if (arg.startswith(ident)): + _token = arg[len(ident):] + + if _token is not None and _token != "": + token.value = _token + return _token + else: + warnings.warn("In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.", Warning, stacklevel=2) + return None; + except Exception as e: raise e; diff --git a/py3/SciServer/Authentication.py b/py3/SciServer/Authentication.py index 9c6622d..35a9986 100644 --- a/py3/SciServer/Authentication.py +++ b/py3/SciServer/Authentication.py @@ -114,29 +114,49 @@ def getToken(): """ try: - if token.value is not None: - return token.value + + if Config.isSciServerComputeEnvironment(): + tokenFile = Config.KeystoneTokenPath; # '/home/idies/keystone.token' + if os.path.isfile(tokenFile): + with open(tokenFile, 'r') as f: + _token = f.read().rstrip('\n') + if _token is not None and _token != "": + token.value = _token; + + found = False + ident = identArgIdentifier() + for arg in sys.argv: + if (arg.startswith(ident)): + sys.argv.remove(arg) + sys.argv.append(ident + _token) + found = True + if not found: + sys.argv.append(ident + _token) + + return _token + else: + warnings.warn("In Authentication.getToken: Cannot find token in system token file " + str(Config.KeystoneTokenPath) + ".", Warning, stacklevel=2) + return None; + else: + warnings.warn("In Authentication.getToken: Cannot find system token file " + str(Config.KeystoneTokenPath) + ".", Warning, stacklevel=2) + return None; else: - _token = None - ident = identArgIdentifier() - for arg in sys.argv: - if (arg.startswith(ident)): - _token = arg[len(ident):] - - if _token is not None and _token != "": - setToken(_token) - return _token + if token.value is not None: + return token.value else: - tokenFile = Config.KeystoneTokenPath; # '/home/idies/keystone.token' - if os.path.isfile(tokenFile) : - with open(tokenFile, 'r') as f: - _token = f.read().rstrip('\n') - if _token is not None and _token != "": - setToken(_token) - return _token - else: - warnings.warn("In Authentication.getToken: Keystone token is not defined. Returning a None token value. Keystone token is not stored in the command line argument --ident, nor in the system token file " + str(Config.KeystoneTokenPath) + ".", Warning, stacklevel=2) - return None; + _token = None + ident = identArgIdentifier() + for arg in sys.argv: + if (arg.startswith(ident)): + _token = arg[len(ident):] + + if _token is not None and _token != "": + token.value = _token + return _token + else: + warnings.warn("In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.", Warning, stacklevel=2) + return None; + except Exception as e: raise e; From 130b14c4d637cb17a1915e5d945688f6240422b5 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Fri, 19 Jan 2018 14:55:43 -0500 Subject: [PATCH 17/79] -Adding 'Jobs' and 'Files' modules, which replace 'ComputeJobs' and 'FileService' modules, respectively. -Added a promt for the username and password for the Authentication.login method. --- py2/SciServer/Authentication.py | 20 +- py2/SciServer/ComputeJobs.py | 238 ------------- py2/SciServer/Config.py | 5 +- py2/SciServer/FileService.py | 181 ---------- py2/SciServer/Files.py | 318 +++++++++++++++++ py2/SciServer/Jobs.py | 599 ++++++++++++++++++++++++++++++++ py3/SciServer/Authentication.py | 20 +- py3/SciServer/Config.py | 8 +- py3/SciServer/Files.py | 318 +++++++++++++++++ py3/SciServer/Jobs.py | 599 ++++++++++++++++++++++++++++++++ 10 files changed, 1868 insertions(+), 438 deletions(-) delete mode 100644 py2/SciServer/ComputeJobs.py delete mode 100644 py2/SciServer/FileService.py create mode 100644 py2/SciServer/Files.py create mode 100644 py2/SciServer/Jobs.py create mode 100644 py3/SciServer/Files.py create mode 100644 py3/SciServer/Jobs.py diff --git a/py2/SciServer/Authentication.py b/py2/SciServer/Authentication.py index e95cda4..a8797c2 100644 --- a/py2/SciServer/Authentication.py +++ b/py2/SciServer/Authentication.py @@ -6,7 +6,7 @@ import requests import os.path import warnings - +import getpass from SciServer import Config @@ -63,7 +63,7 @@ def getKeystoneUserWithToken(token): return ksu -def login(UserName, Password): +def login(UserName=None, Password=None): """ Logs the user into SciServer and returns the authentication token. This function is useful when SciScript-Python library methods are executed outside the SciServer-Compute environment. @@ -71,12 +71,11 @@ def login(UserName, Password): so the user has to use Authentication.login in order to log into SciServer manually and get the authentication token. Authentication.login also sets the token value in the python instance argument variable "--ident", and as the local object Authentication.token (of class Authentication.Token). - :param UserName: name of the user (string) - :param Password: password of the user (string) + :param UserName: name of the user (string). If not set, then a user name prompt will show. + :param Password: password of the user (string). If not set, then a password prompt will show. :return: authentication token (string) :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. - :example: token = Authentication.login('loginName','loginPassword') - + :example: token1 = Authentication.login(); token2 = Authentication.login('loginName','loginPassword'); .. seealso:: Authentication.getKeystoneUserWithToken, Authentication.getToken, Authentication.setToken, Authentication.token. """ taskName = "" @@ -87,7 +86,14 @@ def login(UserName, Password): loginURL = Config.AuthenticationURL + "?TaskName=" + taskName - authJson = {"auth":{"identity":{"password":{"user":{"name":UserName,"password":Password}}}}} + userName = UserName; + password = Password; + if userName is None: + userName = getpass.getpass(prompt="Enter SciServer user name: ") + if password is None: + password = getpass.getpass(prompt="Enter SciServer password: ") + + authJson = {"auth":{"identity":{"password":{"user":{"name":userName,"password":password}}}}} data = json.dumps(authJson).encode() diff --git a/py2/SciServer/ComputeJobs.py b/py2/SciServer/ComputeJobs.py deleted file mode 100644 index 25d63dc..0000000 --- a/py2/SciServer/ComputeJobs.py +++ /dev/null @@ -1,238 +0,0 @@ -__author__ = 'mtaghiza' - -import sys -import os; -import os.path; -from SciServer import Authentication, Config -import requests -import json -import time; - - - -def getJobDirectory(jobId = None): - """ - Gets the root directory path where the job is being executed. Since its value depends on the value of jobID, the path will be defined only after the job is submitted. - """ - computeJobDirectoryFile = Config.ComputeJobDirectoryFile - if os.path.isfile(computeJobDirectoryFile): - jobDirectory = None; - with open(computeJobDirectoryFile, 'r') as f: - jobDirectory = f.read().rstrip('\n') - - return jobDirectory - else: - raise OSError("Cannot find jobs directory for jobId=" + str(jobId) + ".") - - -def getComputeDomains(batch=True, interactive=False): - """ - Gets a list of all registered compute domains that the user has access to. - """ - token = Authentication.getToken() - if token is not None and token != "": - - queryString = ("batch=true&" if bool(batch) == True else "batch=false&"); - queryString = queryString + ("interactive=true" if bool(interactive) == True else "interactive=false"); - url = Config.JobmApiURL + "/computedomains?" + queryString - headers = {'X-Auth-Token': token, "Content-Type": "application/json"} - res = requests.get(url, headers=headers, stream=True) - - if res.status_code != 200: - raise Exception("Error when getting Compute Domains from JOBM API.\nHttp Response from SciDrive API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); - else: - return json.loads(res.content.decode()) - else: - raise Exception("User token is not defined. First log into SciServer.") - - -def getJobTypes(): - return "" - - -def getJobsList(): - """ - Gets the list of Jobs submitted by the user. - :return: - """ - token = Authentication.getToken() - if token is not None and token != "": - url = Config.JobmApiURL + "/jobs" - headers = {'X-Auth-Token': token, "Content-Type": "application/json"} - res = requests.get(url, headers=headers, stream=True) - - if res.status_code != 200: - raise Exception("Error when getting Compute Domains from JOBM API.\nHttp Response from SciDrive API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); - else: - return json.loads(res.content.decode()) - else: - raise Exception("User token is not defined. First log into SciServer.") - -def getJobDescription(jobId): - """ - Gets the definition of the job, including the current status. - :param jobId: Job ID - :return: - """ - token = Authentication.getToken() - if token is not None and token != "": - url = Config.JobmApiURL + "/jobs/" + str(jobId) - headers = {'X-Auth-Token': token, "Content-Type": "application/json"} - res = requests.get(url, headers=headers, stream=True) - - if res.status_code != 200: - raise Exception("Error when getting from JOBM API the job status of jobId=" + str(jobId) + ".\nHttp Response from SciDrive API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); - else: - return json.loads(res.content.decode()) - else: - raise Exception("User token is not defined. First log into SciServer.") - - -def getJobStatus(jobId): - """ - Gets the semantic meaning of the integer status value included in the job description. The integer value is a power of 2, that is, 1:PENDING, 2:QUEUED, 4:ACCEPTED, 8:STARTED, 16:FINISHED, 32:SUCCESS, 64:ERROR and 128:CANCELED - :param jobId: Id of job. - :return: string with job status. - """ - intStatus = getJobDescription(jobId)["status"] - - if intStatus == 1: - return "PENDING" - elif intStatus == 2: - return "QUEUED" - elif intStatus == 4: - return "ACCEPTED" - elif intStatus == 8: - return "STARTED" - elif intStatus == 16: - return "FINISHED" - elif intStatus == 32: - return "SUCCESS" - elif intStatus == 64: - return "ERROR" - elif intStatus == 128: - return "CANCELED" - else: - raise Exception("Invalid integer value given to job status.") - - -def submitNotebookJob(domainApiEndpoint, notebookPath, dockerImage="", volumes=[None], parameters="", jobAlias = ""): - """ - Submits a Jupyter Notebook for execution as an asynchronous job, - :param computeDomainName: Name of compute domain (string). E.g: http://compute.sciserver.org/dashboard/api/container/ - :param notebookPath: path of the notebook within the filesystem mounted in SciServer-Compute (string). - :param dockerImage: name of Docker image where the job is being submitted. E.g., dockerImage=["Python + Dev (astro)"] - :param volumes: volume conatiners mounted in the docker image for the job execution. E.g., volumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}] - :param parameters: string containing parameters. - :param jobAlias: alias of job - :return: - """ - - token = Authentication.getToken() - if token is not None and token != "": - - dockerJobModel = { - "command": parameters, - "scriptURI": notebookPath, - "submitterDID": jobAlias, - "dockerComputeEndpoint": domainApiEndpoint, - "dockerImageName": dockerImage, - "volumeContainers": volumes - } - data = json.dumps(dockerJobModel).encode() - - url = Config.JobmApiURL + "/jobs/docker" - headers = {'X-Auth-Token': token, "Content-Type": "application/json"} - res = requests.post(url, data=data, headers=headers, stream=True) - - if res.status_code != 200: - raise Exception("Error when submitting a job to the JOBM API.\nHttp Response from SciDrive API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); - else: - return json.loads(res.content.decode()) - else: - raise Exception("User token is not defined. First log into SciServer.") - - -def submitShellCommandJob(domainApiEndpoint, shellCommand, dockerImage=[], volumes=[], jobAlias = ""): - """ - Submits a shell command for execution as an asynchronous job, - :param computeDomainName: Name of compute domain (string). E.g: http://compute.sciserver.org/dashboard/api/container/ - :param shellCommand: path of the notebook within the filesystem mounted in SciServer-Compute - :param dockerImage: name of Docker image where the job is being submitted. E.g., dockerImage=["Python + Dev (astro)"] - :param volumes: volume conatiners mounted in the docker image for the job execution. E.g., volumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}] - :param parameters: string containing parameters. - :param jobAlias: alias of job - :return: - """ - - token = Authentication.getToken() - if token is not None and token != "": - - dockerJobModel = { - "command": shellCommand, - "submitterDID": jobAlias, - "dockerComputeEndpoint": domainApiEndpoint, - "dockerImageName": dockerImage, - "volumeContainers": volumes - } - data = json.dumps(dockerJobModel).encode() - - url = Config.JobmApiURL + "/jobs/docker" - headers = {'X-Auth-Token': token, "Content-Type": "application/json"} - res = requests.post(url, data=data, headers=headers, stream=True) - - if res.status_code != 200: - raise Exception("Error when submitting a job to the JOBM API.\nHttp Response from SciDrive API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); - else: - return json.loads(res.content.decode()) - else: - raise Exception("User token is not defined. First log into SciServer.") - - - -def cancelJob(jobId): - - token = Authentication.getToken() - if token is not None and token != "": - url = Config.JobmApiURL + "/jobs/" + str(jobId) + "/cancel" - headers = {'X-Auth-Token': token, "Content-Type": "application/json"} - res = requests.post(url, headers=headers, stream=True) - - if res.status_code != 200: - raise Exception("Error when getting from JOBM API the job status of jobId=" + str(jobId) + ".\nHttp Response from SciDrive API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); - else: - return True; - else: - raise Exception("User token is not defined. First log into SciServer.") - - -def waitForJob(jobId, verbose=True, pollTime = 5): - try: - complete = False - - waitingStr = "Waiting..." - back = "\b" * len(waitingStr) - if verbose: - print waitingStr, - - while not complete: - if verbose: - #print back, - print waitingStr, - jobDesc = getJobDescription(jobId) - jobStatus = jobDesc["status"] - if jobStatus >= 32: - complete = True - if verbose: - #print back, - print "Done!" - else: - time.sleep(max(pollTime,5)); - - return jobDesc - except Exception as e: - raise e; - - - -# for out internal testing only: diff --git a/py2/SciServer/Config.py b/py2/SciServer/Config.py index 5876add..4a23a0f 100644 --- a/py2/SciServer/Config.py +++ b/py2/SciServer/Config.py @@ -12,6 +12,8 @@ - **Config.SkyServerWSurl**: defines the base URL of the SkyServer web service API (string). E.g., "https://skyserver.sdss.org" +- **Config.RacmApiURL**: defines the base URL of the RACM API (string). E.g., "https://www.sciserver.org/racm" + - **Config.DataRelease**: defines the SDSS data release (string), to be used to build the full SkyServer API url along with Config.SkyServerWSurl. E.g., "DR13" - **Config.KeystoneTokenPath**: defines the local path (string) to the file containing the user's authentication token in the SciServer-Compute environment. E.g., "/home/idies/keystone.token". Unlikely to change since it is hardcoded in SciServer-Compute. @@ -24,12 +26,11 @@ SciDriveHost = "https://www.scidrive.org" SkyQueryUrl = "http://voservices.net/skyquery/Api/V1" SkyServerWSurl = "https://skyserver.sdss.org" +RacmApiURL = "http://scitest12.pha.jhu.edu/racm" DataRelease = "DR13" KeystoneTokenPath = "/home/idies/keystone.token" #the path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment version = "sciserver-v1.10.0" #sciserver release version -JobmApiURL = "http://scitest12.pha.jhu.edu/racm/jobm/rest" ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. -FileServiceURL = 'http://scitest12.pha.jhu.edu/fileservice/api'; # returns TRUE if the library is running inside the SciServer-Compute, and FALSE if not import os; diff --git a/py2/SciServer/FileService.py b/py2/SciServer/FileService.py deleted file mode 100644 index 9269f05..0000000 --- a/py2/SciServer/FileService.py +++ /dev/null @@ -1,181 +0,0 @@ -__author__ = 'mtaghiza' - -from SciServer import Authentication, Config -import requests -from io import StringIO -from io import BytesIO - - -def createDir(path): - """ - Create a directory. - :param path: path to the directory. E.g: path="/persistent/newDirectory" - :return: True if the directory creation was successful. - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileSystem API returns an error. - """ - token = Authentication.getToken() - if token is not None and token != "": - - if ~path.startswith("/"): - path = "/" + path; - - url = Config.FileServiceURL + "/data" + path + "?dir=true" - - headers = {'X-Auth-Token': token} - res = requests.put(url, headers=headers) - - if res.status_code >= 200 and res.status_code < 300: - return True; - else: - raise Exception("Error when creating new directory " + path + ".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); - else: - raise Exception("User token is not defined. First log into SciServer.") - - -def getDirList(path, options="X", level=1): - """ - Lists the contents of a directory. - :param path: path to the directory. E.g: path="/persistent/newDirectory" - :return: True if the directory creation was successful. - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileSystem API returns an error. - """ - token = Authentication.getToken() - if token is not None and token != "": - - if ~path.startswith("/"): - path = "/" + path; - - url = Config.FileServiceURL + "/tree" + path + "?options=" + str(options) + "&level=" + str(level) - - headers = {'X-Auth-Token': token} - res = requests.get(url, headers=headers) - - if res.status_code >= 200 and res.status_code < 300: - return res.content.decode(); - else: - raise Exception("Error when creating new directory " + path + ".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); - else: - raise Exception("User token is not defined. First log into SciServer.") - - - - -def delete(path): - """ - Delete a directory or file in the remote File System. - :param path: path to the directory or file. E.g: path="/persistent/directory/file.txt" - :return: True if the deletion was successful. - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileSystem API returns an error. - """ - token = Authentication.getToken() - if token is not None and token != "": - - if ~path.startswith("/"): - path = "/" + path; - - url = Config.FileServiceURL + "/data" + path + "?dir=true" - - headers = {'X-Auth-Token': token} - res = requests.delete(url, headers=headers) - - if res.status_code >= 200 and res.status_code < 300: - return True; - else: - raise Exception("Error when deleting " + path + ".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); - else: - raise Exception("User token is not defined. First log into SciServer.") - - - -def upload(path, data=None, localFilePath=None): - """ - Uploads data or a local file into a path defined in the file system. - :param path: path of the file to be created. E.g: path="/persistent/newDirectory/myNewFile.txt" - :param data: string containing data to be uploaded. - :param localFilePath: path to a local file to be uploaded (string). - :return: True is upload was successful. - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileSystem API returns an error. - """ - token = Authentication.getToken() - if token is not None and token != "": - - if ~path.startswith("/"): - path = "/" + path; - - - fileName = path.split("/")[-1] - dirPath = path.replace(fileName, "") - - createDir(dirPath); - - url = Config.FileServiceURL + "/data" + path - - headers = {'X-Auth-Token': token} - - if localFilePath is not None and localFilePath != "": - with open(localFilePath, "rb") as file: - res = requests.put(url, data=file, headers=headers, stream=True) - else: - if data != None: - res = requests.put(url, data=data, headers=headers, stream=True) - else: - raise Exception("Error when uploading to " + path + ". No local file or data specified for uploading."); - - if res.status_code >= 200 and res.status_code < 300: - return True; - else: - raise Exception("Error when uploading to " + path +".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); - else: - raise Exception("User token is not defined. First log into SciServer.") - - -def download(path, format="text", localFilePath=""): - """ - Downloads a file from the remote file system into the local file system, or returns the file content as an object in several formats. - - :param path: path of the file (or directory) to be downloaded from the remote file system. - :param format: type of the returned object. Can be "StringIO" (io.StringIO object containing readable text), "BytesIO" (io.BytesIO object containing readable binary data), "response" ( the HTTP response as an object of class requests.Response) or "text" (a text string). If the parameter 'localFilePath' is defined, then the 'format' parameter is not used and the file is downloaded to the local file system instead. - :param localFilePath: local path of the file to be downloaded. If 'localFilePath' is defined, then the 'format' parameter is not used. - :return: If the 'localFilePath' parameter is defined, then it will return True when the file is downloaded successfully in the local file system. If the 'localFilePath' is not defined, then the type of the returned object depends on the value of the 'format' parameter (either io.StringIO, io.BytesIO, requests.Response or string). - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileSystem API returns an error. - """ - token = Authentication.getToken() - if token is not None and token != "": - - if ~path.startswith("/"): - path = "/" + path; - - url = Config.FileServiceURL + "/data" + path - headers = {'X-Auth-Token': token} - - res = requests.get(url, stream=True, headers=headers) - - if res.status_code < 200 or res.status_code >= 300: - raise Exception("Error when downloading from remote File Syatem the file " + str(path) + ".\nHttp Response from FileSystem API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); - - if localFilePath is not None and localFilePath != "": - - bytesio = BytesIO(res.content) - theFile = open(localFilePath, "w+b") - theFile.write(bytesio.read()) - theFile.close() - return True - - else: - - if format is not None and format != "": - if format == "StringIO": - return StringIO(res.content.decode()) - if format == "text": - return res.content.decode() - elif format == "BytesIO": - return BytesIO(res.content) - elif format == "response": - return res; - else: - raise Exception("Unknown format '" + format + "' when trying to download from remote File System the file " + str(path) + ".\n"); - else: - raise Exception("Wrong format parameter value\n"); - - else: - raise Exception("User token is not defined. First log into SciServer.") diff --git a/py2/SciServer/Files.py b/py2/SciServer/Files.py new file mode 100644 index 0000000..13b11d2 --- /dev/null +++ b/py2/SciServer/Files.py @@ -0,0 +1,318 @@ +__author__ = 'mtaghiza' + +from SciServer import Authentication, Config +import requests +import json +from io import StringIO +from io import BytesIO + + + +def getFileServices(): + """ + Gets the definitions of the file services that a user is able to access. + :return: list of dictionaries, where each dictionary is is the descriptions of a FileService that a user is able to access. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServices = Files.getFileServices(); + .. seealso:: Files.getFileServiceFromName + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.getFileServices" + else: + taskName = "SciScript-Python.Files.getFileServices" + + url = Config.RacmApiURL + "/storem?TaskName="+taskName; + + headers = {'X-Auth-Token': token} + res = requests.get(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + return json.loads(res.content.decode()); + else: + raise Exception("Error when getting the list of FileServices.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def getFileServiceFromName(fileServiceName, fileServices=None): + """ + Returns a FileService object, given its registered name. + :param fileServiceName: name of the FileService, as shown within the results of Jobs.getFileServices() + :param fileServices: a list of FileService objects (dictionaries), as returned by Jobs.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. + :return: a FileService object (dictionary) that defines a FileService. A list of these kind of objects available to the user is returned by the function Jobs.getFileServices(). + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileService = Files.getFileServiceFromName('fileServiceAtJHU'); + .. seealso:: Files.getFileServices + """ + + if fileServiceName is None: + raise Exception("fileServiceName is not defined.") + else: + if fileServices is None: + fileServices = getFileServices(); + + if fileServices.__len__() > 0: + for fileService in fileServices: + if fileServiceName == fileService.get('name'): + return fileService; + else: + raise Exception("There are no FileServices available for the user."); + + raise Exception("FileService of name '" + fileServiceName + "' is not available or does not exist."); + + + +def __getFileServiceAPIUrl(fileService=None): + """ + Gets the API endpoint URL of a FileService. + :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made, and the returned URL is that of the first FileService found. + :return: API endpoint URL of the FileService (string). + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServiceUrl = Files.__getFileServiceAPIUrl(); + .. seealso:: Files.getFileServiceFromName + """ + return 'http://scitest12.pha.jhu.edu/fileservice/'; # for now + + if fileService is None: + fileServices = getFileServices(); + if fileServices.__len__() > 0: + return fileServices[0].get("URL"); + else: + raise Exception("There are no FileServices available for the user."); + + else: + return fileService.get("URL"); + + +def createDir(path, volume, ownerName=None, fileService=None): + """ + Create a directory. + :param path: path (string) to the directory, relative to the volume level. E.g: path="/newDirectory" + :param volume: volume (string) containing the path. + :param ownerName: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. + :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :return: True if the directory creation was successful. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.createDir("myNewDir","persistent","myUserName", fileServices[0]); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.createDir" + else: + taskName = "SciScript-Python.Files.createDir" + + if ~path.startswith("/"): + path = "/" + path; + + if ownerName is None: + ownerName = Authentication.getKeystoneUserWithToken().userName; + + url = Config.FileServiceURL + "/folder/" + volume + "/" + ownerName + path + "?TaskName="+taskName; + + headers = {'X-Auth-Token': token} + res = requests.put(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + return True; + else: + raise Exception("Error when creating directory '" + path + "' in volume '" + volume + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def dirList(path, volume, ownerName=None, fileService=None, level=1, options=''): + """ + Lists the contents of a directory. + :param path: path (string) to the directory, relative to the volume level. E.g: path="/newDirectory" + :param volume: volume (string) containing the path. + :param ownerName: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. + :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param level: amount (int) of listed directory levels that are below or at the same level of the path. + :param options: string of file filtering options. + :return: dictionary containing the directory listing. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); dirs = Files.dirList("/","persistent","myUserName", fileServices[0], 2); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.createDir + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.dirList" + else: + taskName = "SciScript-Python.Files.dirList" + + if ownerName is None: + ownerName = Authentication.getKeystoneUserWithToken().userName; + + if ~path.startswith("/"): + path = "/" + path; + + url = Config.FileServiceURL + "/jsontree/" + volume + "/" + ownerName + path + "?options=" + str(options) + "&level=" + str(level) + "&TaskName=" + taskName; + + headers = {'X-Auth-Token': token} + res = requests.get(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + return res.content.decode(); + else: + raise Exception("Error when creating new directory " + path + ".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + + + +def delete(path, volume, ownerName=None, fileService=None): + """ + Deletes a directory or file in the File System. + :param path: path (string) to the directory or file to be deleted, relative to the volume level. E.g: path="/directory/file.txt" + :param volume: volume (string) containing the path. + :param ownerName: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. + :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :return: True if the deletion was successful. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.delete" + else: + taskName = "SciScript-Python.Files.delete" + + if ~path.startswith("/"): + path = "/" + path; + + url = Config.FileServiceURL + "/data" + path + "?TaskName="+taskName + + headers = {'X-Auth-Token': token} + res = requests.delete(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + return True; + else: + raise Exception("Error when deleting " + path + ".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + + +def upload(path, volume, ownerName=None, fileService=None, data=None, localFilePath=None): + """ + Uploads data or a local file into a path defined in the file system. + :param path: destination path (string) of the file to be uploaded, relative to the volume level. E.g: path="/newDirectory/myNewFile.txt" + :param volume: volume (string) containing the path. + :param ownerName: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. + :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param data: string containing data to be uploaded. + :param localFilePath: path to a local file to be uploaded (string). + :return: True is upload was successful. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); isUploaded = Files.upload("/myUploadedFile.txt","persistent","myUserName", fileServices[0], localFilePath="/myFile.txt"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.upload" + else: + taskName = "SciScript-Python.Files.upload" + + if ~path.startswith("/"): + path = "/" + path; + + fileName = path.split("/")[-1] + dirPath = path.replace(fileName, "") + + createDir(dirPath); + + url = Config.FileServiceURL + "/data" + path + "?TaskName="+taskName + + headers = {'X-Auth-Token': token} + + if localFilePath is not None and localFilePath != "": + with open(localFilePath, "rb") as file: + res = requests.put(url, data=file, headers=headers, stream=True) + else: + if data != None: + res = requests.put(url, data=data, headers=headers, stream=True) + else: + raise Exception("Error when uploading to " + path + ". No local file or data specified for uploading."); + + if res.status_code >= 200 and res.status_code < 300: + return True; + else: + raise Exception("Error when uploading to " + path +".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def download(path, volume, ownerName=None, fileService=None, localFilePath="", format="text"): + """ + Downloads a file from the remote file system into the local file system, or returns the file content as an object in several formats. + :param path: remote path (string) of the file to be downloaded, relative to the volume level. E.g: path="/newDirectory/myNewFile.txt" + :param volume: volume (string) containing the path. + :param ownerName: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. + :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param localFilePath: local path of the file to be downloaded. If 'localFilePath' is defined, then the 'format' parameter is not used. + :param format: type of the returned object. Can be "StringIO" (io.StringIO object containing readable text), "BytesIO" (io.BytesIO object containing readable binary data), "response" ( the HTTP response as an object of class requests.Response) or "text" (a text string). If the parameter 'localFilePath' is defined, then the 'format' parameter is not used and the file is downloaded to the local file system instead. + :return: If the 'localFilePath' parameter is defined, then it will return True when the file is downloaded successfully in the local file system. If the 'localFilePath' is not defined, then the type of the returned object depends on the value of the 'format' parameter (either io.StringIO, io.BytesIO, requests.Response or string). + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); isDownloaded = Files.upload("/myUploadedFile.txt","persistent","myUserName", fileServices[0], localFilePath="/myDownloadedFile.txt"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.upload, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.download" + else: + taskName = "SciScript-Python.Files.download" + + if ~path.startswith("/"): + path = "/" + path; + + url = Config.FileServiceURL + "/data" + path + "?TaskName="+taskName; + headers = {'X-Auth-Token': token} + + res = requests.get(url, stream=True, headers=headers) + + if res.status_code < 200 or res.status_code >= 300: + raise Exception("Error when downloading from remote File Syatem the file " + str(path) + ".\nHttp Response from FileSystem API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + + if localFilePath is not None and localFilePath != "": + + bytesio = BytesIO(res.content) + theFile = open(localFilePath, "w+b") + theFile.write(bytesio.read()) + theFile.close() + return True + + else: + + if format is not None and format != "": + if format == "StringIO": + return StringIO(res.content.decode()) + if format == "text": + return res.content.decode() + elif format == "BytesIO": + return BytesIO(res.content) + elif format == "response": + return res; + else: + raise Exception("Unknown format '" + format + "' when trying to download from remote File System the file " + str(path) + ".\n"); + else: + raise Exception("Wrong format parameter value\n"); + + else: + raise Exception("User token is not defined. First log into SciServer.") diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py new file mode 100644 index 0000000..f5a4cba --- /dev/null +++ b/py2/SciServer/Jobs.py @@ -0,0 +1,599 @@ +__author__ = 'mtaghiza' + +import sys +import os; +import os.path; +from SciServer import Authentication, Config +import requests +import json +import time; + + + +def getJobDirectory(): + """ + Gets the root directory path where the job is being executed. Since its value depends on the value of jobId, the path will be defined only after the job is submitted. + """ + computeJobDirectoryFile = Config.ComputeJobDirectoryFile + if os.path.isfile(computeJobDirectoryFile): + jobDirectory = None; + with open(computeJobDirectoryFile, 'r') as f: + jobDirectory = f.read().rstrip('\n') + + return jobDirectory + else: + raise OSError("Cannot find job execution directory") + + +def getDockerComputeDomains(): + """ + Gets a list of all registered Docker compute domains that the user has access to. + :return: a list of dictionaries, each one containing the definition of a Docker compute domain. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: dockerComputeDomains = Jobs.getDockerComputeDomains(); + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getRDBComputeDomains, Jobs.cancelJob + """ + token = Authentication.getToken() + if token is not None and token != "": + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.getDockerComputeDomains" + else: + taskName = "SciScript-Python.Jobs.getDockerComputeDomains" + + url = Config.RacmApiURL + "/jobm/rest/computedomains?batch=true&interactive=false&TaskName=" + taskName + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting Docker Compute Domains from JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def getRDBComputeDomains(): + """ + Gets a list of all registered Relational Database (RDB) compute domains that the user has access to. + :return: a list of dictionaries, each one containing the definition of an RDB compute domain. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: rdbComputeDomains = Jobs.getRDBComputeDomains(); + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.getRDBComputeDomains" + else: + taskName = "SciScript-Python.Jobs.getRDBComputeDomains" + + url = Config.RacmApiURL + "/jobm/rest/computedomains/rdb?TaskName=" + taskName + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + if res.status_code != 200: + raise Exception("Error when getting RDB Compute Domains from JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def getDockerComputeDomainFromName(dockerComputeDomainName, dockerComputeDomains = None): + """ + Returns a DockerComputeDomain object, given its registered name. + :param dockerComputeDomainName: name of the DockerComputeDomainName, as shown within the results of Jobs.getDockerComputeDomains() + :param dockerComputeDomains: a list of DockerComputeDomain objects (dictionaries), as returned by Jobs.getDockerComputeDomains(). If not set, then an internal call to Jobs.getDockerComputeDomains() is made. + :return: a DockerComputeDomain object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: dockerComputeDomain = Jobs.getDockerComputeDomainFromName('dockerComputeDomainAtJHU'); + .. seealso:: Jobs.getDockerComputeDomains, Jobs.getRDBComputeDomains, Jobs.getRDBComputeDomainFromName + """ + if dockerComputeDomainName is None: + raise Exception("dockerComputeDomainName is not defined.") + else: + if dockerComputeDomains is None: + dockerComputeDomains = getDockerComputeDomains(); + + if dockerComputeDomains.__len__() > 0: + for dockerComputeDomain in dockerComputeDomains: + if dockerComputeDomainName == dockerComputeDomain.get('name'): + return dockerComputeDomain; + else: + raise Exception("There are no DockerComputeDomains available for the user."); + + raise Exception("DockerComputeDomain of name '" + dockerComputeDomainName + "' is not available or does not exist."); + + +def getRDBComputeDomainFromName(rdbComputeDomainName, rdbComputeDomains = None): + """ + Returns an RDBComputeDomain object, given its registered name. + :param rdbComputeDomainName: name of the RDBComputeDomainName, as shown within the results of Jobs.getRDBComputeDomains() + :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. + :return: an RDBComputeDomain object (dictionary) that defines an RDB compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: rdvComputeDomain = Jobs.getRDBComputeDomainFromName('rdbComputeDomainAtJHU'); + .. seealso:: Jobs.getDockerComputeDomains, Jobs.getRDBComputeDomains, Jobs.getDockerComputeDomainFromName + """ + if rdbComputeDomainName is None: + raise Exception("rdbComputeDomainName is not defined.") + else: + if rdbComputeDomains is None: + rdbComputeDomains = getRDBComputeDomains(); + + if rdbComputeDomains.__len__() > 0: + for rdbComputeDomain in rdbComputeDomains: + if rdbComputeDomainName == rdbComputeDomain.get('name'): + return rdbComputeDomain; + else: + raise Exception("There are no RDBComputeDomains available for the user."); + + raise Exception("RDBComputeDomain of name '" + rdbComputeDomainName + "' is not available or does not exist."); + + +def getJobsList(top=10, open=None, start=None, end=None, type='all'): + """ + Gets the list of Jobs submitted by the user. + :param top: top number of jobs (integer) returned. If top=None, then all jobs are returned. + :param open: If set to 'True', then only returns jobs that have not finished executing and wrapped up (status <= FINISHED). If set to 'False' then only returnes jobs that are still running. If set to 'None', then returns both finished and unfinished jobs. + :param start: The earliest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no lower bound on date. + :param end: The latest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no upper bound on date. + :param type: type (string) of jobs returned. Can take values of 'rdb' (for returning only relational database jobs), 'docker' (for returning only Docker jobs) and 'all' (all job types are returned). + :return: a list of dictionaries, each one containing the definition of a submitted job. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. + :example: jobs = Jobs.getJobsList(top=2); + .. seealso:: Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.getJobsList" + else: + taskName = "SciScript-Python.Jobs.getJobsList" + + + topString = ("top=" + str(top) + "&" if top != None else "") + startString = ("start=" + str(start) + "&" if start != None else "") + endString = ("end=" + str(end) + "&" if end != None else "") + if open is None: + openString = ""; + else: + if open is True: + openString = "open=true&"; + else: + openString = "open=false&"; + + url = Config.RacmApiURL + "/jobm/rest/jobs?" + if(type=='rdb'): + url = Config.RacmApiURL + "/jobm/rest/rdbjobs?" + if(type=='docker'): + url = Config.RacmApiURL + "/jobm/rest/dockerjobs?" + + url = url + topString + startString + endString + "TaskName=" + taskName; + + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting list of jobs from JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + +def getJobDescription(jobId): + """ + Gets the definition of the job, + :param jobId: Id of job + :return: dictionary containing the description or definition of the job. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. + :example: job1 = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); job2 = Jobs.getJobDescription(job1.get('id')); + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.getJobDescription" + else: + taskName = "SciScript-Python.Jobs.getJobDescription" + + url = Config.RacmApiURL + "/jobm/rest/jobs/" + str(jobId) + "?TaskName="+taskName + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting from JOBM API the job status of jobId=" + str(jobId) + ".\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def getJobStatus(jobId): + """ + Gets a dictionary with the job status as an integer value, together with its semantic meaning. The integer value is a power of 2, that is, 1:PENDING, 2:QUEUED, 4:ACCEPTED, 8:STARTED, 16:FINISHED, 32:SUCCESS, 64:ERROR and 128:CANCELED + :param jobId: Id of job (integer). + :return: dictionary with the integer value of the job status, as well as its semantic meaning. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. + :example: job = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); status = Jobs.getJobStatus(job.get('id')); + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + intStatus = getJobDescription(jobId)["status"] + + if intStatus == 1: + return {'status':"PENDING", 'intStatus':intStatus} + elif intStatus == 2: + return {'status': "QUEUED", 'intStatus': intStatus} + elif intStatus == 4: + return {'status': "ACCEPTED", 'intStatus': intStatus} + elif intStatus == 8: + return {'status': "STARTED", 'intStatus': intStatus} + elif intStatus == 16: + return {'status': "FINISHED", 'intStatus': intStatus} + elif intStatus == 32: + return {'status': "SUCCESS", 'intStatus': intStatus} + elif intStatus == 64: + return {'status': "ERROR", 'intStatus': intStatus} + elif intStatus == 128: + return {'status': "CANCELED", 'intStatus': intStatus} + else: + raise Exception("Invalid integer value given to job status.") + + +def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=None, userVolumes=None, dataVolumes=None, parameters="", jobAlias= ""): + """ + Submits a Jupyter Notebook for execution (as an asynchronous job) inside a Docker compute domain, + :param notebookPath: path of the notebook within the filesystem mounted in SciServer-Compute (string). Example: notebookPath = '/home/idies/worskpace/persistent/JupyterNotebook.ipynb' + :param dockerComputeDomain: object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). + :param dockerImageName: name (string) of the Docker image for executing the notebook. E.g., dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object. + :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image. + E.g., userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', , 'needsWriteAccess':True}] + A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=None, then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume. + :param dataVolumes: a list with the names of data volumes that will be mounted to the docker Image. + E.g., dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}]. + A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted. + :param parameters: string containing parameters that the notebook might need during its execution. This string is written in the 'parameters.txt' file in the same directory level where the notebook is being executed. + :param jobAlias: alias (string) of job, defined by the user. + :return: a dictionary containing the definition of the submitted job. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. + :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitNotebookJob('/home/idies/workspace/persistent/Notebook.ipynb', dockerComputeDomain, 'Python (astro)', [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}], [{'name':'SDSS_DAS'}], 'param1=1\nparam2=2\nparam3=3','myNewJob') + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.submitNotebookJob" + else: + taskName = "SciScript-Python.Jobs.submitNotebookJob" + + if dockerComputeDomain is None: + dockerComputeDomains = getDockerComputeDomains(); + if dockerComputeDomains .__len__() > 0: + dockerComputeDomain = dockerComputeDomains[0]; + else: + raise Exception("There are no dockerComputeDomains available for the user."); + + if dockerImageName is None: + images = dockerComputeDomain.get('images'); + if images.__len__() > 0: + dockerImageName = images[0].get('name') + else: + raise Exception("dockerComputeDomain has no docker images available for the user."); + + uVols = []; + if userVolumes is None: + for vol in dockerComputeDomain.get('userVolumes'): + if 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + + + else: + for uVol in userVolumes: + found = False; + for vol in dockerComputeDomain.get('userVolumes'): + if vol.get('name') == uVol.get('name'): + found = True; + if (uVol.has_key('needsWriteAccess')): + if uVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + else: + if 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + + if not found: + raise Exception("User volume '" + uVol.get('name') + "' not found within Compute domain") + + datVols = []; + if dataVolumes is None: + for vol in dockerComputeDomain.get('volumes'): + datVols.append({'name': vol.get('name')}); + + else: + for dVol in dataVolumes: + found = False; + for vol in dockerComputeDomain.get('volumes'): + if vol.get('name') == dVol.get('name'): + found = True; + datVols.append({'name': vol.get('name')}); + + if not found: + raise Exception("Data volume '" + dVol.get('name') + "' not found within Compute domain") + + + dockerComputeEndpoint = dockerComputeDomain.get('apiEndpoint'); + + dockerJobModel = { + "command": parameters, + "scriptURI": notebookPath, + "submitterDID": jobAlias, + "dockerComputeEndpoint": dockerComputeEndpoint, + "dockerImageName": dockerImageName, + "volumeContainers": datVols, + "userVolumes": uVols + } + data = json.dumps(dockerJobModel).encode() + url = Config.RacmApiURL + "/jobm/rest/jobs/docker?TaskName="+taskName; + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.post(url, data=data, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when submitting a notebook job to the JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageName = None, userVolumes = None, dataVolumes = None, jobAlias = ""): + """ + Submits a shell command for execution (as an asynchronous job) inside a Docker compute domain. + :param shellCommand: shell command (string) defined by the user. + :param dockerComputeDomain: object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). + :param dockerImageName: name (string) of the Docker image for executing the notebook. E.g., dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object. + :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image. + E.g., userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', , 'needsWriteAccess':True}] + A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=None, then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume. + :param dataVolumes: a list with the names of data volumes that will be mounted to the docker Image. + E.g., dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}]. + A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted. + :param jobAlias: alias (string) of job, defined by the user. + :return: a dictionary containing the definition of the submitted job. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. + :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitShellCommandJob('pwd', dockerComputeDomain, 'Python (astro)', [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}], [{'name':'SDSS_DAS'}], 'myNewJob') + .. seealso:: Jobs.submitNotebookJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.submitShellCommandJob" + else: + taskName = "SciScript-Python.Jobs.submitShellCommandJob" + + + if dockerComputeDomain is None: + dockerComputeDomains = getDockerComputeDomains(); + if dockerComputeDomains .__len__() > 0: + dockerComputeDomain = dockerComputeDomains[0]; + else: + raise Exception("There are no dockerComputeDomains available for the user."); + + if dockerImageName is None: + images = dockerComputeDomain.get('images'); + if images.__len__() > 0: + dockerImageName = images[0].get('name') + else: + raise Exception("dockerComputeDomain has no docker images available for the user."); + + uVols = []; + if userVolumes is None: + for vol in dockerComputeDomain.get('userVolumes'): + if 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + + + else: + for uVol in userVolumes: + found = False; + for vol in dockerComputeDomain.get('userVolumes'): + if vol.get('name') == uVol.get('name'): + found = True; + if (uVol.has_key('needsWriteAccess')): + if uVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + else: + if 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + + if not found: + raise Exception("User volume '" + uVol.get('name') + "' not found within Compute domain") + + datVols = []; + if dataVolumes is None: + for vol in dockerComputeDomain.get('volumes'): + datVols.append({'name': vol.get('name')}); + + else: + for dVol in dataVolumes: + found = False; + for vol in dockerComputeDomain.get('volumes'): + if vol.get('name') == dVol.get('name'): + found = True; + datVols.append({'name': vol.get('name')}); + + if not found: + raise Exception("Data volume '" + dVol.get('name') + "' not found within Compute domain") + + + dockerComputeEndpoint = dockerComputeDomain.get('apiEndpoint'); + + dockerJobModel = { + "command": shellCommand, + "submitterDID": jobAlias, + "dockerComputeEndpoint": dockerComputeEndpoint, + "dockerImageName": dockerImageName, + "volumeContainers": datVols, + "userVolumes": uVols + } + data = json.dumps(dockerJobModel).encode() + url = Config.RacmApiURL + "/jobm/rest/jobs/docker?TaskName="+taskName; + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.post(url, data=data, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when submitting a job to the JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + +def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = None, resultsName=None, jobAlias = ""): + """ + Submits a sql query for execution (as an asynchronous job) inside a relational database (RDB) compute domain. + :param sqlQuery: sql query (string) + :param rdbComputeDomain: object (dictionary) that defines a relational database (RDB) compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). + :param databaseContextName: database context name (string) on which the sql query is executed. + :param resultsName: name (string) of the table or file containing the query result. + :param jobAlias: alias (string) of job, defined by the user. + :return: a dictionary containing the definition of the submitted job. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. + :example: job = Jobs.submitRDBQueryJob('select 1';,None, None, 'myQueryResults', 'myNewJob') + .. seealso:: Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.submitRDBQueryJob" + else: + taskName = "SciScript-Python.Jobs.submitRDBQueryJob" + + if rdbComputeDomain is None: + rdbComputeDomains = getRDBComputeDomains(); + if rdbComputeDomains .__len__() > 0: + rdbComputeDomain = rdbComputeDomains[0]; + else: + raise Exception("There are no rdbComputeDomains available for the user."); + + if databaseContextName is None: + databaseContexts = rdbComputeDomain.get('databaseContexts'); + if databaseContexts.__len__() > 0: + databaseContextName = databaseContexts[0].get('name') + else: + raise Exception("rbdComputeDomain has no database contexts available for the user."); + + if resultsName is None: + resultsName = 'results' + + targets = [{'location':resultsName+'.csv', 'type':'FILE_CSV', 'resultNumber':1}] + rdbDomainId = rdbComputeDomain.get('id'); + + dockerJobModel = { + "inputSql": sqlQuery, + "submitterDID": jobAlias, + "databaseContextName": databaseContextName, + "rdbDomainId": rdbDomainId, + "targets": targets + } + + data = json.dumps(dockerJobModel).encode() + url = Config.RacmApiURL + "/jobm/rest/jobs/rdb?TaskName="+taskName; + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.post(url, data=data, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when submitting a job to the JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def cancelJob(jobId): + """ + Cancels the execution of a job. + :param jobId: Id of the job (integer) + :return: True if the job is correctly canceled. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error. + :example: job = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); isCanceled = Jobs.cancelJob(job.get('id')); + .. seealso:: Jobs.submitNotebookJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains. + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.cancelJob" + else: + taskName = "SciScript-Python.Jobs.cancelJob" + + + url = Config.RacmApiURL + "/jobm/rest/jobs/" + str(jobId) + "/cancel?TaskName="+taskName + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.post(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting from JOBM API the job status of jobId=" + str(jobId) + ".\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return True; + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def waitForJob(jobId, verbose=True, pollTime = 5): + """ + Queries regularly the job status and waits for the job to be completed. + + :param jobId: id of job (integer) + :param verbose: if True, will print "wait" messages on the screen while the job is not done. If False, will suppress printing messages on the screen. + :param pollTime: idle time interval (integer, in seconds) before a new query for the job status. Minimum value allowed is 5 seconds. + :return: After the job is finished, returns a dictionary object containing the job definition. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitShellCommandJob(dockerComputeDomain,'pwd', 'Python (astro)');Jobs.waitForJob(job.get('id')) + .. seealso:: Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.submitNotebookJob, Jobs.submitShellCommandJob + """ + try: + minPollTime = 5 # in seconds + complete = False + waitingStr = "Waiting..." + back = "\b" * len(waitingStr) + if verbose: + print waitingStr, + + while not complete: + if verbose: + #print back, + print waitingStr, + jobDesc = getJobDescription(jobId) + jobStatus = jobDesc["status"] + if jobStatus >= 32: + complete = True + if verbose: + #print back, + print "Done!" + else: + time.sleep(max(minPollTime,pollTime)); + + return jobDesc + except Exception as e: + raise e; + diff --git a/py3/SciServer/Authentication.py b/py3/SciServer/Authentication.py index 35a9986..ede38b6 100644 --- a/py3/SciServer/Authentication.py +++ b/py3/SciServer/Authentication.py @@ -6,7 +6,7 @@ import requests import os.path import warnings - +import getpass from SciServer import Config @@ -63,7 +63,7 @@ def getKeystoneUserWithToken(token): return ksu -def login(UserName, Password): +def login(UserName=None, Password=None): """ Logs the user into SciServer and returns the authentication token. This function is useful when SciScript-Python library methods are executed outside the SciServer-Compute environment. @@ -71,12 +71,11 @@ def login(UserName, Password): so the user has to use Authentication.login in order to log into SciServer manually and get the authentication token. Authentication.login also sets the token value in the python instance argument variable "--ident", and as the local object Authentication.token (of class Authentication.Token). - :param UserName: name of the user (string) - :param Password: password of the user (string) + :param UserName: name of the user (string). If not set, then a user name prompt will show. + :param Password: password of the user (string). If not set, then a password prompt will show. :return: authentication token (string) :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. - :example: token = Authentication.login('loginName','loginPassword') - + :example: token1 = Authentication.login(); token2 = Authentication.login('loginName','loginPassword'); .. seealso:: Authentication.getKeystoneUserWithToken, Authentication.getToken, Authentication.setToken, Authentication.token. """ taskName = "" @@ -87,7 +86,14 @@ def login(UserName, Password): loginURL = Config.AuthenticationURL + "?TaskName=" + taskName - authJson = {"auth":{"identity":{"password":{"user":{"name":UserName,"password":Password}}}}} + userName = UserName; + password = Password; + if userName is None: + userName = getpass.getpass(prompt="Enter SciServer user name: ") + if password is None: + password = getpass.getpass(prompt="Enter SciServer password: ") + + authJson = {"auth":{"identity":{"password":{"user":{"name":userName,"password":password}}}}} data = json.dumps(authJson).encode() diff --git a/py3/SciServer/Config.py b/py3/SciServer/Config.py index 36981ec..4a23a0f 100644 --- a/py3/SciServer/Config.py +++ b/py3/SciServer/Config.py @@ -12,6 +12,8 @@ - **Config.SkyServerWSurl**: defines the base URL of the SkyServer web service API (string). E.g., "https://skyserver.sdss.org" +- **Config.RacmApiURL**: defines the base URL of the RACM API (string). E.g., "https://www.sciserver.org/racm" + - **Config.DataRelease**: defines the SDSS data release (string), to be used to build the full SkyServer API url along with Config.SkyServerWSurl. E.g., "DR13" - **Config.KeystoneTokenPath**: defines the local path (string) to the file containing the user's authentication token in the SciServer-Compute environment. E.g., "/home/idies/keystone.token". Unlikely to change since it is hardcoded in SciServer-Compute. @@ -24,11 +26,11 @@ SciDriveHost = "https://www.scidrive.org" SkyQueryUrl = "http://voservices.net/skyquery/Api/V1" SkyServerWSurl = "https://skyserver.sdss.org" +RacmApiURL = "http://scitest12.pha.jhu.edu/racm" DataRelease = "DR13" -KeystoneTokenPath = "/home/idies/keystone.token" #this path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment +KeystoneTokenPath = "/home/idies/keystone.token" #the path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment version = "sciserver-v1.10.0" #sciserver release version - - +ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. # returns TRUE if the library is running inside the SciServer-Compute, and FALSE if not import os; diff --git a/py3/SciServer/Files.py b/py3/SciServer/Files.py new file mode 100644 index 0000000..13b11d2 --- /dev/null +++ b/py3/SciServer/Files.py @@ -0,0 +1,318 @@ +__author__ = 'mtaghiza' + +from SciServer import Authentication, Config +import requests +import json +from io import StringIO +from io import BytesIO + + + +def getFileServices(): + """ + Gets the definitions of the file services that a user is able to access. + :return: list of dictionaries, where each dictionary is is the descriptions of a FileService that a user is able to access. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServices = Files.getFileServices(); + .. seealso:: Files.getFileServiceFromName + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.getFileServices" + else: + taskName = "SciScript-Python.Files.getFileServices" + + url = Config.RacmApiURL + "/storem?TaskName="+taskName; + + headers = {'X-Auth-Token': token} + res = requests.get(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + return json.loads(res.content.decode()); + else: + raise Exception("Error when getting the list of FileServices.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def getFileServiceFromName(fileServiceName, fileServices=None): + """ + Returns a FileService object, given its registered name. + :param fileServiceName: name of the FileService, as shown within the results of Jobs.getFileServices() + :param fileServices: a list of FileService objects (dictionaries), as returned by Jobs.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. + :return: a FileService object (dictionary) that defines a FileService. A list of these kind of objects available to the user is returned by the function Jobs.getFileServices(). + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileService = Files.getFileServiceFromName('fileServiceAtJHU'); + .. seealso:: Files.getFileServices + """ + + if fileServiceName is None: + raise Exception("fileServiceName is not defined.") + else: + if fileServices is None: + fileServices = getFileServices(); + + if fileServices.__len__() > 0: + for fileService in fileServices: + if fileServiceName == fileService.get('name'): + return fileService; + else: + raise Exception("There are no FileServices available for the user."); + + raise Exception("FileService of name '" + fileServiceName + "' is not available or does not exist."); + + + +def __getFileServiceAPIUrl(fileService=None): + """ + Gets the API endpoint URL of a FileService. + :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made, and the returned URL is that of the first FileService found. + :return: API endpoint URL of the FileService (string). + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServiceUrl = Files.__getFileServiceAPIUrl(); + .. seealso:: Files.getFileServiceFromName + """ + return 'http://scitest12.pha.jhu.edu/fileservice/'; # for now + + if fileService is None: + fileServices = getFileServices(); + if fileServices.__len__() > 0: + return fileServices[0].get("URL"); + else: + raise Exception("There are no FileServices available for the user."); + + else: + return fileService.get("URL"); + + +def createDir(path, volume, ownerName=None, fileService=None): + """ + Create a directory. + :param path: path (string) to the directory, relative to the volume level. E.g: path="/newDirectory" + :param volume: volume (string) containing the path. + :param ownerName: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. + :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :return: True if the directory creation was successful. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.createDir("myNewDir","persistent","myUserName", fileServices[0]); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.createDir" + else: + taskName = "SciScript-Python.Files.createDir" + + if ~path.startswith("/"): + path = "/" + path; + + if ownerName is None: + ownerName = Authentication.getKeystoneUserWithToken().userName; + + url = Config.FileServiceURL + "/folder/" + volume + "/" + ownerName + path + "?TaskName="+taskName; + + headers = {'X-Auth-Token': token} + res = requests.put(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + return True; + else: + raise Exception("Error when creating directory '" + path + "' in volume '" + volume + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def dirList(path, volume, ownerName=None, fileService=None, level=1, options=''): + """ + Lists the contents of a directory. + :param path: path (string) to the directory, relative to the volume level. E.g: path="/newDirectory" + :param volume: volume (string) containing the path. + :param ownerName: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. + :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param level: amount (int) of listed directory levels that are below or at the same level of the path. + :param options: string of file filtering options. + :return: dictionary containing the directory listing. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); dirs = Files.dirList("/","persistent","myUserName", fileServices[0], 2); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.createDir + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.dirList" + else: + taskName = "SciScript-Python.Files.dirList" + + if ownerName is None: + ownerName = Authentication.getKeystoneUserWithToken().userName; + + if ~path.startswith("/"): + path = "/" + path; + + url = Config.FileServiceURL + "/jsontree/" + volume + "/" + ownerName + path + "?options=" + str(options) + "&level=" + str(level) + "&TaskName=" + taskName; + + headers = {'X-Auth-Token': token} + res = requests.get(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + return res.content.decode(); + else: + raise Exception("Error when creating new directory " + path + ".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + + + +def delete(path, volume, ownerName=None, fileService=None): + """ + Deletes a directory or file in the File System. + :param path: path (string) to the directory or file to be deleted, relative to the volume level. E.g: path="/directory/file.txt" + :param volume: volume (string) containing the path. + :param ownerName: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. + :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :return: True if the deletion was successful. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.delete" + else: + taskName = "SciScript-Python.Files.delete" + + if ~path.startswith("/"): + path = "/" + path; + + url = Config.FileServiceURL + "/data" + path + "?TaskName="+taskName + + headers = {'X-Auth-Token': token} + res = requests.delete(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + return True; + else: + raise Exception("Error when deleting " + path + ".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + + +def upload(path, volume, ownerName=None, fileService=None, data=None, localFilePath=None): + """ + Uploads data or a local file into a path defined in the file system. + :param path: destination path (string) of the file to be uploaded, relative to the volume level. E.g: path="/newDirectory/myNewFile.txt" + :param volume: volume (string) containing the path. + :param ownerName: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. + :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param data: string containing data to be uploaded. + :param localFilePath: path to a local file to be uploaded (string). + :return: True is upload was successful. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); isUploaded = Files.upload("/myUploadedFile.txt","persistent","myUserName", fileServices[0], localFilePath="/myFile.txt"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.upload" + else: + taskName = "SciScript-Python.Files.upload" + + if ~path.startswith("/"): + path = "/" + path; + + fileName = path.split("/")[-1] + dirPath = path.replace(fileName, "") + + createDir(dirPath); + + url = Config.FileServiceURL + "/data" + path + "?TaskName="+taskName + + headers = {'X-Auth-Token': token} + + if localFilePath is not None and localFilePath != "": + with open(localFilePath, "rb") as file: + res = requests.put(url, data=file, headers=headers, stream=True) + else: + if data != None: + res = requests.put(url, data=data, headers=headers, stream=True) + else: + raise Exception("Error when uploading to " + path + ". No local file or data specified for uploading."); + + if res.status_code >= 200 and res.status_code < 300: + return True; + else: + raise Exception("Error when uploading to " + path +".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def download(path, volume, ownerName=None, fileService=None, localFilePath="", format="text"): + """ + Downloads a file from the remote file system into the local file system, or returns the file content as an object in several formats. + :param path: remote path (string) of the file to be downloaded, relative to the volume level. E.g: path="/newDirectory/myNewFile.txt" + :param volume: volume (string) containing the path. + :param ownerName: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. + :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param localFilePath: local path of the file to be downloaded. If 'localFilePath' is defined, then the 'format' parameter is not used. + :param format: type of the returned object. Can be "StringIO" (io.StringIO object containing readable text), "BytesIO" (io.BytesIO object containing readable binary data), "response" ( the HTTP response as an object of class requests.Response) or "text" (a text string). If the parameter 'localFilePath' is defined, then the 'format' parameter is not used and the file is downloaded to the local file system instead. + :return: If the 'localFilePath' parameter is defined, then it will return True when the file is downloaded successfully in the local file system. If the 'localFilePath' is not defined, then the type of the returned object depends on the value of the 'format' parameter (either io.StringIO, io.BytesIO, requests.Response or string). + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); isDownloaded = Files.upload("/myUploadedFile.txt","persistent","myUserName", fileServices[0], localFilePath="/myDownloadedFile.txt"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.upload, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.download" + else: + taskName = "SciScript-Python.Files.download" + + if ~path.startswith("/"): + path = "/" + path; + + url = Config.FileServiceURL + "/data" + path + "?TaskName="+taskName; + headers = {'X-Auth-Token': token} + + res = requests.get(url, stream=True, headers=headers) + + if res.status_code < 200 or res.status_code >= 300: + raise Exception("Error when downloading from remote File Syatem the file " + str(path) + ".\nHttp Response from FileSystem API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + + if localFilePath is not None and localFilePath != "": + + bytesio = BytesIO(res.content) + theFile = open(localFilePath, "w+b") + theFile.write(bytesio.read()) + theFile.close() + return True + + else: + + if format is not None and format != "": + if format == "StringIO": + return StringIO(res.content.decode()) + if format == "text": + return res.content.decode() + elif format == "BytesIO": + return BytesIO(res.content) + elif format == "response": + return res; + else: + raise Exception("Unknown format '" + format + "' when trying to download from remote File System the file " + str(path) + ".\n"); + else: + raise Exception("Wrong format parameter value\n"); + + else: + raise Exception("User token is not defined. First log into SciServer.") diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py new file mode 100644 index 0000000..6315b9a --- /dev/null +++ b/py3/SciServer/Jobs.py @@ -0,0 +1,599 @@ +__author__ = 'mtaghiza' + +import sys +import os; +import os.path; +from SciServer import Authentication, Config +import requests +import json +import time; + + + +def getJobDirectory(): + """ + Gets the root directory path where the job is being executed. Since its value depends on the value of jobId, the path will be defined only after the job is submitted. + """ + computeJobDirectoryFile = Config.ComputeJobDirectoryFile + if os.path.isfile(computeJobDirectoryFile): + jobDirectory = None; + with open(computeJobDirectoryFile, 'r') as f: + jobDirectory = f.read().rstrip('\n') + + return jobDirectory + else: + raise OSError("Cannot find job execution directory") + + +def getDockerComputeDomains(): + """ + Gets a list of all registered Docker compute domains that the user has access to. + :return: a list of dictionaries, each one containing the definition of a Docker compute domain. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: dockerComputeDomains = Jobs.getDockerComputeDomains(); + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getRDBComputeDomains, Jobs.cancelJob + """ + token = Authentication.getToken() + if token is not None and token != "": + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.getDockerComputeDomains" + else: + taskName = "SciScript-Python.Jobs.getDockerComputeDomains" + + url = Config.RacmApiURL + "/jobm/rest/computedomains?batch=true&interactive=false&TaskName=" + taskName + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting Docker Compute Domains from JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def getRDBComputeDomains(): + """ + Gets a list of all registered Relational Database (RDB) compute domains that the user has access to. + :return: a list of dictionaries, each one containing the definition of an RDB compute domain. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: rdbComputeDomains = Jobs.getRDBComputeDomains(); + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.getRDBComputeDomains" + else: + taskName = "SciScript-Python.Jobs.getRDBComputeDomains" + + url = Config.RacmApiURL + "/jobm/rest/computedomains/rdb?TaskName=" + taskName + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + if res.status_code != 200: + raise Exception("Error when getting RDB Compute Domains from JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def getDockerComputeDomainFromName(dockerComputeDomainName, dockerComputeDomains = None): + """ + Returns a DockerComputeDomain object, given its registered name. + :param dockerComputeDomainName: name of the DockerComputeDomainName, as shown within the results of Jobs.getDockerComputeDomains() + :param dockerComputeDomains: a list of DockerComputeDomain objects (dictionaries), as returned by Jobs.getDockerComputeDomains(). If not set, then an internal call to Jobs.getDockerComputeDomains() is made. + :return: a DockerComputeDomain object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: dockerComputeDomain = Jobs.getDockerComputeDomainFromName('dockerComputeDomainAtJHU'); + .. seealso:: Jobs.getDockerComputeDomains, Jobs.getRDBComputeDomains, Jobs.getRDBComputeDomainFromName + """ + if dockerComputeDomainName is None: + raise Exception("dockerComputeDomainName is not defined.") + else: + if dockerComputeDomains is None: + dockerComputeDomains = getDockerComputeDomains(); + + if dockerComputeDomains.__len__() > 0: + for dockerComputeDomain in dockerComputeDomains: + if dockerComputeDomainName == dockerComputeDomain.get('name'): + return dockerComputeDomain; + else: + raise Exception("There are no DockerComputeDomains available for the user."); + + raise Exception("DockerComputeDomain of name '" + dockerComputeDomainName + "' is not available or does not exist."); + + +def getRDBComputeDomainFromName(rdbComputeDomainName, rdbComputeDomains = None): + """ + Returns an RDBComputeDomain object, given its registered name. + :param rdbComputeDomainName: name of the RDBComputeDomainName, as shown within the results of Jobs.getRDBComputeDomains() + :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. + :return: an RDBComputeDomain object (dictionary) that defines an RDB compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: rdvComputeDomain = Jobs.getRDBComputeDomainFromName('rdbComputeDomainAtJHU'); + .. seealso:: Jobs.getDockerComputeDomains, Jobs.getRDBComputeDomains, Jobs.getDockerComputeDomainFromName + """ + if rdbComputeDomainName is None: + raise Exception("rdbComputeDomainName is not defined.") + else: + if rdbComputeDomains is None: + rdbComputeDomains = getRDBComputeDomains(); + + if rdbComputeDomains.__len__() > 0: + for rdbComputeDomain in rdbComputeDomains: + if rdbComputeDomainName == rdbComputeDomain.get('name'): + return rdbComputeDomain; + else: + raise Exception("There are no RDBComputeDomains available for the user."); + + raise Exception("RDBComputeDomain of name '" + rdbComputeDomainName + "' is not available or does not exist."); + + +def getJobsList(top=10, open=None, start=None, end=None, type='all'): + """ + Gets the list of Jobs submitted by the user. + :param top: top number of jobs (integer) returned. If top=None, then all jobs are returned. + :param open: If set to 'True', then only returns jobs that have not finished executing and wrapped up (status <= FINISHED). If set to 'False' then only returnes jobs that are still running. If set to 'None', then returns both finished and unfinished jobs. + :param start: The earliest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no lower bound on date. + :param end: The latest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no upper bound on date. + :param type: type (string) of jobs returned. Can take values of 'rdb' (for returning only relational database jobs), 'docker' (for returning only Docker jobs) and 'all' (all job types are returned). + :return: a list of dictionaries, each one containing the definition of a submitted job. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. + :example: jobs = Jobs.getJobsList(top=2); + .. seealso:: Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.getJobsList" + else: + taskName = "SciScript-Python.Jobs.getJobsList" + + + topString = ("top=" + str(top) + "&" if top != None else "") + startString = ("start=" + str(start) + "&" if start != None else "") + endString = ("end=" + str(end) + "&" if end != None else "") + if open is None: + openString = ""; + else: + if open is True: + openString = "open=true&"; + else: + openString = "open=false&"; + + url = Config.RacmApiURL + "/jobm/rest/jobs?" + if(type=='rdb'): + url = Config.RacmApiURL + "/jobm/rest/rdbjobs?" + if(type=='docker'): + url = Config.RacmApiURL + "/jobm/rest/dockerjobs?" + + url = url + topString + startString + endString + "TaskName=" + taskName; + + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting list of jobs from JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + +def getJobDescription(jobId): + """ + Gets the definition of the job, + :param jobId: Id of job + :return: dictionary containing the description or definition of the job. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. + :example: job1 = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); job2 = Jobs.getJobDescription(job1.get('id')); + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.getJobDescription" + else: + taskName = "SciScript-Python.Jobs.getJobDescription" + + url = Config.RacmApiURL + "/jobm/rest/jobs/" + str(jobId) + "?TaskName="+taskName + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting from JOBM API the job status of jobId=" + str(jobId) + ".\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def getJobStatus(jobId): + """ + Gets a dictionary with the job status as an integer value, together with its semantic meaning. The integer value is a power of 2, that is, 1:PENDING, 2:QUEUED, 4:ACCEPTED, 8:STARTED, 16:FINISHED, 32:SUCCESS, 64:ERROR and 128:CANCELED + :param jobId: Id of job (integer). + :return: dictionary with the integer value of the job status, as well as its semantic meaning. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. + :example: job = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); status = Jobs.getJobStatus(job.get('id')); + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + intStatus = getJobDescription(jobId)["status"] + + if intStatus == 1: + return {'status':"PENDING", 'intStatus':intStatus} + elif intStatus == 2: + return {'status': "QUEUED", 'intStatus': intStatus} + elif intStatus == 4: + return {'status': "ACCEPTED", 'intStatus': intStatus} + elif intStatus == 8: + return {'status': "STARTED", 'intStatus': intStatus} + elif intStatus == 16: + return {'status': "FINISHED", 'intStatus': intStatus} + elif intStatus == 32: + return {'status': "SUCCESS", 'intStatus': intStatus} + elif intStatus == 64: + return {'status': "ERROR", 'intStatus': intStatus} + elif intStatus == 128: + return {'status': "CANCELED", 'intStatus': intStatus} + else: + raise Exception("Invalid integer value given to job status.") + + +def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=None, userVolumes=None, dataVolumes=None, parameters="", jobAlias= ""): + """ + Submits a Jupyter Notebook for execution (as an asynchronous job) inside a Docker compute domain, + :param notebookPath: path of the notebook within the filesystem mounted in SciServer-Compute (string). Example: notebookPath = '/home/idies/worskpace/persistent/JupyterNotebook.ipynb' + :param dockerComputeDomain: object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). + :param dockerImageName: name (string) of the Docker image for executing the notebook. E.g., dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object. + :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image. + E.g., userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', , 'needsWriteAccess':True}] + A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=None, then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume. + :param dataVolumes: a list with the names of data volumes that will be mounted to the docker Image. + E.g., dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}]. + A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted. + :param parameters: string containing parameters that the notebook might need during its execution. This string is written in the 'parameters.txt' file in the same directory level where the notebook is being executed. + :param jobAlias: alias (string) of job, defined by the user. + :return: a dictionary containing the definition of the submitted job. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. + :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitNotebookJob('/home/idies/workspace/persistent/Notebook.ipynb', dockerComputeDomain, 'Python (astro)', [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}], [{'name':'SDSS_DAS'}], 'param1=1\nparam2=2\nparam3=3','myNewJob') + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.submitNotebookJob" + else: + taskName = "SciScript-Python.Jobs.submitNotebookJob" + + if dockerComputeDomain is None: + dockerComputeDomains = getDockerComputeDomains(); + if dockerComputeDomains .__len__() > 0: + dockerComputeDomain = dockerComputeDomains[0]; + else: + raise Exception("There are no dockerComputeDomains available for the user."); + + if dockerImageName is None: + images = dockerComputeDomain.get('images'); + if images.__len__() > 0: + dockerImageName = images[0].get('name') + else: + raise Exception("dockerComputeDomain has no docker images available for the user."); + + uVols = []; + if userVolumes is None: + for vol in dockerComputeDomain.get('userVolumes'): + if 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + + + else: + for uVol in userVolumes: + found = False; + for vol in dockerComputeDomain.get('userVolumes'): + if vol.get('name') == uVol.get('name'): + found = True; + if (uVol.has_key('needsWriteAccess')): + if uVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + else: + if 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + + if not found: + raise Exception("User volume '" + uVol.get('name') + "' not found within Compute domain") + + datVols = []; + if dataVolumes is None: + for vol in dockerComputeDomain.get('volumes'): + datVols.append({'name': vol.get('name')}); + + else: + for dVol in dataVolumes: + found = False; + for vol in dockerComputeDomain.get('volumes'): + if vol.get('name') == dVol.get('name'): + found = True; + datVols.append({'name': vol.get('name')}); + + if not found: + raise Exception("Data volume '" + dVol.get('name') + "' not found within Compute domain") + + + dockerComputeEndpoint = dockerComputeDomain.get('apiEndpoint'); + + dockerJobModel = { + "command": parameters, + "scriptURI": notebookPath, + "submitterDID": jobAlias, + "dockerComputeEndpoint": dockerComputeEndpoint, + "dockerImageName": dockerImageName, + "volumeContainers": datVols, + "userVolumes": uVols + } + data = json.dumps(dockerJobModel).encode() + url = Config.RacmApiURL + "/jobm/rest/jobs/docker?TaskName="+taskName; + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.post(url, data=data, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when submitting a notebook job to the JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageName = None, userVolumes = None, dataVolumes = None, jobAlias = ""): + """ + Submits a shell command for execution (as an asynchronous job) inside a Docker compute domain. + :param shellCommand: shell command (string) defined by the user. + :param dockerComputeDomain: object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). + :param dockerImageName: name (string) of the Docker image for executing the notebook. E.g., dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object. + :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image. + E.g., userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', , 'needsWriteAccess':True}] + A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=None, then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume. + :param dataVolumes: a list with the names of data volumes that will be mounted to the docker Image. + E.g., dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}]. + A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted. + :param jobAlias: alias (string) of job, defined by the user. + :return: a dictionary containing the definition of the submitted job. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. + :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitShellCommandJob('pwd', dockerComputeDomain, 'Python (astro)', [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}], [{'name':'SDSS_DAS'}], 'myNewJob') + .. seealso:: Jobs.submitNotebookJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.submitShellCommandJob" + else: + taskName = "SciScript-Python.Jobs.submitShellCommandJob" + + + if dockerComputeDomain is None: + dockerComputeDomains = getDockerComputeDomains(); + if dockerComputeDomains .__len__() > 0: + dockerComputeDomain = dockerComputeDomains[0]; + else: + raise Exception("There are no dockerComputeDomains available for the user."); + + if dockerImageName is None: + images = dockerComputeDomain.get('images'); + if images.__len__() > 0: + dockerImageName = images[0].get('name') + else: + raise Exception("dockerComputeDomain has no docker images available for the user."); + + uVols = []; + if userVolumes is None: + for vol in dockerComputeDomain.get('userVolumes'): + if 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + + + else: + for uVol in userVolumes: + found = False; + for vol in dockerComputeDomain.get('userVolumes'): + if vol.get('name') == uVol.get('name'): + found = True; + if (uVol.has_key('needsWriteAccess')): + if uVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + else: + if 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + + if not found: + raise Exception("User volume '" + uVol.get('name') + "' not found within Compute domain") + + datVols = []; + if dataVolumes is None: + for vol in dockerComputeDomain.get('volumes'): + datVols.append({'name': vol.get('name')}); + + else: + for dVol in dataVolumes: + found = False; + for vol in dockerComputeDomain.get('volumes'): + if vol.get('name') == dVol.get('name'): + found = True; + datVols.append({'name': vol.get('name')}); + + if not found: + raise Exception("Data volume '" + dVol.get('name') + "' not found within Compute domain") + + + dockerComputeEndpoint = dockerComputeDomain.get('apiEndpoint'); + + dockerJobModel = { + "command": shellCommand, + "submitterDID": jobAlias, + "dockerComputeEndpoint": dockerComputeEndpoint, + "dockerImageName": dockerImageName, + "volumeContainers": datVols, + "userVolumes": uVols + } + data = json.dumps(dockerJobModel).encode() + url = Config.RacmApiURL + "/jobm/rest/jobs/docker?TaskName="+taskName; + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.post(url, data=data, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when submitting a job to the JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + +def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = None, resultsName=None, jobAlias = ""): + """ + Submits a sql query for execution (as an asynchronous job) inside a relational database (RDB) compute domain. + :param sqlQuery: sql query (string) + :param rdbComputeDomain: object (dictionary) that defines a relational database (RDB) compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). + :param databaseContextName: database context name (string) on which the sql query is executed. + :param resultsName: name (string) of the table or file containing the query result. + :param jobAlias: alias (string) of job, defined by the user. + :return: a dictionary containing the definition of the submitted job. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. + :example: job = Jobs.submitRDBQueryJob('select 1';,None, None, 'myQueryResults', 'myNewJob') + .. seealso:: Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.submitRDBQueryJob" + else: + taskName = "SciScript-Python.Jobs.submitRDBQueryJob" + + if rdbComputeDomain is None: + rdbComputeDomains = getRDBComputeDomains(); + if rdbComputeDomains .__len__() > 0: + rdbComputeDomain = rdbComputeDomains[0]; + else: + raise Exception("There are no rdbComputeDomains available for the user."); + + if databaseContextName is None: + databaseContexts = rdbComputeDomain.get('databaseContexts'); + if databaseContexts.__len__() > 0: + databaseContextName = databaseContexts[0].get('name') + else: + raise Exception("rbdComputeDomain has no database contexts available for the user."); + + if resultsName is None: + resultsName = 'results' + + targets = [{'location':resultsName+'.csv', 'type':'FILE_CSV', 'resultNumber':1}] + rdbDomainId = rdbComputeDomain.get('id'); + + dockerJobModel = { + "inputSql": sqlQuery, + "submitterDID": jobAlias, + "databaseContextName": databaseContextName, + "rdbDomainId": rdbDomainId, + "targets": targets + } + + data = json.dumps(dockerJobModel).encode() + url = Config.RacmApiURL + "/jobm/rest/jobs/rdb?TaskName="+taskName; + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.post(url, data=data, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when submitting a job to the JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def cancelJob(jobId): + """ + Cancels the execution of a job. + :param jobId: Id of the job (integer) + :return: True if the job is correctly canceled. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error. + :example: job = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); isCanceled = Jobs.cancelJob(job.get('id')); + .. seealso:: Jobs.submitNotebookJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains. + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.cancelJob" + else: + taskName = "SciScript-Python.Jobs.cancelJob" + + + url = Config.RacmApiURL + "/jobm/rest/jobs/" + str(jobId) + "/cancel?TaskName="+taskName + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.post(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting from JOBM API the job status of jobId=" + str(jobId) + ".\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return True; + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def waitForJob(jobId, verbose=True, pollTime = 5): + """ + Queries regularly the job status and waits for the job to be completed. + + :param jobId: id of job (integer) + :param verbose: if True, will print "wait" messages on the screen while the job is not done. If False, will suppress printing messages on the screen. + :param pollTime: idle time interval (integer, in seconds) before a new query for the job status. Minimum value allowed is 5 seconds. + :return: After the job is finished, returns a dictionary object containing the job definition. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitShellCommandJob(dockerComputeDomain,'pwd', 'Python (astro)');Jobs.waitForJob(job.get('id')) + .. seealso:: Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.submitNotebookJob, Jobs.submitShellCommandJob + """ + try: + minPollTime = 5 # in seconds + complete = False + waitingStr = "Waiting..." + back = "\b" * len(waitingStr) + if verbose: + print(waitingStr, end="") + + while not complete: + if verbose: + #print back, + print(waitingStr, end="") + jobDesc = getJobDescription(jobId) + jobStatus = jobDesc["status"] + if jobStatus >= 32: + complete = True + if verbose: + #print back, + print("Done!") + else: + time.sleep(max(minPollTime,pollTime)); + + return jobDesc + except Exception as e: + raise e; + From b3666192b0b1bf33b725c2ed2dbe3ea74a7b9e5f Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Fri, 19 Jan 2018 15:23:32 -0500 Subject: [PATCH 18/79] Updated Files module with FileServices information --- py2/SciServer/Files.py | 10 ++++------ py3/SciServer/Files.py | 10 ++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/py2/SciServer/Files.py b/py2/SciServer/Files.py index 13b11d2..42cea58 100644 --- a/py2/SciServer/Files.py +++ b/py2/SciServer/Files.py @@ -24,7 +24,7 @@ def getFileServices(): else: taskName = "SciScript-Python.Files.getFileServices" - url = Config.RacmApiURL + "/storem?TaskName="+taskName; + url = Config.RacmApiURL + "/storem/fileservices?TaskName="+taskName; headers = {'X-Auth-Token': token} res = requests.get(url, headers=headers) @@ -44,7 +44,7 @@ def getFileServiceFromName(fileServiceName, fileServices=None): :param fileServices: a list of FileService objects (dictionaries), as returned by Jobs.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. :return: a FileService object (dictionary) that defines a FileService. A list of these kind of objects available to the user is returned by the function Jobs.getFileServices(). :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. - :example: fileService = Files.getFileServiceFromName('fileServiceAtJHU'); + :example: fileService = Files.getFileServiceFromName('FileServiceAtJHU'); .. seealso:: Files.getFileServices """ @@ -74,17 +74,15 @@ def __getFileServiceAPIUrl(fileService=None): :example: fileServiceUrl = Files.__getFileServiceAPIUrl(); .. seealso:: Files.getFileServiceFromName """ - return 'http://scitest12.pha.jhu.edu/fileservice/'; # for now - if fileService is None: fileServices = getFileServices(); if fileServices.__len__() > 0: - return fileServices[0].get("URL"); + return fileServices[0].get("apiEndpoint"); else: raise Exception("There are no FileServices available for the user."); else: - return fileService.get("URL"); + return fileService.get("apiEndpoint"); def createDir(path, volume, ownerName=None, fileService=None): diff --git a/py3/SciServer/Files.py b/py3/SciServer/Files.py index 13b11d2..42cea58 100644 --- a/py3/SciServer/Files.py +++ b/py3/SciServer/Files.py @@ -24,7 +24,7 @@ def getFileServices(): else: taskName = "SciScript-Python.Files.getFileServices" - url = Config.RacmApiURL + "/storem?TaskName="+taskName; + url = Config.RacmApiURL + "/storem/fileservices?TaskName="+taskName; headers = {'X-Auth-Token': token} res = requests.get(url, headers=headers) @@ -44,7 +44,7 @@ def getFileServiceFromName(fileServiceName, fileServices=None): :param fileServices: a list of FileService objects (dictionaries), as returned by Jobs.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. :return: a FileService object (dictionary) that defines a FileService. A list of these kind of objects available to the user is returned by the function Jobs.getFileServices(). :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. - :example: fileService = Files.getFileServiceFromName('fileServiceAtJHU'); + :example: fileService = Files.getFileServiceFromName('FileServiceAtJHU'); .. seealso:: Files.getFileServices """ @@ -74,17 +74,15 @@ def __getFileServiceAPIUrl(fileService=None): :example: fileServiceUrl = Files.__getFileServiceAPIUrl(); .. seealso:: Files.getFileServiceFromName """ - return 'http://scitest12.pha.jhu.edu/fileservice/'; # for now - if fileService is None: fileServices = getFileServices(); if fileServices.__len__() > 0: - return fileServices[0].get("URL"); + return fileServices[0].get("apiEndpoint"); else: raise Exception("There are no FileServices available for the user."); else: - return fileService.get("URL"); + return fileService.get("apiEndpoint"); def createDir(path, volume, ownerName=None, fileService=None): From 9638c948518e7691a893a6e9bb1ac06cfa5dc058 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Sat, 3 Feb 2018 20:13:54 -0500 Subject: [PATCH 19/79] Updated Jobs and Files modules, including tests. --- ....ipynb => Examples_SciScript-Python.ipynb} | 625 +++++++++++++++++- ...ython3.py => Examples_SciScript-Python.py} | 369 ++++++++++- Examples/TestNotebook.ipynb | 154 +++++ Tests/TestNotebook.ipynb | 106 ++- Tests/UnitTests.py | 437 +++++++----- py2/SciServer/Authentication.py | 48 +- py2/SciServer/Config.py | 2 +- py2/SciServer/Files.py | 495 ++++++++++---- py2/SciServer/Jobs.py | 119 ++-- py3/SciServer/Authentication.py | 46 +- py3/SciServer/Config.py | 2 +- py3/SciServer/Files.py | 495 ++++++++++---- py3/SciServer/Jobs.py | 123 ++-- 13 files changed, 2456 insertions(+), 565 deletions(-) rename Examples/{Examples_SciScript-Python3.ipynb => Examples_SciScript-Python.ipynb} (61%) rename Examples/{Examples_SciScript-Python3.py => Examples_SciScript-Python.py} (58%) create mode 100644 Examples/TestNotebook.ipynb diff --git a/Examples/Examples_SciScript-Python3.ipynb b/Examples/Examples_SciScript-Python.ipynb similarity index 61% rename from Examples/Examples_SciScript-Python3.ipynb rename to Examples/Examples_SciScript-Python.ipynb index 0be144c..8b219a2 100644 --- a/Examples/Examples_SciScript-Python3.ipynb +++ b/Examples/Examples_SciScript-Python.ipynb @@ -10,7 +10,7 @@ "outputs": [], "source": [ "import SciServer\n", - "from SciServer import Authentication, LoginPortal, Config, CasJobs, SkyQuery, SciDrive, SkyServer\n", + "from SciServer import Authentication, LoginPortal, Config, CasJobs, SkyQuery, SciDrive, SkyServer, Files, Jobs\n", "import os;\n", "import pandas;\n", "import sys;\n", @@ -23,7 +23,10 @@ "\n", "# Define login Name and password before running these examples\n", "Authentication_loginName = '***';\n", - "Authentication_loginPassword = '***'" + "Authentication_loginPassword = '***'\n", + "\n", + "Authentication_login_sharedWithName = '***'\n", + "Authentication_login_sharedWithPassword = '***'" ] }, { @@ -811,6 +814,17 @@ "#help(SkyQuery)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword);" + ] + }, { "cell_type": "code", "execution_count": null, @@ -962,7 +976,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": false }, "outputs": [], "source": [ @@ -1071,6 +1085,611 @@ "result = SkyQuery.dropTable(tableName=SkyQuery_TestTableName, datasetName=\"MyDB\");\n", "print(result)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# *******************************************************************************************************\n", + "# Files section:\n", + "# *******************************************************************************************************" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "#help(Files)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# defining the FileService name\n", + "\n", + "Files_FileServiceName = \"FileServiceJHU\"\n", + "Files_RootVolumeName1 = \"volumes\"\n", + "Files_UserVolumeName1 = Authentication_loginName + \"_UserVolume555\"\n", + "Files_RootVolumeName2 = \"volumes\"\n", + "Files_UserVolumeName2 = Authentication_loginName + \"_UserVolume999\"\n", + "Files_NewDirectoryName1 = \"myNewDirectory555\"\n", + "Files_NewDirectoryName2 = \"myNewDirectory999\"\n", + "Files_LocalFileName = \"MyNewFile.txt\"\n", + "Files_LocalFileContent = \"#ID,Column1,Column2\\n1,4.5,5.5\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# get available FileServices\n", + "\n", + "fileServices = Files.getFileServices();\n", + "print(fileServices)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# get names of file services available to the user.\n", + "\n", + "fileServiceNames = Files.getFileServicesNames();\n", + "print(fileServiceNames)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# get FileService from Name\n", + "\n", + "fileService = Files.getFileServiceFromName(Files_FileServiceName);\n", + "print(fileService)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# get the API endpoint URL of a FileService\n", + "\n", + "fileServiceAPIUrl = Files.__getFileServiceAPIUrl(fileService);\n", + "print(fileServiceAPIUrl)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# get root volumes\n", + "\n", + "rootVolumes = Files.getRootVolumes(fileService)\n", + "print(rootVolumes)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# create user volume\n", + "\n", + "Files.createUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName1)\n", + "Files.createUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# create a directory in the persistent volume\n", + "\n", + "Files.createDir(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1);\n", + "Files.createDir(fileService, Files_RootVolumeName2, Files_UserVolumeName2, Files_NewDirectoryName2);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# upload a text string into a file in the remote directory\n", + "\n", + "Files.upload(fileService, Files_RootVolumeName1, Files_UserVolumeName1, \n", + " Files_NewDirectoryName1 + \"/\" + Files_LocalFileName, \n", + " data=Files_LocalFileContent);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# List content of remote directory\n", + "\n", + "dirList = Files.dirList(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1, level=2)\n", + "print(dirList)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# download a remote text file into a local directory\n", + "\n", + "Files.download(fileService, Files_RootVolumeName1, Files_UserVolumeName1, \n", + " Files_NewDirectoryName1 + \"/\" + Files_LocalFileName, \n", + " localFilePath=Files_LocalFileName);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Delete remote file\n", + "\n", + "Files.delete(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1 + \"/\" + Files_LocalFileName)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# upload a local file into a remote directory\n", + "\n", + "Files.upload(fileService, Files_RootVolumeName1, Files_UserVolumeName1,\n", + " Files_NewDirectoryName1 + \"/\" + Files_LocalFileName, localFilePath=Files_LocalFileName, quiet=False);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# copy remote file into remote directory\n", + "\n", + "Files.move(fileService, Files_RootVolumeName1, Files_UserVolumeName1,Files_NewDirectoryName1 + \"/\" + Files_LocalFileName, \n", + " fileService, Files_RootVolumeName2, Files_UserVolumeName2,Files_UserVolumeName2 + \"/\" + Files_NewDirectoryName2 + \"/\" + Files_LocalFileName\n", + " );" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# sharing user volume with another user\n", + "\n", + "Files.shareUserVolume(fileService, Files_RootVolumeName2, Files_UserVolumeName2, \n", + " sharedWith=Authentication_login_sharedWithName, type=\"USER\", \n", + " userVolumeOwner=Authentication_loginName,\n", + " allowedActions=[\"read\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# let the other user log-in\n", + "\n", + "token1 = Authentication.login(Authentication_login_sharedWithName, Authentication_login_sharedWithPassword);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# the other user downloads the remote text file (from the shared user volume) into a local string variable\n", + "\n", + "string = Files.download(fileService, Files_RootVolumeName2, Files_UserVolumeName2, \n", + " Files_NewDirectoryName2 + \"/\" + Files_LocalFileName, \n", + " format=\"txt\", userVolumeOwner=Authentication_loginName);\n", + "print(string)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# the original user logs in again\n", + "\n", + "token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# delete folders\n", + "\n", + "Files.delete(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1)\n", + "Files.delete(fileService, Files_RootVolumeName2, Files_UserVolumeName2, Files_NewDirectoryName2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# delete user volumes\n", + "\n", + "Files.deleteUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName1)\n", + "Files.deleteUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# *******************************************************************************************************\n", + "# Jobs section:\n", + "# *******************************************************************************************************" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "#help(Jobs)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "Jobs_DockerComputeDomainName = 'Small Jobs Batch Domain'\n", + "\n", + "Jobs_FileServiceName = \"FileServiceJHU\"\n", + "Jobs_RootVolumeName = \"volumes\"\n", + "Jobs_UserVolumeName = Authentication_loginName + \"_JobsTestVolume\"\n", + "Jobs_DirectoryName = \"JobsTestDirectory\"\n", + "Jobs_NotebookName = 'TestNotebook.ipynb'\n", + "Jobs_ShellCommand = 'ls -laht'\n", + "Jobs_DockerImageName = 'Python (astro)'\n", + "\n", + "Jobs_UserVolumes = [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}]\n", + "Jobs_DataVolumes = [{'name':'SDSS_DAS'}]\n", + "Jobs_Parameters = 'param1=1\\nparam2=2\\nparam3=3'\n", + "Jobs_Alias = 'MyNewJob'\n", + "\n", + "Jobs_SqlQuery = \"select 1;\"\n", + "Jobs_RBBComputeDomainName = 'Manga (long)'\n", + "Jobs_DatabaseContextName = \"manga\"\n", + "Jobs_QueryResultsFile = 'myQueryResults'\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# get docker compute domains\n", + "\n", + "dockerComputeDomains = Jobs.getDockerComputeDomains();\n", + "print(dockerComputeDomains)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# get names of docker compute domains available to the user\n", + "\n", + "dockerComputeDomainsNames = Jobs.getDockerComputeDomainsNames()\n", + "print(dockerComputeDomainsNames)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# get docker compute domain from name\n", + "\n", + "dockerComputeDomain = Jobs.getDockerComputeDomainFromName(Jobs_DockerComputeDomainName)\n", + "print(dockerComputeDomain)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# uploading Jupyter notebook to remote directory\n", + "\n", + "fileService = Files.getFileServiceFromName(Jobs_FileServiceName);\n", + "Files.createUserVolume(fileService, Jobs_RootVolumeName, Jobs_UserVolumeName)\n", + "Files.upload(fileService, Jobs_RootVolumeName, Jobs_UserVolumeName, \n", + " Jobs_DirectoryName + \"/\" + Jobs_NotebookName, \n", + " localFilePath=Jobs_NotebookName);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# submit a Jupyter notebook job.\n", + "\n", + "jobId = Jobs.submitNotebookJob('/home/idies/workspace/' + Jobs_UserVolumeName + '/' + Jobs_DirectoryName + '/' + Jobs_NotebookName,\n", + " dockerComputeDomain,\n", + " Jobs_DockerImageName, \n", + " Jobs_UserVolumes, Jobs_DataVolumes,\n", + " Jobs_Alias)\n", + "print(jobId)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# get job status\n", + "\n", + "jobStatus = Jobs.getJobStatus(jobId)\n", + "print(jobStatus)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# get job description\n", + "\n", + "job = Jobs.getJobDescription(jobId)\n", + "print(job)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# wait until job is finsihed and get job status\n", + "\n", + "jobId = Jobs.submitNotebookJob('/home/idies/workspace/' + Jobs_UserVolumeName + '/' + Jobs_DirectoryName + '/' + Jobs_NotebookName,\n", + " dockerComputeDomain,\n", + " Jobs_DockerImageName, \n", + " Jobs_UserVolumes, Jobs_DataVolumes,\n", + " Jobs_Alias)\n", + "jobStatus = Jobs.waitForJob(jobId)\n", + "print(jobStatus)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# cancel job\n", + "\n", + "jobId = Jobs.submitNotebookJob('/home/idies/workspace/' + Jobs_UserVolumeName + '/' + Jobs_DirectoryName + '/' + Jobs_NotebookName,\n", + " dockerComputeDomain,\n", + " Jobs_DockerImageName, \n", + " Jobs_UserVolumes, Jobs_DataVolumes,\n", + " Jobs_Alias)\n", + "Jobs.cancelJob(jobId)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "Files.deleteUserVolume(fileService, Jobs_RootVolumeName, Jobs_UserVolumeName, quiet=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# submit shell command job\n", + "\n", + "jobId = Jobs.submitShellCommandJob(Jobs_ShellCommand,\n", + " dockerComputeDomain,\n", + " Jobs_DockerImageName,\n", + " Jobs_UserVolumes, Jobs_DataVolumes,\n", + " Jobs_Alias)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# get relational database (RDB) compute domains\n", + "\n", + "rdbComputeDomains = Jobs.getRDBComputeDomains();\n", + "print(rdbComputeDomains)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# get names of relational database (RDB) compute domains\n", + "\n", + "rdbComputeDomainsNames = Jobs.getRDBComputeDomainsNames()\n", + "print(rdbComputeDomainsNames)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# get relational database (RDB) compute domain from name\n", + "\n", + "rdbComputeDomain = Jobs.getRDBComputeDomainFromName(Jobs_RBBComputeDomainName);\n", + "print(rdbComputeDomain)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# submit RDB (relatinal database) query job\n", + "\n", + "jobId = Jobs.submitRDBQueryJob(Jobs_SqlQuery, rdbComputeDomain, Jobs_DatabaseContextName, Jobs_QueryResultsFile, Jobs_Alias)\n", + "print(jobId)" + ] } ], "metadata": { diff --git a/Examples/Examples_SciScript-Python3.py b/Examples/Examples_SciScript-Python.py similarity index 58% rename from Examples/Examples_SciScript-Python3.py rename to Examples/Examples_SciScript-Python.py index 2e94851..9cdc89b 100644 --- a/Examples/Examples_SciScript-Python3.py +++ b/Examples/Examples_SciScript-Python.py @@ -4,7 +4,7 @@ # In[ ]: import SciServer -from SciServer import Authentication, LoginPortal, Config, CasJobs, SkyQuery, SciDrive, SkyServer +from SciServer import Authentication, LoginPortal, Config, CasJobs, SkyQuery, SciDrive, SkyServer, Files, Jobs import os; import pandas; import sys; @@ -19,6 +19,9 @@ Authentication_loginName = '***'; Authentication_loginPassword = '***' +Authentication_login_sharedWithName = '***' +Authentication_login_sharedWithPassword = '***' + # In[ ]: @@ -475,6 +478,11 @@ #help(SkyQuery) +# In[ ]: + +token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword); + + # In[ ]: #list all databses or datasets available @@ -627,3 +635,362 @@ result = SkyQuery.dropTable(tableName=SkyQuery_TestTableName, datasetName="MyDB"); print(result) + +# In[ ]: + +# ******************************************************************************************************* +# Files section: +# ******************************************************************************************************* + + +# In[ ]: + +#help(Files) + + +# In[ ]: + +# defining the FileService name + +Files_FileServiceName = "FileServiceJHU" +Files_RootVolumeName1 = "volumes" +Files_UserVolumeName1 = Authentication_loginName + "_UserVolume555" +Files_RootVolumeName2 = "volumes" +Files_UserVolumeName2 = Authentication_loginName + "_UserVolume999" +Files_NewDirectoryName1 = "myNewDirectory555" +Files_NewDirectoryName2 = "myNewDirectory999" +Files_LocalFileName = "MyNewFile.txt" +Files_LocalFileContent = "#ID,Column1,Column2\n1,4.5,5.5" + + +# In[ ]: + +token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword); + + +# In[ ]: + +# get available FileServices + +fileServices = Files.getFileServices(); +print(fileServices) + + +# In[ ]: + +# get names of file services available to the user. + +fileServiceNames = Files.getFileServicesNames(); +print(fileServiceNames) + + +# In[ ]: + +# get FileService from Name + +fileService = Files.getFileServiceFromName(Files_FileServiceName); +print(fileService) + + +# In[ ]: + +# get the API endpoint URL of a FileService + +fileServiceAPIUrl = Files.__getFileServiceAPIUrl(fileService); +print(fileServiceAPIUrl) + + +# In[ ]: + +# get root volumes + +rootVolumes = Files.getRootVolumes(fileService) +print(rootVolumes) + + +# In[ ]: + +# create user volume + +Files.createUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName1) +Files.createUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName2) + + +# In[ ]: + +# create a directory in the persistent volume + +Files.createDir(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1); +Files.createDir(fileService, Files_RootVolumeName2, Files_UserVolumeName2, Files_NewDirectoryName2); + + +# In[ ]: + +# upload a text string into a file in the remote directory + +Files.upload(fileService, Files_RootVolumeName1, Files_UserVolumeName1, + Files_NewDirectoryName1 + "/" + Files_LocalFileName, + data=Files_LocalFileContent); + + +# In[ ]: + +# List content of remote directory + +dirList = Files.dirList(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1, level=2) +print(dirList) + + +# In[ ]: + +# download a remote text file into a local directory + +Files.download(fileService, Files_RootVolumeName1, Files_UserVolumeName1, + Files_NewDirectoryName1 + "/" + Files_LocalFileName, + localFilePath=Files_LocalFileName); + + +# In[ ]: + +# Delete remote file + +Files.delete(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1 + "/" + Files_LocalFileName) + + +# In[ ]: + +# upload a local file into a remote directory + +Files.upload(fileService, Files_RootVolumeName1, Files_UserVolumeName1, + Files_NewDirectoryName1 + "/" + Files_LocalFileName, localFilePath=Files_LocalFileName, quiet=False); + + +# In[ ]: + +# copy remote file into remote directory + +Files.move(fileService, Files_RootVolumeName1, Files_UserVolumeName1,Files_NewDirectoryName1 + "/" + Files_LocalFileName, + fileService, Files_RootVolumeName2, Files_UserVolumeName2,Files_UserVolumeName2 + "/" + Files_NewDirectoryName2 + "/" + Files_LocalFileName + ); + + +# In[ ]: + +# sharing user volume with another user + +Files.shareUserVolume(fileService, Files_RootVolumeName2, Files_UserVolumeName2, + sharedWith=Authentication_login_sharedWithName, type="USER", + userVolumeOwner=Authentication_loginName, + allowedActions=["read"]) + + +# In[ ]: + +# let the other user log-in + +token1 = Authentication.login(Authentication_login_sharedWithName, Authentication_login_sharedWithPassword); + + +# In[ ]: + +# the other user downloads the remote text file (from the shared user volume) into a local string variable + +string = Files.download(fileService, Files_RootVolumeName2, Files_UserVolumeName2, + Files_NewDirectoryName2 + "/" + Files_LocalFileName, + format="txt", userVolumeOwner=Authentication_loginName); +print(string) + + +# In[ ]: + +# the original user logs in again + +token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword); + + +# In[ ]: + +# delete folders + +Files.delete(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1) +Files.delete(fileService, Files_RootVolumeName2, Files_UserVolumeName2, Files_NewDirectoryName2) + + +# In[ ]: + +# delete user volumes + +Files.deleteUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName1) +Files.deleteUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName2) + + +# In[ ]: + +# ******************************************************************************************************* +# Jobs section: +# ******************************************************************************************************* + + +# In[ ]: + +#help(Jobs) + + +# In[ ]: + +Jobs_DockerComputeDomainName = 'Small Jobs Batch Domain' + +Jobs_FileServiceName = "FileServiceJHU" +Jobs_RootVolumeName = "volumes" +Jobs_UserVolumeName = Authentication_loginName + "_JobsTestVolume" +Jobs_DirectoryName = "JobsTestDirectory" +Jobs_NotebookName = 'TestNotebook.ipynb' +Jobs_ShellCommand = 'ls -laht' +Jobs_DockerImageName = 'Python (astro)' + +Jobs_UserVolumes = [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}] +Jobs_DataVolumes = [{'name':'SDSS_DAS'}] +Jobs_Parameters = 'param1=1\nparam2=2\nparam3=3' +Jobs_Alias = 'MyNewJob' + +Jobs_SqlQuery = "select 1;" +Jobs_RBBComputeDomainName = 'Manga (long)' +Jobs_DatabaseContextName = "manga" +Jobs_QueryResultsFile = 'myQueryResults' + + +# In[ ]: + +# get docker compute domains + +dockerComputeDomains = Jobs.getDockerComputeDomains(); +print(dockerComputeDomains) + + +# In[ ]: + +# get names of docker compute domains available to the user + +dockerComputeDomainsNames = Jobs.getDockerComputeDomainsNames() +print(dockerComputeDomainsNames) + + +# In[ ]: + +# get docker compute domain from name + +dockerComputeDomain = Jobs.getDockerComputeDomainFromName(Jobs_DockerComputeDomainName) +print(dockerComputeDomain) + + +# In[ ]: + +# uploading Jupyter notebook to remote directory + +fileService = Files.getFileServiceFromName(Jobs_FileServiceName); +Files.createUserVolume(fileService, Jobs_RootVolumeName, Jobs_UserVolumeName) +Files.upload(fileService, Jobs_RootVolumeName, Jobs_UserVolumeName, + Jobs_DirectoryName + "/" + Jobs_NotebookName, + localFilePath=Jobs_NotebookName); + + +# In[ ]: + +# submit a Jupyter notebook job. + +jobId = Jobs.submitNotebookJob('/home/idies/workspace/' + Jobs_UserVolumeName + '/' + Jobs_DirectoryName + '/' + Jobs_NotebookName, + dockerComputeDomain, + Jobs_DockerImageName, + Jobs_UserVolumes, Jobs_DataVolumes, + Jobs_Alias) +print(jobId) + + +# In[ ]: + +# get job status + +jobStatus = Jobs.getJobStatus(jobId) +print(jobStatus) + + +# In[ ]: + +# get job description + +job = Jobs.getJobDescription(jobId) +print(job) + + +# In[ ]: + +# wait until job is finsihed and get job status + +jobId = Jobs.submitNotebookJob('/home/idies/workspace/' + Jobs_UserVolumeName + '/' + Jobs_DirectoryName + '/' + Jobs_NotebookName, + dockerComputeDomain, + Jobs_DockerImageName, + Jobs_UserVolumes, Jobs_DataVolumes, + Jobs_Alias) +jobStatus = Jobs.waitForJob(jobId) +print(jobStatus) + + +# In[ ]: + +# cancel job + +jobId = Jobs.submitNotebookJob('/home/idies/workspace/' + Jobs_UserVolumeName + '/' + Jobs_DirectoryName + '/' + Jobs_NotebookName, + dockerComputeDomain, + Jobs_DockerImageName, + Jobs_UserVolumes, Jobs_DataVolumes, + Jobs_Alias) +Jobs.cancelJob(jobId) + + +# In[ ]: + +Files.deleteUserVolume(fileService, Jobs_RootVolumeName, Jobs_UserVolumeName, quiet=True) + + +# In[ ]: + +# submit shell command job + +jobId = Jobs.submitShellCommandJob(Jobs_ShellCommand, + dockerComputeDomain, + Jobs_DockerImageName, + Jobs_UserVolumes, Jobs_DataVolumes, + Jobs_Alias) + + +# In[ ]: + +# get relational database (RDB) compute domains + +rdbComputeDomains = Jobs.getRDBComputeDomains(); +print(rdbComputeDomains) + + +# In[ ]: + +# get names of relational database (RDB) compute domains + +rdbComputeDomainsNames = Jobs.getRDBComputeDomainsNames() +print(rdbComputeDomainsNames) + + +# In[ ]: + +# get relational database (RDB) compute domain from name + +rdbComputeDomain = Jobs.getRDBComputeDomainFromName(Jobs_RBBComputeDomainName); +print(rdbComputeDomain) + + +# In[ ]: + +# submit RDB (relatinal database) query job + +jobId = Jobs.submitRDBQueryJob(Jobs_SqlQuery, rdbComputeDomain, Jobs_DatabaseContextName, Jobs_QueryResultsFile, Jobs_Alias) +print(jobId) + diff --git a/Examples/TestNotebook.ipynb b/Examples/TestNotebook.ipynb new file mode 100644 index 0000000..51dce8c --- /dev/null +++ b/Examples/TestNotebook.ipynb @@ -0,0 +1,154 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "import time;\n", + "import random;\n", + "import os;\n", + "from datetime import datetime" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "print(str(datetime.now()))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "now = str(datetime.now());\n", + "print(now + \" Starting Notebook execution...\")\n", + "os.system(\"echo '\" + now + \" Starting Notebook execution...'\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "# sleep time.\n", + "time.sleep(5);#5sec" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "now = str(datetime.now());\n", + "print(now + \" Now writing HelloWorld.txt file into job directory.\");\n", + "os.system(\"echo '\" + now + \" Now writing HelloWorld.txt file into job directory.'\");\n", + "\n", + "\n", + "jobsPath = None\n", + "with open('/home/idies/job.path', 'r') as f:\n", + " jobsPath = f.read().rstrip('\\n')\n", + "\n", + "if jobsPath == \"\" or jobsPath == None: \n", + " raise Exception(now + ' Could not read /home/idies/job.path') \n", + " \n", + "print(now + \" Was able to read /home/idies/job.path : \" + jobsPath);\n", + "os.system(\"echo '\" + now + \" Was able to read /home/idies/job.path : \" + jobsPath +\"'\");\n", + "\n", + "\n", + "target = open(jobsPath + \"HelloWorld.txt\", 'w')\n", + "target.truncate()\n", + "target.write(jobsPath)\n", + "target.write(\"\\n\")\n", + "target.close()\n", + "\n", + "print(now + \" Finished writing HelloWorld.txt file into job directory.\");\n", + "os.system(\"echo '\" + now + \" Finished writing HelloWorld.txt file into job directory.'\");" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "now = str(datetime.now());\n", + "parameters = None;\n", + "with open('parameters.txt') as f:\n", + " parameters = f.read().rstrip('\\n')\n", + "\n", + "if parameters is None:\n", + " raise Exception('Could not read parameters.txt')\n", + " \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "now = str(datetime.now());\n", + "print(now + \" Ending TestNotebook execution.\");\n", + "os.system(\"echo '\" + now + \" Ending TestNotebook execution.\");" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/Tests/TestNotebook.ipynb b/Tests/TestNotebook.ipynb index 84d9835..51dce8c 100644 --- a/Tests/TestNotebook.ipynb +++ b/Tests/TestNotebook.ipynb @@ -4,40 +4,38 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ "import time;\n", "import random;\n", "import os;\n", - "from datetime import datetime\n" + "from datetime import datetime" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ - "print(str(datetime.now()))\n", - "\n", - "# sleep times.\n", - "sleepTime1 = 5;#5sec\n", - "sleepTime2 = 300;#5min\n", - "sleepTime3 = 3900;#1h5min, thought for testing the case of a job timeout of 1hr.\n", - "#rand = random.random();\n", - "rand = 0;#force fixed sleepTime \n", - "print(\"rand=\" + str(rand))" + "print(str(datetime.now()))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -50,36 +48,59 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ - "if rand < 0.3:\n", - " time.sleep(sleepTime1);\n", - "elif rand >= 0.3 and rand < 0.6:\n", - " time.sleep(sleepTime2);\n", - "else:\n", - " time.sleep(sleepTime3);" + "# sleep time.\n", + "time.sleep(5);#5sec" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ "now = str(datetime.now());\n", "print(now + \" Now writing HelloWorld.txt file into job directory.\");\n", - "os.system(\"echo '\" + now + \" Now writing HelloWorld.txt file into job directory.'\");\n" + "os.system(\"echo '\" + now + \" Now writing HelloWorld.txt file into job directory.'\");\n", + "\n", + "\n", + "jobsPath = None\n", + "with open('/home/idies/job.path', 'r') as f:\n", + " jobsPath = f.read().rstrip('\\n')\n", + "\n", + "if jobsPath == \"\" or jobsPath == None: \n", + " raise Exception(now + ' Could not read /home/idies/job.path') \n", + " \n", + "print(now + \" Was able to read /home/idies/job.path : \" + jobsPath);\n", + "os.system(\"echo '\" + now + \" Was able to read /home/idies/job.path : \" + jobsPath +\"'\");\n", + "\n", + "\n", + "target = open(jobsPath + \"HelloWorld.txt\", 'w')\n", + "target.truncate()\n", + "target.write(jobsPath)\n", + "target.write(\"\\n\")\n", + "target.close()\n", + "\n", + "print(now + \" Finished writing HelloWorld.txt file into job directory.\");\n", + "os.system(\"echo '\" + now + \" Finished writing HelloWorld.txt file into job directory.'\");" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -89,46 +110,23 @@ " parameters = f.read().rstrip('\\n')\n", "\n", "if parameters is None:\n", - " raise Exception('Could not read /home/idies/jobs.path') \n", - " \n", - "jobsPath = None\n", - "with open('/home/idies/jobs.path') as f:\n", - " jobsPath = f.read().rstrip('\\n')\n", - "\n", - "if jobsPath == \"\" or jobsPath == None: \n", - " raise Exception(now + ' Could not read /home/idies/jobs.path') \n", - " \n", - "print(now + \" Was able to read /home/idies/jobs.path : \" + jobsPath);\n", - "os.system(\"echo '\" + now + \" Was able to read /home/idies/jobs.path : \" + jobsPath +\"'\");\n", - " \n", - "target = open(jobsPath + \"HelloWorld.txt\", 'w')\n", - "target.truncate()\n", - "line1 =\"Hello world!\"\n", - "target.write(line1)\n", - "target.write(\"\\n\")\n", - "target.close()" + " raise Exception('Could not read parameters.txt')\n", + " \n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ "now = str(datetime.now());\n", - "rand2 = random.random();\n", - "#rand2 = 0.1 # force one result type: python exception or not.\n", - "\n", - "if rand2 < 0.5:\n", - " print(now + \" Ending TestNotebook execution with a python exception (dividing by zero).\");\n", - " os.system(\"echo '\" + now + \" Ending TestNotebook execution with a python exception (dividing by zero).'\");\n", - " a = 1/0;\n", - " print(a);\n", - "else:\n", - " print(now + \" Ending TestNotebook execution with no python exception.\");\n", - " os.system(\"echo '\" + now + \" Ending TestNotebook execution with no python exception.'\");" + "print(now + \" Ending TestNotebook execution.\");\n", + "os.system(\"echo '\" + now + \" Ending TestNotebook execution.\");" ] } ], @@ -148,7 +146,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.1" + "version": "3.5.2" } }, "nbformat": 4, diff --git a/Tests/UnitTests.py b/Tests/UnitTests.py index 40640dd..bb8eadd 100644 --- a/Tests/UnitTests.py +++ b/Tests/UnitTests.py @@ -1,5 +1,5 @@ #!/usr/bin/python -from SciServer import Authentication, LoginPortal, Config, CasJobs, SkyServer, SkyQuery, SciDrive, FileService, ComputeJobs +from SciServer import Authentication, LoginPortal, Config, CasJobs, SkyServer, SkyQuery, SciDrive, Files, Jobs import unittest2 as unittest import os; import pandas; @@ -13,6 +13,8 @@ Authentication_loginName = '***'; Authentication_loginPassword = '***' +Authentication_login_sharedWithName = '***' +Authentication_login_sharedWithPassword = '***' #skyserver SkyServer_TestQuery = "select top 1 specobjid, ra, dec from specobj order by specobjid" @@ -39,13 +41,41 @@ SciDrive_FileName = "TestFile.csv" SciDrive_FileContent = "Column1,Column2\n4.5,5.5\n" -FileService_NewDirName = "UnitTestDir" -FileService_NewFileName = "MyNewFile.txt" -FileService_newFileContent = "#ID,Column1,Column2\n1,4.5,5.5\n" -FileService_RemoteFileName = "persistent/" + FileService_NewDirName + "/" + FileService_NewFileName -FileService_RemoteDirName = "persistent/" + FileService_NewDirName -ComputeJobs_ = "" +Files_FileServiceName = "FileServiceJHU" +Files_RootVolumeName1 = "volumes" +Files_UserVolumeName1 = Authentication_loginName + "_UserVolume555" +Files_RootVolumeName2 = "volumes" +Files_UserVolumeName2 = Authentication_loginName + "_UserVolume999" +Files_NewDirectoryName1 = "myNewDirectory555" +Files_NewDirectoryName2 = "myNewDirectory999" +Files_LocalFileName = "MyNewFile.txt" +Files_LocalFileContent = "#ID,Column1,Column2\n1,4.5,5.5" + + + +Jobs_DockerComputeDomainName = 'Small Jobs Batch Domain' +Jobs_FileServiceName = "FileServiceJHU" +Jobs_RootVolumeName = "volumes" +Jobs_UserVolumeName = Authentication_loginName + "_JobsTestVolume" +Jobs_DirectoryName = "JobsTestDirectory" +Jobs_NotebookName = 'TestNotebook.ipynb' +Jobs_NoteBookOutPutFile = 'HelloWorld.txt' +Jobs_ShellCommand = 'pwd' +Jobs_DockerImageName = 'Python (astro)' + +Jobs_UserVolumes = [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}] +Jobs_DataVolumes = [{'name':'SDSS_DAS'}] +Jobs_Parameters = 'param1=1\nparam2=2\nparam3=3' +Jobs_Alias = 'MyNewJob' + +Jobs_SqlQuery = 'select 1;' +Jobs_SqlQueryResult = 'column1\n1\n' +Jobs_RDBComputeDomainName = 'Manga (long)' +Jobs_DatabaseContextName = "manga" +Jobs_RemoteNotebookPath='/home/idies/workspace/' + Jobs_UserVolumeName + '/' + Jobs_DirectoryName + '/' + Jobs_NotebookName +Jobs_QueryResultsFile = 'myQueryResults' + class TestAuthentication(unittest.TestCase): @@ -406,192 +436,295 @@ def test_SkyQuery_listDatasetTables(self): -class TestComputeJobs(unittest.TestCase): +class TestFileService(unittest.TestCase): token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword); - def setUp(self): pass # ******************************************************************************************************* - # ComputeJobs section - - - - - ComputeJobs_NotebookName = "TestNotebook.ipynb" - ComputeJobs_LocalNotebookPath = "./" + ComputeJobs_NotebookName - ComputeJobs_RemoteNotebookPath = "/home/idies/workspace/persistent/" + ComputeJobs_NotebookName - ComputeJobs_JobAlias = "MyJobAlias" - ComputeJobs_Parameters = "param1" - ComputeJobs_ShellCommand = "ls"; - ComputeJobs_ImageNameLike = "python" #must be all lowercase - - def test_ComputeJobs_getComputeDomains(self): - domains = ComputeJobs.getComputeDomains(batch=True, interactive=False) - self.assertTrue(domains[0]["apiEndpoint"] != "" and domains[0]["apiEndpoint"] is not None) - - - - - - def test_ComputeJobs_submitNotebookJob_getJobsList_getJobDescription_waitForJob_getJobStatus_submitShellCommandJob_cancelJob(self): + # Files section + + def test_Files_getFileServices(self): + fileServices = Files.getFileServices(); + self.assertTrue(fileServices.__len__() > 0) + + def test_Files_getFileServicesNames(self): + fileServiceNames = Files.getFileServicesNames(); + self.assertTrue(fileServiceNames.__len__() > 0) + + def test_Files_getFileServicesNames(self): + fileServiceNames = Files.getFileServicesNames(); + self.assertTrue(Files_FileServiceName in fileServiceNames) + + def test_Files_getFileServiceFromName(self): + fileService = Files.getFileServiceFromName(Files_FileServiceName); + self.assertTrue(fileService.get('name') == Files_FileServiceName); + + def test_Files_getRootVolumes(self): + fileService = Files.getFileServiceFromName(Files_FileServiceName); + rootVolumes = Files.getRootVolumes(fileService) + self.assertTrue(rootVolumes.__len__() > 0) + found = False + for rootVolume in rootVolumes: + if rootVolume.get('name') == Files_RootVolumeName1: + found = True + self.assertTrue(found) + + found = False + for rootVolume in rootVolumes: + if rootVolume.get('name') == Files_RootVolumeName2: + found = True + self.assertTrue(found) + + def test_Files_createUserVolume_deleteUserVolume(self): + fileService = Files.getFileServiceFromName(Files_FileServiceName); + Files.createUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName1,quiet=False) + Files.deleteUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName1,quiet=False) + + def test_Files_createDir_upload_dirList_download_download_shareUserVolume(self): try: - isUploaded = FileService.upload(path=ComputeJobs_RemoteNotebookPath,localFilePath=ComputeJobs_LocalNotebookPath) + fileService = Files.getFileServiceFromName(Files_FileServiceName); + os.remove(Files_LocalFileName); + Files.deleteUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName1, quiet=True) + Files.deleteUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName2, quiet=True) except: pass; try: + fileService = Files.getFileServiceFromName(Files_FileServiceName); + Files.createUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName1,quiet=False) + Files.createUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName2,quiet=False) - domains = ComputeJobs.getComputeDomains(batch=True, interactive=False) - self.assertTrue( domains[0]['id'] > 0 ) + Files.createDir(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1); + Files.createDir(fileService, Files_RootVolumeName2, Files_UserVolumeName2, Files_NewDirectoryName2); - domainIndex = None - imageIndex = None - try: - for i in range(len(domains)): - images = domains[i]['images'] - for j in range(len(images)): - if ComputeJobs_ImageNameLike in str(images[j]['name']).lower() or ComputeJobs_ImageNameLike in str(images[j]['description']).lower() : - domainIndex = i - imageIndex = j - raise Exception(""); - except: - pass; + dirList = Files.dirList(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1,level=2) + self.assertTrue(dirList.get('root').get('name') == Files_NewDirectoryName1) - domainIndex = 1 - imageIndex = 0 + Files.upload(fileService, Files_RootVolumeName1, Files_UserVolumeName1, + Files_NewDirectoryName1 + "/" + Files_LocalFileName, + data=Files_LocalFileContent); - if domainIndex is None or imageIndex is None: - raise Exception("Cannot find '" + ComputeJobs_ImageNameLike +"' image in any Compute Domain.") + dirList = Files.dirList(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1,level=2) + self.assertTrue(dirList.get('root').get('files')[0].get('name') == Files_LocalFileName) - domainApiEndpoint = domains[domainIndex]['apiEndpoint'] - dockerImage = domains[domainIndex]['images'][imageIndex]['name'] + Files.download(fileService, Files_RootVolumeName1, Files_UserVolumeName1, + Files_NewDirectoryName1 + "/" + Files_LocalFileName, + localFilePath=Files_LocalFileName); - volumes = []; - if len(domains[domainIndex]['volumes']) > 0: - volumes = [ { "name" : domains[domainIndex]['volumes'][0]['name'] } ] + with open(Files_LocalFileName, 'r') as myfile: + downloadedFileContent = myfile.read() + assert(downloadedFileContent == Files_LocalFileContent) - notebookPath = ComputeJobs_RemoteNotebookPath - parameters = ComputeJobs_Parameters - jobAlias = ComputeJobs_JobAlias + Files.delete(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1 + "/" + Files_LocalFileName) - job = ComputeJobs.submitNotebookJob(domainApiEndpoint=domainApiEndpoint, notebookPath=notebookPath, dockerImage=dockerImage, volumes=volumes, parameters=parameters, jobAlias=jobAlias) - self.assertTrue(job['scriptURI'] == ComputeJobs_RemoteNotebookPath) - self.assertTrue(job['dockerImageName'] == dockerImage) - self.assertTrue(job['dockerComputeEndpoint'] in domainApiEndpoint ) - self.assertTrue(job['submitterDID'] == jobAlias) - self.assertTrue(job['command'] == parameters) - self.assertTrue(job['status'] >= 1) + dirList = Files.dirList(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1,level=2) + self.assertIsNone(dirList.get('root').get('files')) - jobList = ComputeJobs.getJobsList(); - jobId = jobList[0]['id']; + Files.upload(fileService, Files_RootVolumeName1, Files_UserVolumeName1, + Files_NewDirectoryName1 + "/" + Files_LocalFileName, localFilePath=Files_LocalFileName, + quiet=False); - job = ComputeJobs.waitForJob(jobId=jobId, verbose=True, pollTime=2) - self.assertTrue(jobList[0] == job) + Files.move(fileService, Files_RootVolumeName1, Files_UserVolumeName1, + Files_NewDirectoryName1 + "/" + Files_LocalFileName, + fileService, Files_RootVolumeName2, Files_UserVolumeName2, + Files_UserVolumeName2 + "/" + Files_NewDirectoryName2 + "/" + Files_LocalFileName + ); - jobDesc = ComputeJobs.getJobDescription(jobId) - self.assertTrue(jobDesc == job) + Files.shareUserVolume(fileService, Files_RootVolumeName2, Files_UserVolumeName2, + sharedWith=Authentication_login_sharedWithName, type="USER", + userVolumeOwner=Authentication_loginName, + allowedActions=["read"]) - status = ComputeJobs.getJobStatus(jobId=jobId) - self.assertTrue(status == "SUCCESS") + token1 = Authentication.login(Authentication_login_sharedWithName, Authentication_login_sharedWithPassword); - job = ComputeJobs.submitShellCommandJob(domainApiEndpoint=domainApiEndpoint, shellCommand=ComputeJobs_ShellCommand, dockerImage=dockerImage, volumes=volumes, jobAlias=jobAlias) - self.assertTrue(job['dockerImageName'] == dockerImage) - self.assertTrue(job['dockerComputeEndpoint'] in domainApiEndpoint ) - self.assertTrue(job['submitterDID'] == jobAlias) - self.assertTrue(job['command'] == ComputeJobs_ShellCommand) - self.assertTrue(job['status'] >= 1) + string = Files.download(fileService, Files_RootVolumeName2, Files_UserVolumeName2, + Files_NewDirectoryName2 + "/" + Files_LocalFileName, + format="txt", userVolumeOwner=Authentication_loginName); - job = ComputeJobs.submitNotebookJob(domainApiEndpoint=domainApiEndpoint, notebookPath=notebookPath, dockerImage=dockerImage, volumes=volumes, parameters=parameters, jobAlias=jobAlias) - isCanceled = ComputeJobs.cancelJob(job['jobId']) - self.assertTrue(isCanceled) - status = ComputeJobs.getJobStatus(jobId=jobId) - self.assertTrue(status == "CANCELED") - - finally: - FileService.delete(path=ComputeJobs_RemoteNotebookPath) - - - - def test_ComputeJobs_getJobDirectory(self): - pass; + self.assertTrue(string, Files_LocalFileContent) + token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword); + finally: + try: + os.remove(Files_LocalFileName); + Files.deleteUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName1,quiet=True) + Files.deleteUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName2,quiet=True) + except: + pass; -class TestFileService(unittest.TestCase): +class TestJobs(unittest.TestCase): token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword); + def setUp(self): pass # ******************************************************************************************************* - # FileService section + # Jobs section - def test_FileService_createDir_getDirList_delete(self): + # Docker Jobs ################################################################################################ - try: - wasDirDeleted = FileService.delete(FileService_RemoteDirName) - except: - pass - - try: - wasDirCreated = FileService.createDir(FileService_RemoteDirName); - self.assertTrue(wasDirCreated) + def test_Jobs_getDockerComputeDomains(self): + dockerComputeDomains = Jobs.getDockerComputeDomains() + self.assertTrue(dockerComputeDomains.__len__() > 0) + found = False + for dockerComputeDomain in dockerComputeDomains: + if dockerComputeDomain.get('name') == Jobs_DockerComputeDomainName: + found = True + self.assertTrue(found) - dirList = FileService.getDirList(FileService_RemoteDirName) - self.assertTrue(dirList["path"].__contains__(FileService_NewDirName)) - - wasDirDeleted = FileService.delete(FileService_RemoteDirName) - self.assertTrue(wasDirDeleted) - - finally: - try: - wasDirDeleted = FileService.delete(FileService_RemoteDirName) - except: - pass; + def test_Jobs_getDockerComputeDomainsNames(self): + dockerComputeDomainsNames = Jobs.getDockerComputeDomainsNames() + self.assertTrue(Jobs_DockerComputeDomainName in dockerComputeDomainsNames) + def test_Jobs_getDockerComputeDomainFromName(self): + dockerComputeDomain = Jobs.getDockerComputeDomainFromName(Jobs_DockerComputeDomainName) + self.assertTrue(dockerComputeDomain.get('name') in Jobs_DockerComputeDomainName) + def test_Jobs_submitNotebookJob_cancel_waitForJob_getJobStatus_getJobDescription_submitShellCommandJob(self): - def test_FileService_upload_getDirList_download_delete(self): + fileService = Files.getFileServiceFromName(Jobs_FileServiceName); try: + Files.deleteUserVolume(fileService, Jobs_RootVolumeName, Jobs_UserVolumeName) + except: + pass + + Files.createUserVolume(fileService, Jobs_RootVolumeName, Jobs_UserVolumeName) + Files.upload(fileService, Jobs_RootVolumeName, Jobs_UserVolumeName, + Jobs_DirectoryName + "/" + Jobs_NotebookName, + localFilePath=Jobs_NotebookName); + + dockerComputeDomain = Jobs.getDockerComputeDomainFromName(Jobs_DockerComputeDomainName) + + jobId_1 = Jobs.submitNotebookJob( + '/home/idies/workspace/' + Jobs_UserVolumeName + '/' + Jobs_DirectoryName + '/' + Jobs_NotebookName, + dockerComputeDomain, + Jobs_DockerImageName, + Jobs_UserVolumes, Jobs_DataVolumes, + Jobs_Alias) + Jobs.cancelJob(jobId_1) + jobStatus = Jobs.getJobStatus(jobId_1) + self.assertTrue(jobStatus.get('status') == 128) + + jobId_2 = Jobs.submitNotebookJob( + Jobs_RemoteNotebookPath, + dockerComputeDomain, + Jobs_DockerImageName, + Jobs_UserVolumes, Jobs_DataVolumes, + Jobs_Alias) + + jobStatus = Jobs.waitForJob(jobId_2) + self.assertTrue(jobStatus == Jobs.getJobStatus(jobId_2)) + self.assertTrue(jobStatus.get('status') == 32) + + job = Jobs.getJobDescription(jobId_2) + self.assertTrue(job.get('username') == Authentication_loginName) + self.assertTrue(job.get('dockerImageName') == Jobs_DockerImageName) + self.assertTrue(job.get('scriptURI') == Jobs_RemoteNotebookPath) + #self.assertTrue(job.get('submitterDID') == Jobs_Alias) + + jobDirectory = job.get('resultsFolderURI') + relativePath = jobDirectory.split('scratch/')[1] + Jobs_NoteBookOutPutFile; + string = Files.download(fileService, 'scratch', '',relativePath,format="txt", userVolumeOwner=Authentication_loginName); + string.rstrip("\n") + self.assertTrue(string, job.get('resultsFolderURI') ) + + jobs = Jobs.getJobsList(top=2) + found = False + for job in jobs: + if jobId_1 == job.get("id"): + found = True; + self.assertTrue(found) + + found = False + for job in jobs: + if jobId_2 == job.get("id"): + found = True; + self.assertTrue(found) + + + jobId = Jobs.submitShellCommandJob(Jobs_ShellCommand, + dockerComputeDomain, + Jobs_DockerImageName, + Jobs_UserVolumes, Jobs_DataVolumes, + Jobs_Alias) + + jobStatus = Jobs.waitForJob(jobId) + self.assertTrue(jobStatus == Jobs.getJobStatus(jobId)) + self.assertTrue(jobStatus.get('status') == 32) + + job = Jobs.getJobDescription(jobId) + self.assertTrue(job.get('username') == Authentication_loginName) + self.assertTrue(job.get('dockerImageName') == Jobs_DockerImageName) + self.assertTrue(job.get('command') == Jobs_ShellCommand) + self.assertTrue(job.get('submitterDID') == Jobs_Alias) + + jobDirectory = job.get('resultsFolderURI') + relativePath = jobDirectory.split('scratch/')[1] + "command.txt"; + string = Files.download(fileService, 'scratch', '',relativePath,format="txt", userVolumeOwner=Authentication_loginName); + string.rstrip("\n") + self.assertTrue(string, job.get('resultsFolderURI') ) + + Files.deleteUserVolume(fileService, Jobs_RootVolumeName, Jobs_UserVolumeName) + + + + # RDB Jobs ################################################################################################ + + def test_Jobs_getRDBComputeDomains(self): + rdbComputeDomains = Jobs.getRDBComputeDomains() + self.assertTrue(rdbComputeDomains.__len__() > 0) + found = False + for rdbComputeDomain in rdbComputeDomains: + if rdbComputeDomain.get('name') == Jobs_RDBComputeDomainName: + found = True + self.assertTrue(found) + + def test_Jobs_getRDBComputeDomainsNames(self): + rdbComputeDomainsNames = Jobs.getRDBComputeDomainsNames() + self.assertTrue(Jobs_RDBComputeDomainName in rdbComputeDomainsNames) + + def test_Jobs_getRDBComputeDomainFromName(self): + rdbComputeDomain = Jobs.getRDBComputeDomainFromName(Jobs_RDBComputeDomainName) + self.assertTrue(rdbComputeDomain.get('name') in Jobs_RDBComputeDomainName) + + + def test_Jobs_submitRDBJob(self): + + rdbComputeDomain = Jobs.getRDBComputeDomainFromName(Jobs_RDBComputeDomainName) + + jobId = Jobs.submitRDBQueryJob(Jobs_SqlQuery, rdbComputeDomain, Jobs_DatabaseContextName, Jobs_QueryResultsFile,Jobs_Alias) + + jobStatus = Jobs.waitForJob(jobId) + self.assertTrue(jobStatus == Jobs.getJobStatus(jobId)) + self.assertTrue(jobStatus.get('status') == 32) + + job = Jobs.getJobDescription(jobId) + self.assertTrue(job.get('username') == Authentication_loginName) + self.assertTrue(job.get('rdbDomainName') == Jobs_RDBComputeDomainName) + self.assertTrue(job.get('databaseContextName') == Jobs_DatabaseContextName) + self.assertTrue(job.get('inputSql') == Jobs_SqlQuery) + self.assertTrue(job.get('submitterDID') == Jobs_Alias) - if (sys.version_info > (3, 0)): #python3 - file = open(FileService_NewFileName, "w") - else: #python2 - file = open(FileService_NewFileName, "wb") + fileService = Files.getFileServiceFromName(Jobs_FileServiceName); + jobDirectory = job.get('resultsFolderURI') + relativePath = jobDirectory.split('scratch/')[1] + Jobs_QueryResultsFile + '.csv'; + string = Files.download(fileService, 'scratch', '',relativePath,format="txt", userVolumeOwner=Authentication_loginName); + string.rstrip("\n") + self.assertTrue(string, Jobs_SqlQueryResult) - file.write(FileService_newFileContent) - file.close() - isUploaded = FileService.upload(path=FileService_RemoteFileName, localFilePath=FileService_NewFileName) - dirList = FileService.getDirList(FileService_RemoteDirName) - stringio = FileService.download(path=FileService_RemoteFileName, format="StringIO") - fileContent = stringio.read() - isDeleted = FileService.delete(FileService_RemoteFileName) - self.assertTrue(isUploaded) - self.assertTrue( dirList.__contains__(FileService_NewFileName)) - self.assertEqual(fileContent, FileService_newFileContent) - self.assertTrue(isDeleted) - - isUploaded = FileService.upload(path=FileService_RemoteFileName, data=FileService_newFileContent) - dirList = FileService.getDirList(FileService_RemoteDirName) - fileContent = FileService.download(path=FileService_RemoteFileName, format="text") - isDeleted = FileService.delete(FileService_RemoteFileName) - self.assertTrue(isUploaded) - self.assertTrue( dirList.__contains__(FileService_NewFileName)) - self.assertEqual(fileContent, FileService_newFileContent) - self.assertTrue(isDeleted) + def test_Jobs_getJobDirectory(self): + #TBD + pass; - finally: - try: - os.remove(FileService_newFileContent) - except: - pass; @@ -600,12 +733,12 @@ def test_FileService_upload_getDirList_download_delete(self): unittest.TestLoader.sortTestMethodsUsing = lambda x, y: cmp(x,y); testLoader = unittest.TestLoader() testLoader.sortTestMethodsUsing = lambda x, y: 0; - #suite = testLoader.loadTestsFromTestCase(TestAuthentication); unittest.TextTestRunner(verbosity=2).run(suite) - #suite = testLoader.loadTestsFromTestCase(TestLoginPortal); unittest.TextTestRunner(verbosity=2).run(suite) - #suite = testLoader.loadTestsFromTestCase(TestCasJobs); unittest.TextTestRunner(verbosity=2).run(suite) - #suite = testLoader.loadTestsFromTestCase(TestSkyServer); unittest.TextTestRunner(verbosity=2).run(suite) - #suite = testLoader.loadTestsFromTestCase(TestSciDrive); unittest.TextTestRunner(verbosity=2).run(suite) - #suite = testLoader.loadTestsFromTestCase(TestSkyQuery); unittest.TextTestRunner(verbosity=2).run(suite) - #suite = testLoader.loadTestsFromTestCase(TestComputeJobs); unittest.TextTestRunner(verbosity=2).run(suite) - #suite = testLoader.loadTestsFromTestCase(TestFileService); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestAuthentication); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestLoginPortal); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestCasJobs); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestSkyServer); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestSciDrive); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestSkyQuery); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestFileService); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestJobs); unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/py2/SciServer/Authentication.py b/py2/SciServer/Authentication.py index a8797c2..73e7ca0 100644 --- a/py2/SciServer/Authentication.py +++ b/py2/SciServer/Authentication.py @@ -17,6 +17,7 @@ class KeystoneUser: """ id = None userName = None + token = None class Token: """ @@ -26,6 +27,7 @@ class Token: token = Token(); +keystoneUser = KeystoneUser(); def getKeystoneUserWithToken(token): """ @@ -38,29 +40,37 @@ def getKeystoneUserWithToken(token): .. seealso:: Authentication.getToken, Authentication.login, Authentication.setToken. """ - taskName = "" - if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.Authentication.getKeystoneUserWithToken" + + if keystoneUser.token == token and keystoneUser.token is not None: + return keystoneUser; + else: - taskName = "SciScript-Python.Authentication.getKeystoneUserWithToken" - - loginURL = Config.AuthenticationURL - if ~loginURL.endswith("/"): - loginURL = loginURL + "/" + taskName = "" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Authentication.getKeystoneUserWithToken" + else: + taskName = "SciScript-Python.Authentication.getKeystoneUserWithToken" + + loginURL = Config.AuthenticationURL + if ~loginURL.endswith("/"): + loginURL = loginURL + "/" - loginURL = loginURL + token + "?TaskName=" + taskName; + loginURL = loginURL + token + "?TaskName=" + taskName; - getResponse = requests.get(loginURL) - if getResponse.status_code != 200: - raise Exception("Error when getting the keystone user with token " + str(token) +".\nHttp Response from the Authentication API returned status code " + str(getResponse.status_code) + ":\n" + getResponse.content.decode()); + getResponse = requests.get(loginURL) + if getResponse.status_code != 200: + raise Exception("Error when getting the keystone user with token " + str(token) +".\nHttp Response from the Authentication API returned status code " + str(getResponse.status_code) + ":\n" + getResponse.content.decode()); - responseJson = json.loads((getResponse.content.decode())) + responseJson = json.loads((getResponse.content.decode())) - ksu = KeystoneUser() - ksu.userName = responseJson["token"]["user"]["name"] - ksu.id = responseJson["token"]["user"]["id"] + ksu = KeystoneUser() + ksu.userName = responseJson["token"]["user"]["name"] + ksu.id = responseJson["token"]["user"]["id"] + keystoneUser.token = token; + keystoneUser.userName = ksu.userName + keystoneUser.id = ksu.id - return ksu + return ksu def login(UserName=None, Password=None): @@ -80,9 +90,9 @@ def login(UserName=None, Password=None): """ taskName = "" if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.Authentication.login" + taskName = "Compute.SciScript-Python.Authentication.Login" else: - taskName = "SciScript-Python.Authentication.login" + taskName = "SciScript-Python.Authentication.Login" loginURL = Config.AuthenticationURL + "?TaskName=" + taskName diff --git a/py2/SciServer/Config.py b/py2/SciServer/Config.py index 4a23a0f..f922b4a 100644 --- a/py2/SciServer/Config.py +++ b/py2/SciServer/Config.py @@ -22,7 +22,7 @@ """ # URLs for accessing SciServer web services (API endpoints) CasJobsRESTUri = "https://skyserver.sdss.org/CasJobs/RestApi" -AuthenticationURL = "https://portal.sciserver.org/login-portal/keystone/v3/tokens" +AuthenticationURL = "http://scitest12.pha.jhu.edu/login-portal/keystone/v3/tokens" SciDriveHost = "https://www.scidrive.org" SkyQueryUrl = "http://voservices.net/skyquery/Api/V1" SkyServerWSurl = "https://skyserver.sdss.org" diff --git a/py2/SciServer/Files.py b/py2/SciServer/Files.py index 42cea58..6f4c99c 100644 --- a/py2/SciServer/Files.py +++ b/py2/SciServer/Files.py @@ -5,13 +5,15 @@ import json from io import StringIO from io import BytesIO +import warnings +import os - -def getFileServices(): +def getFileServices(verbose=True): """ - Gets the definitions of the file services that a user is able to access. - :return: list of dictionaries, where each dictionary is is the descriptions of a FileService that a user is able to access. + Gets the definitions of file services that a user is able to access. A FileService represents a file system that contains root volumes accessible to the user for public/private data storage. Within each rootVolume, users can create sharable userVolumes for storing files. + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: list of dictionaries, where each dictionary represents the description of a FileService that the user is able to access. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. :example: fileServices = Files.getFileServices(); .. seealso:: Files.getFileServiceFromName @@ -30,20 +32,69 @@ def getFileServices(): res = requests.get(url, headers=headers) if res.status_code >= 200 and res.status_code < 300: - return json.loads(res.content.decode()); + fileServices = []; + fileServicesAPIs = json.loads(res.content.decode()) + for fileServicesAPI in fileServicesAPIs: + url = fileServicesAPI.get("apiEndpoint") + name = fileServicesAPI.get("name") + url = url + "api/volumes/?TaskName="+taskName; + try: + res = requests.get(url, headers=headers) + except: + if verbose: + warnings.warn("Error when getting definition of FileService named '" + name + "' with API URL '" + fileServicesAPI.get("apiEndpoint") + "'. This FileService might be not available", Warning, stacklevel=2) + + if res.status_code >= 200 and res.status_code < 300: + fileServices.append(json.loads(res.content.decode())); + else: + if verbose: + warnings.warn("Error when getting definition of FileService named '" + name + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode(),Warning, stacklevel=2) + return fileServices; else: raise Exception("Error when getting the list of FileServices.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") -def getFileServiceFromName(fileServiceName, fileServices=None): + +def getFileServicesNames(fileServices=None, verbose=True): + """ + Returns the names of the fileServices available to the user. + :param fileServices: a list of FileService objects (dictionaries), as returned by Jobs.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: an array of strings, each being the name of a file service available to the user. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServiceNames = Files.getFileServicesNames(); + .. seealso:: Files.getFileServices + """ + + if fileServices is None: + fileServices = getFileServices(verbose); + + fileServiceNames = []; + if fileServices.__len__() > 0: + for fileService in fileServices: + fileServiceNames.append(fileService.get('name')) + + return fileServiceNames; + else: + #raise Exception("There are no FileServices available for the user."); + if verbose: + warnings.warn("There are no FileServices available for the user.", Warning, stacklevel=2) + return fileServiceNames + + + + + +def getFileServiceFromName(fileServiceName, fileServices=None, verbose=True): """ Returns a FileService object, given its registered name. :param fileServiceName: name of the FileService, as shown within the results of Jobs.getFileServices() :param fileServices: a list of FileService objects (dictionaries), as returned by Jobs.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. - :return: a FileService object (dictionary) that defines a FileService. A list of these kind of objects available to the user is returned by the function Jobs.getFileServices(). - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return a FileService object (dictionary) that defines a FileService. A list of these kind of objects available to the user is returned by the function Jobs.getFileServices(). If no fileService can be found, then returns None. + :raises Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. :example: fileService = Files.getFileServiceFromName('FileServiceAtJHU'); .. seealso:: Files.getFileServices """ @@ -52,189 +103,204 @@ def getFileServiceFromName(fileServiceName, fileServices=None): raise Exception("fileServiceName is not defined.") else: if fileServices is None: - fileServices = getFileServices(); + fileServices = getFileServices(verbose); if fileServices.__len__() > 0: for fileService in fileServices: if fileServiceName == fileService.get('name'): return fileService; + + #raise Exception("FileService of name '" + fileServiceName + "' is not available or does not exist."); + if verbose: + warnings.warn("FileService of name '" + fileServiceName + "' is not available or does not exist.", Warning, stacklevel=2) + return None else: - raise Exception("There are no FileServices available for the user."); + #raise Exception("There are no FileServices available for the user."); + if verbose: + warnings.warn("There are no FileServices available for the user.", Warning, stacklevel=2) + return None - raise Exception("FileService of name '" + fileServiceName + "' is not available or does not exist."); -def __getFileServiceAPIUrl(fileService=None): +def __getFileServiceAPIUrl(fileService): """ Gets the API endpoint URL of a FileService. - :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made, and the returned URL is that of the first FileService found. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :return: API endpoint URL of the FileService (string). - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. - :example: fileServiceUrl = Files.__getFileServiceAPIUrl(); + :example: fileServiceAPIUrl = Files.__getFileServiceAPIUrl(); .. seealso:: Files.getFileServiceFromName """ - if fileService is None: - fileServices = getFileServices(); - if fileServices.__len__() > 0: - return fileServices[0].get("apiEndpoint"); - else: - raise Exception("There are no FileServices available for the user."); + + url = None; + if type(fileService) == type(""): + fileServices = getFileServices(False); + _fileService = getFileServiceFromName(fileService, fileServices, verbose=False); + url = _fileService.get("apiEndpoint"); else: - return fileService.get("apiEndpoint"); + url = fileService.get("apiEndpoint"); + + if not url.endswith("/"): + url = url + "/" + return url -def createDir(path, volume, ownerName=None, fileService=None): + +def getRootVolumes(fileService): """ - Create a directory. - :param path: path (string) to the directory, relative to the volume level. E.g: path="/newDirectory" - :param volume: volume (string) containing the path. - :param ownerName: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. - :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :return: True if the directory creation was successful. + Gets the definitions the RootVolumes available in a particular FileService, and of the file services that a user is able to access. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :return: list of dictionaries, where each dictionary contains the definition of a FileService that a user is able to access. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServices = Files.getFileServices(); rootVolumes = getRootVolumes(fileServices[0]) + .. seealso:: Files.getFileServiceFromName + """ + return fileService.get("rootVolumes"); + + +def createUserVolume(fileService, rootVolume, userVolume, userVolumeOwner=None, quiet=True): + """ + Create a user volume. + :param fileService: name of fileService (string), or object (dictionary) that defines the file service that contains the root volume. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolume: name of root volume (string) that contains the user volume to be created. A list of user volumes is given by getRootVolumes(fileService) + :param userVolume: name of user volume (string) to be created, one level below the root volume level. + :param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume. + :param quiet: if set to False, will throw an error if the User Volume already exists. If True, won't throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); Files.createDir("myNewDir","persistent","myUserName", fileServices[0]); + :example: fileServices = Files.getFileServices(); Files.createUserVolume(fileServices[0], "volumes","newUserVolume"); .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList """ token = Authentication.getToken() if token is not None and token != "": if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.Files.createDir" + taskName = "Compute.SciScript-Python.Files.createUserVolume" else: - taskName = "SciScript-Python.Files.createDir" - - if ~path.startswith("/"): - path = "/" + path; + taskName = "SciScript-Python.Files.createUserVolume" - if ownerName is None: - ownerName = Authentication.getKeystoneUserWithToken().userName; + if userVolumeOwner is None: + userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - url = Config.FileServiceURL + "/folder/" + volume + "/" + ownerName + path + "?TaskName="+taskName; + url = __getFileServiceAPIUrl(fileService) + "api/volume/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?quiet="+str(quiet) + "&TaskName="+taskName; headers = {'X-Auth-Token': token} res = requests.put(url, headers=headers) if res.status_code >= 200 and res.status_code < 300: - return True; + pass; else: - raise Exception("Error when creating directory '" + path + "' in volume '" + volume + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when creating user volume '" + userVolume + "' in root volume '" + rootVolume + "' in file service named '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") -def dirList(path, volume, ownerName=None, fileService=None, level=1, options=''): +def deleteUserVolume(fileService, rootVolume, userVolume, userVolumeOwner=None, quiet=True): """ - Lists the contents of a directory. - :param path: path (string) to the directory, relative to the volume level. E.g: path="/newDirectory" - :param volume: volume (string) containing the path. - :param ownerName: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. - :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param level: amount (int) of listed directory levels that are below or at the same level of the path. - :param options: string of file filtering options. - :return: dictionary containing the directory listing. + Delete a user volume. + :param fileService: name of fileService (string), or object (dictionary) that defines the file service that contains the root volume. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolume: root volume name (string) that contains the user volume to be deleted. A list of user volumes is given by getRootVolumes(fileService) + :param userVolume: user volume name (string) to be deleted. + :param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume. + :param quiet: If set to False, it will throw an error if user volume does not exists. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); dirs = Files.dirList("/","persistent","myUserName", fileServices[0], 2); - .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.createDir + :example: fileServices = Files.getFileServices(); Files.deleteUserVolume("volumes","newUserVolume",fileServices[0]); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList """ token = Authentication.getToken() if token is not None and token != "": if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.Files.dirList" + taskName = "Compute.SciScript-Python.Files.createUserVolume" else: - taskName = "SciScript-Python.Files.dirList" - - if ownerName is None: - ownerName = Authentication.getKeystoneUserWithToken().userName; + taskName = "SciScript-Python.Files.createUserVolume" - if ~path.startswith("/"): - path = "/" + path; + if userVolumeOwner is None: + userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - url = Config.FileServiceURL + "/jsontree/" + volume + "/" + ownerName + path + "?options=" + str(options) + "&level=" + str(level) + "&TaskName=" + taskName; + url = __getFileServiceAPIUrl(fileService) + "api/volume/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?quiet="+str(quiet)+"&TaskName="+taskName; headers = {'X-Auth-Token': token} - res = requests.get(url, headers=headers) + res = requests.delete(url, headers=headers) if res.status_code >= 200 and res.status_code < 300: - return res.content.decode(); + pass else: - raise Exception("Error when creating new directory " + path + ".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when deleting user volume '" + userVolume + "' in root volume '" + rootVolume + "' in file service named '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") - - -def delete(path, volume, ownerName=None, fileService=None): +def createDir(fileService, rootVolume, userVolume, relativePath, userVolumeOwner=None, quiet=True): """ - Deletes a directory or file in the File System. - :param path: path (string) to the directory or file to be deleted, relative to the volume level. E.g: path="/directory/file.txt" - :param volume: volume (string) containing the path. - :param ownerName: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. - :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :return: True if the deletion was successful. + Create a directory. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. + :param userVolume: user volume name (string) that contains the relativePath. + :param relativePath: path (string) to the directory, relative to the userVolume level. E.g: path="/newDirectory" + :param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume. + :param quiet: If set to False, it will throw an error if the directory already exists. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]); - .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList + :example: fileServices = Files.getFileServices(); Files.createDir(fileServices[0], "myRootVolume","myUserVolume", "myNewDir"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList """ token = Authentication.getToken() if token is not None and token != "": if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.Files.delete" + taskName = "Compute.SciScript-Python.Files.createDir" else: - taskName = "SciScript-Python.Files.delete" + taskName = "SciScript-Python.Files.createDir" + + if not relativePath.startswith("/"): + relativePath = "/" + relativePath; - if ~path.startswith("/"): - path = "/" + path; + if userVolumeOwner is None: + userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - url = Config.FileServiceURL + "/data" + path + "?TaskName="+taskName + url = __getFileServiceAPIUrl(fileService) + "api/folder/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName; headers = {'X-Auth-Token': token} - res = requests.delete(url, headers=headers) + res = requests.put(url, headers=headers) if res.status_code >= 200 and res.status_code < 300: - return True; + pass; else: - raise Exception("Error when deleting " + path + ".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when creating directory '" + relativePath + "' in user volume '" + userVolume + "' in root volume '" + rootVolume + "' in file service named '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") - -def upload(path, volume, ownerName=None, fileService=None, data=None, localFilePath=None): +def upload(fileService, rootVolume, userVolume, relativePath, data="", localFilePath=None, userVolumeOwner=None, quiet=True): """ Uploads data or a local file into a path defined in the file system. - :param path: destination path (string) of the file to be uploaded, relative to the volume level. E.g: path="/newDirectory/myNewFile.txt" - :param volume: volume (string) containing the path. - :param ownerName: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. - :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param data: string containing data to be uploaded. - :param localFilePath: path to a local file to be uploaded (string). - :return: True is upload was successful. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. + :param userVolume: user volume name (string) that contains the relativePath. + :param relativePath: destination path (string) of the file to be uploaded, relative to the user volume level. E.g: path="/newDirectory/myNewFile.txt". + :param data: string containing data to be uploaded, in case localFilePath is not set. + :param localFilePath: path to a local file to be uploaded (string), + :param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume. + :param quiet: If set to False, it will throw an error if the file already exists. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); isUploaded = Files.upload("/myUploadedFile.txt","persistent","myUserName", fileServices[0], localFilePath="/myFile.txt"); + :example: fileServices = Files.getFileServices(); Files.upload(fileServices[0], "myRootVolume", "myUserVolume", "/myUploadedFile.txt", None, None, localFilePath="/myFile.txt"); .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.download, Files.dirList """ token = Authentication.getToken() if token is not None and token != "": if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.Files.upload" + taskName = "Compute.SciScript-Python.Files.UploadFile" else: - taskName = "SciScript-Python.Files.upload" + taskName = "SciScript-Python.Files.UploadFile" - if ~path.startswith("/"): - path = "/" + path; + if not relativePath.startswith("/"): + relativePath = "/" + relativePath; - fileName = path.split("/")[-1] - dirPath = path.replace(fileName, "") + if userVolumeOwner is None: + userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - createDir(dirPath); - - url = Config.FileServiceURL + "/data" + path + "?TaskName="+taskName + url = __getFileServiceAPIUrl(fileService) + "api/file/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName headers = {'X-Auth-Token': token} @@ -245,25 +311,27 @@ def upload(path, volume, ownerName=None, fileService=None, data=None, localFileP if data != None: res = requests.put(url, data=data, headers=headers, stream=True) else: - raise Exception("Error when uploading to " + path + ". No local file or data specified for uploading."); + raise Exception("Error: No local file or data specified for uploading."); if res.status_code >= 200 and res.status_code < 300: - return True; + pass; else: - raise Exception("Error when uploading to " + path +".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when uploading file '" + relativePath + "' in user volume '" + userVolume + "' in root volume '" + rootVolume + "' in file service named '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") -def download(path, volume, ownerName=None, fileService=None, localFilePath="", format="text"): +def download(fileService, rootVolume, userVolume, relativePath, localFilePath=None, format="txt", userVolumeOwner=None, quiet=True): """ Downloads a file from the remote file system into the local file system, or returns the file content as an object in several formats. - :param path: remote path (string) of the file to be downloaded, relative to the volume level. E.g: path="/newDirectory/myNewFile.txt" - :param volume: volume (string) containing the path. - :param ownerName: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. - :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param localFilePath: local path of the file to be downloaded. If 'localFilePath' is defined, then the 'format' parameter is not used. - :param format: type of the returned object. Can be "StringIO" (io.StringIO object containing readable text), "BytesIO" (io.BytesIO object containing readable binary data), "response" ( the HTTP response as an object of class requests.Response) or "text" (a text string). If the parameter 'localFilePath' is defined, then the 'format' parameter is not used and the file is downloaded to the local file system instead. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. + :param userVolume: user volume name (string) that contains the relativePath. + :param relativePath: remote path (string) of the file to be downloaded, relative to the user volume level. E.g: path="/newDirectory/myNewFile.txt" + :param localFilePath: local destination path of the file to be downloaded. If set to None, then an object of format 'format' will be returned. + :param format: name (string) of the returned object's type (if localFilePath is not defined). This parameter can be "StringIO" (io.StringIO object containing readable text), "BytesIO" (io.BytesIO object containing readable binary data), "response" ( the HTTP response as an object of class requests.Response) or "txt" (a text string). If the parameter 'localFilePath' is defined, then the 'format' parameter is not used and the file is downloaded to the local file system instead. + :param userVolumeOwner: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. + :param quiet: If set to False, it will throw an error if the file already exists. If set to True. it will not throw an error. :return: If the 'localFilePath' parameter is defined, then it will return True when the file is downloaded successfully in the local file system. If the 'localFilePath' is not defined, then the type of the returned object depends on the value of the 'format' parameter (either io.StringIO, io.BytesIO, requests.Response or string). :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); isDownloaded = Files.upload("/myUploadedFile.txt","persistent","myUserName", fileServices[0], localFilePath="/myDownloadedFile.txt"); @@ -272,21 +340,29 @@ def download(path, volume, ownerName=None, fileService=None, localFilePath="", f token = Authentication.getToken() if token is not None and token != "": + + if localFilePath is not None: + if os.path.isfile(localFilePath) and not quiet: + raise Exception("Error when downloading from remote File Syatem the file " + str(relativePath) + ".\nLocal file '" + localFilePath + "' already exists."); + if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.Files.download" + taskName = "Compute.SciScript-Python.Files.DownloadFile" else: - taskName = "SciScript-Python.Files.download" + taskName = "SciScript-Python.Files.DownloadFile" + + if not relativePath.startswith("/"): + relativePath = "/" + relativePath; - if ~path.startswith("/"): - path = "/" + path; + if userVolumeOwner is None: + userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - url = Config.FileServiceURL + "/data" + path + "?TaskName="+taskName; + url = __getFileServiceAPIUrl(fileService) + "api/file/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?TaskName=" + taskName; headers = {'X-Auth-Token': token} res = requests.get(url, stream=True, headers=headers) if res.status_code < 200 or res.status_code >= 300: - raise Exception("Error when downloading from remote File Syatem the file " + str(path) + ".\nHttp Response from FileSystem API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when downloading from remote File Syatem the file " + str(relativePath) + ".\nHttp Response from FileSystem API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); if localFilePath is not None and localFilePath != "": @@ -301,16 +377,199 @@ def download(path, volume, ownerName=None, fileService=None, localFilePath="", f if format is not None and format != "": if format == "StringIO": return StringIO(res.content.decode()) - if format == "text": + if format == "txt": return res.content.decode() elif format == "BytesIO": return BytesIO(res.content) elif format == "response": return res; else: - raise Exception("Unknown format '" + format + "' when trying to download from remote File System the file " + str(path) + ".\n"); + raise Exception("Unknown format '" + format + "' when trying to download from remote File System the file " + str(relativePath) + ".\n"); else: raise Exception("Wrong format parameter value\n"); else: raise Exception("User token is not defined. First log into SciServer.") + + + +def dirList(fileService, rootVolume, userVolume, relativePath, level=1, options='', userVolumeOwner=None): + """ + Lists the contents of a directory. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. + :param userVolume: user volume name (string) that contains the relativePath. + :param relativePath: remote path (string) of the directory, relative to the user volume level. E.g: path="/newDirectory/myNewFile.txt" + :param level: amount (int) of listed directory levels that are below or at the same level to that of the relativePath. + :param options: string of file filtering options. + :param userVolumeOwner: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. + :return: dictionary containing the directory listing. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); dirs = Files.dirList("/","persistent","myUserName", fileServices[0], 2); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.createDir + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.dirList" + else: + taskName = "SciScript-Python.Files.dirList" + + if userVolumeOwner is None: + userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + + if not relativePath.startswith("/"): + path = "/" + relativePath; + + url = __getFileServiceAPIUrl(fileService) + "api/jsontree/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + path + "?options=" + options + "&level=" + str(level) + "&TaskName=" + taskName; + + headers = {'X-Auth-Token': token} + res = requests.get(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + return json.loads(res.content.decode()); + else: + raise Exception("Error when listing directory.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def move(fileService, rootVolume, userVolume, relativePath, destinationFileService, destinationRootVolume, destinationUserVolume, destinationRelativePath, replaceExisting=True, doCopy=True, userVolumeOwner=None, destinationUserVolumeOwner=None): + """ + Moves or copies a file or folder. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileService object. + :param userVolume: user volume name (string) that contains the relativePath. + :param relativePath: remote path (string) of the file to be moved/copied, relative to the user volume level. E.g: path="/newDirectory/myNewFile.txt" + :param destinationFileService: name of fileService (string), or object (dictionary) that defines a destination file service (where the file is moved/copied into). A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param destinationRootVolume: root volume name (string) that contains the userVolume where the file is moved/copied into. A list of root volumes is contained by the fileService object. + :param destinationUserVolume: user volume name (string) that contains the relativePath where the file is moved/copied into. + :param destinationRelativePath: remote path (string) of the destination file, relative to the user volume level. E.g: path="/newDirectory2/myNewFile.txt" + :param replaceExisting: If set to False, it will throw an error if the file already exists, If set to True, it will not throw and eeror in that case. + :param doCopy: if set to True, then it will copy the file or folder. If set to False, then the file or folder will be moved. + :param userVolumeOwner: name (string) of owner of the user volume. Can be left undefined if requester is the owner of the volume. + :param destinationUserVolumeOwner: name (string) of owner of the destination user volume. Can be left undefined if requester is the owner of it. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); isDownloaded = Files.upload("/myUploadedFile.txt","persistent","myUserName", fileServices[0], localFilePath="/myDownloadedFile.txt"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.upload, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.DownloadFile" + else: + taskName = "SciScript-Python.Files.DownloadFile" + + if __getFileServiceAPIUrl(fileService) != __getFileServiceAPIUrl(fileService): + raise Exception("Use of multiple file services is not supported for now."); + + if not relativePath.startswith("/"): + relativePath = "/" + relativePath; + + if not destinationRelativePath.startswith("/"): + destinationRelativePath = "/" + destinationRelativePath; + + if userVolumeOwner is None: + userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + + if destinationUserVolumeOwner is None: + destinationuserVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + + url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName; + headers = {'X-Auth-Token': token} + jsonDict = {'destinationPath': destinationRelativePath, 'destinationRootFolder': destinationRootVolume, 'destinationUserVolume':destinationUserVolume, 'destinationOwner': destinationuserVolumeOwner}; + data = json.dumps(jsonDict).encode() + res = requests.put(url, stream=True, headers=headers, data=data) + + if res.status_code < 200 or res.status_code >= 300: + raise Exception("Error when downloading from remote File System the file " + str(relativePath) + ".\nHttp Response from FileSystem API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + + else: + raise Exception("User token is not defined. First log into SciServer.") + + + +def delete(fileService, rootVolume, userVolume, relativePath, userVolumeOwner=None, quiet=True): + """ + Deletes a directory or file in the File System. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. + :param userVolume: user volume name (string) that contains the relativePath. + :param relativePath: remote path (string) of the directory or file to be deleted, relative to the user volume level. E.g: path="/newDirectory/myNewFile.txt" + :param userVolumeOwner: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. + :param quiet: If set to False, it will throw an error if the file does not exist. If set to True. it will not throw an error. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.delete" + else: + taskName = "SciScript-Python.Files.delete" + + if not relativePath.startswith("/"): + relativePath = "/" + relativePath; + + if userVolumeOwner is None: + userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + + url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName + + headers = {'X-Auth-Token': token} + res = requests.delete(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + pass; + else: + raise Exception("Error when deleting " + relativePath + ".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + + +def shareUserVolume(fileService, rootVolume, userVolume, sharedWith, allowedActions, type="USER", userVolumeOwner=None): + """ + Deletes a directory or file in the File System. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. + :param userVolume: user volume name (string) that contains the relativePath. + :param sharedWith: name (string) of user or group that the userVolume is shared with. + :param allowedActions: array of strings defining actions the user or group is allowed to do with respect to the shared user volume. E.g.: ["read","write","grant","delete"]. The "grant" action means that the user or group can also share the user volume with another user or group. The "delete" action meand ability to delete the user volume (use with care). + :param type: type (string) of the entity defined by the "sharedWith" parameter. Can be set to "USER" or "GROUP". + :param userVolumeOwner: name (string) of owner of the user volume. Can be left undefined if requester is the owner of the volume. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.shareUserVolume(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]); + .. seealso:: Files.getFileServices(), Files.getFilrmsieServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.ShareUserVolume" + else: + taskName = "SciScript-Python.Files.ShareUserVolume" + + if userVolumeOwner is None: + userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + + + data = [{'name': sharedWith, 'type':type, 'allowedActions': allowedActions }] + body = json.dumps(data).encode() + + url = __getFileServiceAPIUrl(fileService) + "api/share/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?TaskName="+taskName + + headers = {'X-Auth-Token': token} + res = requests.patch(url, headers=headers, data=body) + + if res.status_code >= 200 and res.status_code < 300: + pass; + else: + raise Exception("Error when sharing userVolume '" + userVolume + "' in rootVolume '" + rootVolume + "' with '" + sharedWith + "' in file service named '" + fileService.get( + 'name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py index f5a4cba..935b01f 100644 --- a/py2/SciServer/Jobs.py +++ b/py2/SciServer/Jobs.py @@ -12,7 +12,11 @@ def getJobDirectory(): """ - Gets the root directory path where the job is being executed. Since its value depends on the value of jobId, the path will be defined only after the job is submitted. + Gets the root directory path where the job is being executed. Since its value depends on the value of jobId, the path will be defined only after the job is submitted. It can be called within Jupyter notebooks or shell commands running as jobs. + :return: a string defining the path to the job directory. + :raises: Throws an exception if job directory cannot be found. + :example: jobDirectory = getJobDirectory(); + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getRDBComputeDomains, Jobs.cancelJob """ computeJobDirectoryFile = Config.ComputeJobDirectoryFile if os.path.isfile(computeJobDirectoryFile): @@ -54,6 +58,51 @@ def getDockerComputeDomains(): raise Exception("User token is not defined. First log into SciServer.") +def getDockerComputeDomainsNames(dockerComputeDomains=None): + """ + Returns the names of the docker compute domains available to the user. + :param dockerComputeDomains: a list of dockerComputeDomain objects (dictionaries), as returned by Jobs.getDockerComputeDomains(). If not set, then an extra internal call to Jobs.getDockerComputeDomains() is made. + :return: an array of strings, each being the name of a docker compute domain available to the user. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: dockerComputeDomainsNames = Files.getDockerComputeDomainsNames(); + .. seealso:: Files.getDockerComputeDomains + """ + if dockerComputeDomains is None: + dockerComputeDomains = getDockerComputeDomains(); + + dockerComputeDomainsNames = []; + for dockerComputeDomain in dockerComputeDomains: + dockerComputeDomainsNames.append(dockerComputeDomain.get('name')) + + return dockerComputeDomainsNames; + + +def getDockerComputeDomainFromName(dockerComputeDomainName, dockerComputeDomains = None): + """ + Returns a DockerComputeDomain object, given its registered name. + :param dockerComputeDomainName: name of the DockerComputeDomainName, as shown within the results of Jobs.getDockerComputeDomains() + :param dockerComputeDomains: a list of dockerComputeDomain objects (dictionaries), as returned by Jobs.getDockerComputeDomains(). If not set, then an internal call to Jobs.getDockerComputeDomains() is made. + :return: a DockerComputeDomain object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: dockerComputeDomain = Jobs.getDockerComputeDomainFromName('dockerComputeDomainAtJHU'); + .. seealso:: Jobs.getDockerComputeDomains, Jobs.getRDBComputeDomains, Jobs.getRDBComputeDomainFromName + """ + if dockerComputeDomainName is None: + raise Exception("dockerComputeDomainName is not defined.") + else: + if dockerComputeDomains is None: + dockerComputeDomains = getDockerComputeDomains(); + + if dockerComputeDomains.__len__() > 0: + for dockerComputeDomain in dockerComputeDomains: + if dockerComputeDomainName == dockerComputeDomain.get('name'): + return dockerComputeDomain; + else: + raise Exception("There are no DockerComputeDomains available for the user."); + + raise Exception("DockerComputeDomain of name '" + dockerComputeDomainName + "' is not available or does not exist."); + + def getRDBComputeDomains(): """ Gets a list of all registered Relational Database (RDB) compute domains that the user has access to. @@ -81,30 +130,23 @@ def getRDBComputeDomains(): raise Exception("User token is not defined. First log into SciServer.") -def getDockerComputeDomainFromName(dockerComputeDomainName, dockerComputeDomains = None): +def getRDBComputeDomainsNames(rdbComputeDomains=None): """ - Returns a DockerComputeDomain object, given its registered name. - :param dockerComputeDomainName: name of the DockerComputeDomainName, as shown within the results of Jobs.getDockerComputeDomains() - :param dockerComputeDomains: a list of DockerComputeDomain objects (dictionaries), as returned by Jobs.getDockerComputeDomains(). If not set, then an internal call to Jobs.getDockerComputeDomains() is made. - :return: a DockerComputeDomain object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. - :example: dockerComputeDomain = Jobs.getDockerComputeDomainFromName('dockerComputeDomainAtJHU'); - .. seealso:: Jobs.getDockerComputeDomains, Jobs.getRDBComputeDomains, Jobs.getRDBComputeDomainFromName + Returns the names of the RDB compute domains available to the user. + :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. + :return: an array of strings, each being the name of a rdb compute domain available to the user. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: dockerComputeDomainsNames = Files.getDockerComputeDomainsNames(); + .. seealso:: Files.getRDBComputeDomains """ - if dockerComputeDomainName is None: - raise Exception("dockerComputeDomainName is not defined.") - else: - if dockerComputeDomains is None: - dockerComputeDomains = getDockerComputeDomains(); + if rdbComputeDomains is None: + rdbComputeDomains = getRDBComputeDomains(); - if dockerComputeDomains.__len__() > 0: - for dockerComputeDomain in dockerComputeDomains: - if dockerComputeDomainName == dockerComputeDomain.get('name'): - return dockerComputeDomain; - else: - raise Exception("There are no DockerComputeDomains available for the user."); + rdbComputeDomainsNames = []; + for rdbComputeDomain in rdbComputeDomains: + rdbComputeDomainsNames.append(rdbComputeDomain.get('name')) - raise Exception("DockerComputeDomain of name '" + dockerComputeDomainName + "' is not available or does not exist."); + return rdbComputeDomainsNames; def getRDBComputeDomainFromName(rdbComputeDomainName, rdbComputeDomains = None): @@ -114,7 +156,7 @@ def getRDBComputeDomainFromName(rdbComputeDomainName, rdbComputeDomains = None): :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. :return: an RDBComputeDomain object (dictionary) that defines an RDB compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. - :example: rdvComputeDomain = Jobs.getRDBComputeDomainFromName('rdbComputeDomainAtJHU'); + :example: rdbComputeDomain = Jobs.getRDBComputeDomainFromName('rdbComputeDomainAtJHU'); .. seealso:: Jobs.getDockerComputeDomains, Jobs.getRDBComputeDomains, Jobs.getDockerComputeDomainFromName """ if rdbComputeDomainName is None: @@ -219,27 +261,27 @@ def getJobStatus(jobId): :param jobId: Id of job (integer). :return: dictionary with the integer value of the job status, as well as its semantic meaning. :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. - :example: job = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); status = Jobs.getJobStatus(job.get('id')); + :example: jobId = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); status = Jobs.getJobStatus(jobId); .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob """ intStatus = getJobDescription(jobId)["status"] if intStatus == 1: - return {'status':"PENDING", 'intStatus':intStatus} + return {'status':intStatus, 'statusMeaning':"PENDING", 'jobId':jobId} elif intStatus == 2: - return {'status': "QUEUED", 'intStatus': intStatus} + return {'status': intStatus, 'statusMeaning': "QUEUED", 'jobId':jobId} elif intStatus == 4: - return {'status': "ACCEPTED", 'intStatus': intStatus} + return {'status': intStatus, 'statusMeaning': "ACCEPTED", 'jobId':jobId} elif intStatus == 8: - return {'status': "STARTED", 'intStatus': intStatus} + return {'status': intStatus, 'statusMeaning': "STARTED", 'jobId':jobId} elif intStatus == 16: - return {'status': "FINISHED", 'intStatus': intStatus} + return {'status': intStatus, 'statusMeaning': "FINISHED", 'jobId':jobId} elif intStatus == 32: - return {'status': "SUCCESS", 'intStatus': intStatus} + return {'status': intStatus, 'statusMeaning': "SUCCESS", 'jobId':jobId} elif intStatus == 64: - return {'status': "ERROR", 'intStatus': intStatus} + return {'status': intStatus, 'statusMeaning': "ERROR", 'jobId':jobId} elif intStatus == 128: - return {'status': "CANCELED", 'intStatus': intStatus} + return {'status': intStatus, 'statusMeaning': "CANCELED", 'jobId':jobId} else: raise Exception("Invalid integer value given to job status.") @@ -258,7 +300,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted. :param parameters: string containing parameters that the notebook might need during its execution. This string is written in the 'parameters.txt' file in the same directory level where the notebook is being executed. :param jobAlias: alias (string) of job, defined by the user. - :return: a dictionary containing the definition of the submitted job. + :return: the job ID (int) :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitNotebookJob('/home/idies/workspace/persistent/Notebook.ipynb', dockerComputeDomain, 'Python (astro)', [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}], [{'name':'SDSS_DAS'}], 'param1=1\nparam2=2\nparam3=3','myNewJob') .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob @@ -351,7 +393,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No if res.status_code != 200: raise Exception("Error when submitting a notebook job to the JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: - return json.loads(res.content.decode()) + return (json.loads(res.content.decode())).get('id') else: raise Exception("User token is not defined. First log into SciServer.") @@ -369,7 +411,7 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN E.g., dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}]. A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted. :param jobAlias: alias (string) of job, defined by the user. - :return: a dictionary containing the definition of the submitted job. + :return: the job ID (int) :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitShellCommandJob('pwd', dockerComputeDomain, 'Python (astro)', [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}], [{'name':'SDSS_DAS'}], 'myNewJob') .. seealso:: Jobs.submitNotebookJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob @@ -462,7 +504,7 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN if res.status_code != 200: raise Exception("Error when submitting a job to the JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: - return json.loads(res.content.decode()) + return (json.loads(res.content.decode())).get('id') else: raise Exception("User token is not defined. First log into SciServer.") @@ -524,7 +566,7 @@ def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = Non if res.status_code != 200: raise Exception("Error when submitting a job to the JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: - return json.loads(res.content.decode()) + return (json.loads(res.content.decode())).get('id') else: raise Exception("User token is not defined. First log into SciServer.") @@ -533,7 +575,6 @@ def cancelJob(jobId): """ Cancels the execution of a job. :param jobId: Id of the job (integer) - :return: True if the job is correctly canceled. :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error. :example: job = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); isCanceled = Jobs.cancelJob(job.get('id')); .. seealso:: Jobs.submitNotebookJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains. @@ -554,7 +595,7 @@ def cancelJob(jobId): if res.status_code != 200: raise Exception("Error when getting from JOBM API the job status of jobId=" + str(jobId) + ".\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: - return True; + pass; else: raise Exception("User token is not defined. First log into SciServer.") @@ -583,7 +624,7 @@ def waitForJob(jobId, verbose=True, pollTime = 5): if verbose: #print back, print waitingStr, - jobDesc = getJobDescription(jobId) + jobDesc = getJobStatus(jobId) jobStatus = jobDesc["status"] if jobStatus >= 32: complete = True diff --git a/py3/SciServer/Authentication.py b/py3/SciServer/Authentication.py index ede38b6..73e7ca0 100644 --- a/py3/SciServer/Authentication.py +++ b/py3/SciServer/Authentication.py @@ -17,6 +17,7 @@ class KeystoneUser: """ id = None userName = None + token = None class Token: """ @@ -26,6 +27,7 @@ class Token: token = Token(); +keystoneUser = KeystoneUser(); def getKeystoneUserWithToken(token): """ @@ -38,29 +40,37 @@ def getKeystoneUserWithToken(token): .. seealso:: Authentication.getToken, Authentication.login, Authentication.setToken. """ - taskName = "" - if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.Authentication.getKeystoneUserWithToken" + + if keystoneUser.token == token and keystoneUser.token is not None: + return keystoneUser; + else: - taskName = "SciScript-Python.Authentication.getKeystoneUserWithToken" + taskName = "" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Authentication.getKeystoneUserWithToken" + else: + taskName = "SciScript-Python.Authentication.getKeystoneUserWithToken" - loginURL = Config.AuthenticationURL - if ~loginURL.endswith("/"): - loginURL = loginURL + "/" + loginURL = Config.AuthenticationURL + if ~loginURL.endswith("/"): + loginURL = loginURL + "/" - loginURL = loginURL + token + "?TaskName=" + taskName; + loginURL = loginURL + token + "?TaskName=" + taskName; - getResponse = requests.get(loginURL) - if getResponse.status_code != 200: - raise Exception("Error when getting the keystone user with token " + str(token) +".\nHttp Response from the Authentication API returned status code " + str(getResponse.status_code) + ":\n" + getResponse.content.decode()); + getResponse = requests.get(loginURL) + if getResponse.status_code != 200: + raise Exception("Error when getting the keystone user with token " + str(token) +".\nHttp Response from the Authentication API returned status code " + str(getResponse.status_code) + ":\n" + getResponse.content.decode()); - responseJson = json.loads((getResponse.content.decode())) + responseJson = json.loads((getResponse.content.decode())) - ksu = KeystoneUser() - ksu.userName = responseJson["token"]["user"]["name"] - ksu.id = responseJson["token"]["user"]["id"] + ksu = KeystoneUser() + ksu.userName = responseJson["token"]["user"]["name"] + ksu.id = responseJson["token"]["user"]["id"] + keystoneUser.token = token; + keystoneUser.userName = ksu.userName + keystoneUser.id = ksu.id - return ksu + return ksu def login(UserName=None, Password=None): @@ -80,9 +90,9 @@ def login(UserName=None, Password=None): """ taskName = "" if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.Authentication.login" + taskName = "Compute.SciScript-Python.Authentication.Login" else: - taskName = "SciScript-Python.Authentication.login" + taskName = "SciScript-Python.Authentication.Login" loginURL = Config.AuthenticationURL + "?TaskName=" + taskName diff --git a/py3/SciServer/Config.py b/py3/SciServer/Config.py index 4a23a0f..f922b4a 100644 --- a/py3/SciServer/Config.py +++ b/py3/SciServer/Config.py @@ -22,7 +22,7 @@ """ # URLs for accessing SciServer web services (API endpoints) CasJobsRESTUri = "https://skyserver.sdss.org/CasJobs/RestApi" -AuthenticationURL = "https://portal.sciserver.org/login-portal/keystone/v3/tokens" +AuthenticationURL = "http://scitest12.pha.jhu.edu/login-portal/keystone/v3/tokens" SciDriveHost = "https://www.scidrive.org" SkyQueryUrl = "http://voservices.net/skyquery/Api/V1" SkyServerWSurl = "https://skyserver.sdss.org" diff --git a/py3/SciServer/Files.py b/py3/SciServer/Files.py index 42cea58..6f4c99c 100644 --- a/py3/SciServer/Files.py +++ b/py3/SciServer/Files.py @@ -5,13 +5,15 @@ import json from io import StringIO from io import BytesIO +import warnings +import os - -def getFileServices(): +def getFileServices(verbose=True): """ - Gets the definitions of the file services that a user is able to access. - :return: list of dictionaries, where each dictionary is is the descriptions of a FileService that a user is able to access. + Gets the definitions of file services that a user is able to access. A FileService represents a file system that contains root volumes accessible to the user for public/private data storage. Within each rootVolume, users can create sharable userVolumes for storing files. + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: list of dictionaries, where each dictionary represents the description of a FileService that the user is able to access. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. :example: fileServices = Files.getFileServices(); .. seealso:: Files.getFileServiceFromName @@ -30,20 +32,69 @@ def getFileServices(): res = requests.get(url, headers=headers) if res.status_code >= 200 and res.status_code < 300: - return json.loads(res.content.decode()); + fileServices = []; + fileServicesAPIs = json.loads(res.content.decode()) + for fileServicesAPI in fileServicesAPIs: + url = fileServicesAPI.get("apiEndpoint") + name = fileServicesAPI.get("name") + url = url + "api/volumes/?TaskName="+taskName; + try: + res = requests.get(url, headers=headers) + except: + if verbose: + warnings.warn("Error when getting definition of FileService named '" + name + "' with API URL '" + fileServicesAPI.get("apiEndpoint") + "'. This FileService might be not available", Warning, stacklevel=2) + + if res.status_code >= 200 and res.status_code < 300: + fileServices.append(json.loads(res.content.decode())); + else: + if verbose: + warnings.warn("Error when getting definition of FileService named '" + name + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode(),Warning, stacklevel=2) + return fileServices; else: raise Exception("Error when getting the list of FileServices.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") -def getFileServiceFromName(fileServiceName, fileServices=None): + +def getFileServicesNames(fileServices=None, verbose=True): + """ + Returns the names of the fileServices available to the user. + :param fileServices: a list of FileService objects (dictionaries), as returned by Jobs.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: an array of strings, each being the name of a file service available to the user. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServiceNames = Files.getFileServicesNames(); + .. seealso:: Files.getFileServices + """ + + if fileServices is None: + fileServices = getFileServices(verbose); + + fileServiceNames = []; + if fileServices.__len__() > 0: + for fileService in fileServices: + fileServiceNames.append(fileService.get('name')) + + return fileServiceNames; + else: + #raise Exception("There are no FileServices available for the user."); + if verbose: + warnings.warn("There are no FileServices available for the user.", Warning, stacklevel=2) + return fileServiceNames + + + + + +def getFileServiceFromName(fileServiceName, fileServices=None, verbose=True): """ Returns a FileService object, given its registered name. :param fileServiceName: name of the FileService, as shown within the results of Jobs.getFileServices() :param fileServices: a list of FileService objects (dictionaries), as returned by Jobs.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. - :return: a FileService object (dictionary) that defines a FileService. A list of these kind of objects available to the user is returned by the function Jobs.getFileServices(). - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return a FileService object (dictionary) that defines a FileService. A list of these kind of objects available to the user is returned by the function Jobs.getFileServices(). If no fileService can be found, then returns None. + :raises Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. :example: fileService = Files.getFileServiceFromName('FileServiceAtJHU'); .. seealso:: Files.getFileServices """ @@ -52,189 +103,204 @@ def getFileServiceFromName(fileServiceName, fileServices=None): raise Exception("fileServiceName is not defined.") else: if fileServices is None: - fileServices = getFileServices(); + fileServices = getFileServices(verbose); if fileServices.__len__() > 0: for fileService in fileServices: if fileServiceName == fileService.get('name'): return fileService; + + #raise Exception("FileService of name '" + fileServiceName + "' is not available or does not exist."); + if verbose: + warnings.warn("FileService of name '" + fileServiceName + "' is not available or does not exist.", Warning, stacklevel=2) + return None else: - raise Exception("There are no FileServices available for the user."); + #raise Exception("There are no FileServices available for the user."); + if verbose: + warnings.warn("There are no FileServices available for the user.", Warning, stacklevel=2) + return None - raise Exception("FileService of name '" + fileServiceName + "' is not available or does not exist."); -def __getFileServiceAPIUrl(fileService=None): +def __getFileServiceAPIUrl(fileService): """ Gets the API endpoint URL of a FileService. - :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made, and the returned URL is that of the first FileService found. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :return: API endpoint URL of the FileService (string). - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. - :example: fileServiceUrl = Files.__getFileServiceAPIUrl(); + :example: fileServiceAPIUrl = Files.__getFileServiceAPIUrl(); .. seealso:: Files.getFileServiceFromName """ - if fileService is None: - fileServices = getFileServices(); - if fileServices.__len__() > 0: - return fileServices[0].get("apiEndpoint"); - else: - raise Exception("There are no FileServices available for the user."); + + url = None; + if type(fileService) == type(""): + fileServices = getFileServices(False); + _fileService = getFileServiceFromName(fileService, fileServices, verbose=False); + url = _fileService.get("apiEndpoint"); else: - return fileService.get("apiEndpoint"); + url = fileService.get("apiEndpoint"); + + if not url.endswith("/"): + url = url + "/" + return url -def createDir(path, volume, ownerName=None, fileService=None): + +def getRootVolumes(fileService): """ - Create a directory. - :param path: path (string) to the directory, relative to the volume level. E.g: path="/newDirectory" - :param volume: volume (string) containing the path. - :param ownerName: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. - :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :return: True if the directory creation was successful. + Gets the definitions the RootVolumes available in a particular FileService, and of the file services that a user is able to access. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :return: list of dictionaries, where each dictionary contains the definition of a FileService that a user is able to access. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServices = Files.getFileServices(); rootVolumes = getRootVolumes(fileServices[0]) + .. seealso:: Files.getFileServiceFromName + """ + return fileService.get("rootVolumes"); + + +def createUserVolume(fileService, rootVolume, userVolume, userVolumeOwner=None, quiet=True): + """ + Create a user volume. + :param fileService: name of fileService (string), or object (dictionary) that defines the file service that contains the root volume. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolume: name of root volume (string) that contains the user volume to be created. A list of user volumes is given by getRootVolumes(fileService) + :param userVolume: name of user volume (string) to be created, one level below the root volume level. + :param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume. + :param quiet: if set to False, will throw an error if the User Volume already exists. If True, won't throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); Files.createDir("myNewDir","persistent","myUserName", fileServices[0]); + :example: fileServices = Files.getFileServices(); Files.createUserVolume(fileServices[0], "volumes","newUserVolume"); .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList """ token = Authentication.getToken() if token is not None and token != "": if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.Files.createDir" + taskName = "Compute.SciScript-Python.Files.createUserVolume" else: - taskName = "SciScript-Python.Files.createDir" - - if ~path.startswith("/"): - path = "/" + path; + taskName = "SciScript-Python.Files.createUserVolume" - if ownerName is None: - ownerName = Authentication.getKeystoneUserWithToken().userName; + if userVolumeOwner is None: + userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - url = Config.FileServiceURL + "/folder/" + volume + "/" + ownerName + path + "?TaskName="+taskName; + url = __getFileServiceAPIUrl(fileService) + "api/volume/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?quiet="+str(quiet) + "&TaskName="+taskName; headers = {'X-Auth-Token': token} res = requests.put(url, headers=headers) if res.status_code >= 200 and res.status_code < 300: - return True; + pass; else: - raise Exception("Error when creating directory '" + path + "' in volume '" + volume + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when creating user volume '" + userVolume + "' in root volume '" + rootVolume + "' in file service named '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") -def dirList(path, volume, ownerName=None, fileService=None, level=1, options=''): +def deleteUserVolume(fileService, rootVolume, userVolume, userVolumeOwner=None, quiet=True): """ - Lists the contents of a directory. - :param path: path (string) to the directory, relative to the volume level. E.g: path="/newDirectory" - :param volume: volume (string) containing the path. - :param ownerName: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. - :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param level: amount (int) of listed directory levels that are below or at the same level of the path. - :param options: string of file filtering options. - :return: dictionary containing the directory listing. + Delete a user volume. + :param fileService: name of fileService (string), or object (dictionary) that defines the file service that contains the root volume. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolume: root volume name (string) that contains the user volume to be deleted. A list of user volumes is given by getRootVolumes(fileService) + :param userVolume: user volume name (string) to be deleted. + :param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume. + :param quiet: If set to False, it will throw an error if user volume does not exists. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); dirs = Files.dirList("/","persistent","myUserName", fileServices[0], 2); - .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.createDir + :example: fileServices = Files.getFileServices(); Files.deleteUserVolume("volumes","newUserVolume",fileServices[0]); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList """ token = Authentication.getToken() if token is not None and token != "": if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.Files.dirList" + taskName = "Compute.SciScript-Python.Files.createUserVolume" else: - taskName = "SciScript-Python.Files.dirList" - - if ownerName is None: - ownerName = Authentication.getKeystoneUserWithToken().userName; + taskName = "SciScript-Python.Files.createUserVolume" - if ~path.startswith("/"): - path = "/" + path; + if userVolumeOwner is None: + userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - url = Config.FileServiceURL + "/jsontree/" + volume + "/" + ownerName + path + "?options=" + str(options) + "&level=" + str(level) + "&TaskName=" + taskName; + url = __getFileServiceAPIUrl(fileService) + "api/volume/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?quiet="+str(quiet)+"&TaskName="+taskName; headers = {'X-Auth-Token': token} - res = requests.get(url, headers=headers) + res = requests.delete(url, headers=headers) if res.status_code >= 200 and res.status_code < 300: - return res.content.decode(); + pass else: - raise Exception("Error when creating new directory " + path + ".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when deleting user volume '" + userVolume + "' in root volume '" + rootVolume + "' in file service named '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") - - -def delete(path, volume, ownerName=None, fileService=None): +def createDir(fileService, rootVolume, userVolume, relativePath, userVolumeOwner=None, quiet=True): """ - Deletes a directory or file in the File System. - :param path: path (string) to the directory or file to be deleted, relative to the volume level. E.g: path="/directory/file.txt" - :param volume: volume (string) containing the path. - :param ownerName: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. - :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :return: True if the deletion was successful. + Create a directory. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. + :param userVolume: user volume name (string) that contains the relativePath. + :param relativePath: path (string) to the directory, relative to the userVolume level. E.g: path="/newDirectory" + :param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume. + :param quiet: If set to False, it will throw an error if the directory already exists. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]); - .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList + :example: fileServices = Files.getFileServices(); Files.createDir(fileServices[0], "myRootVolume","myUserVolume", "myNewDir"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList """ token = Authentication.getToken() if token is not None and token != "": if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.Files.delete" + taskName = "Compute.SciScript-Python.Files.createDir" else: - taskName = "SciScript-Python.Files.delete" + taskName = "SciScript-Python.Files.createDir" + + if not relativePath.startswith("/"): + relativePath = "/" + relativePath; - if ~path.startswith("/"): - path = "/" + path; + if userVolumeOwner is None: + userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - url = Config.FileServiceURL + "/data" + path + "?TaskName="+taskName + url = __getFileServiceAPIUrl(fileService) + "api/folder/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName; headers = {'X-Auth-Token': token} - res = requests.delete(url, headers=headers) + res = requests.put(url, headers=headers) if res.status_code >= 200 and res.status_code < 300: - return True; + pass; else: - raise Exception("Error when deleting " + path + ".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when creating directory '" + relativePath + "' in user volume '" + userVolume + "' in root volume '" + rootVolume + "' in file service named '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") - -def upload(path, volume, ownerName=None, fileService=None, data=None, localFilePath=None): +def upload(fileService, rootVolume, userVolume, relativePath, data="", localFilePath=None, userVolumeOwner=None, quiet=True): """ Uploads data or a local file into a path defined in the file system. - :param path: destination path (string) of the file to be uploaded, relative to the volume level. E.g: path="/newDirectory/myNewFile.txt" - :param volume: volume (string) containing the path. - :param ownerName: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. - :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param data: string containing data to be uploaded. - :param localFilePath: path to a local file to be uploaded (string). - :return: True is upload was successful. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. + :param userVolume: user volume name (string) that contains the relativePath. + :param relativePath: destination path (string) of the file to be uploaded, relative to the user volume level. E.g: path="/newDirectory/myNewFile.txt". + :param data: string containing data to be uploaded, in case localFilePath is not set. + :param localFilePath: path to a local file to be uploaded (string), + :param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume. + :param quiet: If set to False, it will throw an error if the file already exists. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); isUploaded = Files.upload("/myUploadedFile.txt","persistent","myUserName", fileServices[0], localFilePath="/myFile.txt"); + :example: fileServices = Files.getFileServices(); Files.upload(fileServices[0], "myRootVolume", "myUserVolume", "/myUploadedFile.txt", None, None, localFilePath="/myFile.txt"); .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.download, Files.dirList """ token = Authentication.getToken() if token is not None and token != "": if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.Files.upload" + taskName = "Compute.SciScript-Python.Files.UploadFile" else: - taskName = "SciScript-Python.Files.upload" + taskName = "SciScript-Python.Files.UploadFile" - if ~path.startswith("/"): - path = "/" + path; + if not relativePath.startswith("/"): + relativePath = "/" + relativePath; - fileName = path.split("/")[-1] - dirPath = path.replace(fileName, "") + if userVolumeOwner is None: + userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - createDir(dirPath); - - url = Config.FileServiceURL + "/data" + path + "?TaskName="+taskName + url = __getFileServiceAPIUrl(fileService) + "api/file/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName headers = {'X-Auth-Token': token} @@ -245,25 +311,27 @@ def upload(path, volume, ownerName=None, fileService=None, data=None, localFileP if data != None: res = requests.put(url, data=data, headers=headers, stream=True) else: - raise Exception("Error when uploading to " + path + ". No local file or data specified for uploading."); + raise Exception("Error: No local file or data specified for uploading."); if res.status_code >= 200 and res.status_code < 300: - return True; + pass; else: - raise Exception("Error when uploading to " + path +".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when uploading file '" + relativePath + "' in user volume '" + userVolume + "' in root volume '" + rootVolume + "' in file service named '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") -def download(path, volume, ownerName=None, fileService=None, localFilePath="", format="text"): +def download(fileService, rootVolume, userVolume, relativePath, localFilePath=None, format="txt", userVolumeOwner=None, quiet=True): """ Downloads a file from the remote file system into the local file system, or returns the file content as an object in several formats. - :param path: remote path (string) of the file to be downloaded, relative to the volume level. E.g: path="/newDirectory/myNewFile.txt" - :param volume: volume (string) containing the path. - :param ownerName: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. - :param fileService: object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param localFilePath: local path of the file to be downloaded. If 'localFilePath' is defined, then the 'format' parameter is not used. - :param format: type of the returned object. Can be "StringIO" (io.StringIO object containing readable text), "BytesIO" (io.BytesIO object containing readable binary data), "response" ( the HTTP response as an object of class requests.Response) or "text" (a text string). If the parameter 'localFilePath' is defined, then the 'format' parameter is not used and the file is downloaded to the local file system instead. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. + :param userVolume: user volume name (string) that contains the relativePath. + :param relativePath: remote path (string) of the file to be downloaded, relative to the user volume level. E.g: path="/newDirectory/myNewFile.txt" + :param localFilePath: local destination path of the file to be downloaded. If set to None, then an object of format 'format' will be returned. + :param format: name (string) of the returned object's type (if localFilePath is not defined). This parameter can be "StringIO" (io.StringIO object containing readable text), "BytesIO" (io.BytesIO object containing readable binary data), "response" ( the HTTP response as an object of class requests.Response) or "txt" (a text string). If the parameter 'localFilePath' is defined, then the 'format' parameter is not used and the file is downloaded to the local file system instead. + :param userVolumeOwner: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. + :param quiet: If set to False, it will throw an error if the file already exists. If set to True. it will not throw an error. :return: If the 'localFilePath' parameter is defined, then it will return True when the file is downloaded successfully in the local file system. If the 'localFilePath' is not defined, then the type of the returned object depends on the value of the 'format' parameter (either io.StringIO, io.BytesIO, requests.Response or string). :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); isDownloaded = Files.upload("/myUploadedFile.txt","persistent","myUserName", fileServices[0], localFilePath="/myDownloadedFile.txt"); @@ -272,21 +340,29 @@ def download(path, volume, ownerName=None, fileService=None, localFilePath="", f token = Authentication.getToken() if token is not None and token != "": + + if localFilePath is not None: + if os.path.isfile(localFilePath) and not quiet: + raise Exception("Error when downloading from remote File Syatem the file " + str(relativePath) + ".\nLocal file '" + localFilePath + "' already exists."); + if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.Files.download" + taskName = "Compute.SciScript-Python.Files.DownloadFile" else: - taskName = "SciScript-Python.Files.download" + taskName = "SciScript-Python.Files.DownloadFile" + + if not relativePath.startswith("/"): + relativePath = "/" + relativePath; - if ~path.startswith("/"): - path = "/" + path; + if userVolumeOwner is None: + userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - url = Config.FileServiceURL + "/data" + path + "?TaskName="+taskName; + url = __getFileServiceAPIUrl(fileService) + "api/file/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?TaskName=" + taskName; headers = {'X-Auth-Token': token} res = requests.get(url, stream=True, headers=headers) if res.status_code < 200 or res.status_code >= 300: - raise Exception("Error when downloading from remote File Syatem the file " + str(path) + ".\nHttp Response from FileSystem API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when downloading from remote File Syatem the file " + str(relativePath) + ".\nHttp Response from FileSystem API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); if localFilePath is not None and localFilePath != "": @@ -301,16 +377,199 @@ def download(path, volume, ownerName=None, fileService=None, localFilePath="", f if format is not None and format != "": if format == "StringIO": return StringIO(res.content.decode()) - if format == "text": + if format == "txt": return res.content.decode() elif format == "BytesIO": return BytesIO(res.content) elif format == "response": return res; else: - raise Exception("Unknown format '" + format + "' when trying to download from remote File System the file " + str(path) + ".\n"); + raise Exception("Unknown format '" + format + "' when trying to download from remote File System the file " + str(relativePath) + ".\n"); else: raise Exception("Wrong format parameter value\n"); else: raise Exception("User token is not defined. First log into SciServer.") + + + +def dirList(fileService, rootVolume, userVolume, relativePath, level=1, options='', userVolumeOwner=None): + """ + Lists the contents of a directory. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. + :param userVolume: user volume name (string) that contains the relativePath. + :param relativePath: remote path (string) of the directory, relative to the user volume level. E.g: path="/newDirectory/myNewFile.txt" + :param level: amount (int) of listed directory levels that are below or at the same level to that of the relativePath. + :param options: string of file filtering options. + :param userVolumeOwner: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. + :return: dictionary containing the directory listing. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); dirs = Files.dirList("/","persistent","myUserName", fileServices[0], 2); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.createDir + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.dirList" + else: + taskName = "SciScript-Python.Files.dirList" + + if userVolumeOwner is None: + userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + + if not relativePath.startswith("/"): + path = "/" + relativePath; + + url = __getFileServiceAPIUrl(fileService) + "api/jsontree/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + path + "?options=" + options + "&level=" + str(level) + "&TaskName=" + taskName; + + headers = {'X-Auth-Token': token} + res = requests.get(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + return json.loads(res.content.decode()); + else: + raise Exception("Error when listing directory.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def move(fileService, rootVolume, userVolume, relativePath, destinationFileService, destinationRootVolume, destinationUserVolume, destinationRelativePath, replaceExisting=True, doCopy=True, userVolumeOwner=None, destinationUserVolumeOwner=None): + """ + Moves or copies a file or folder. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileService object. + :param userVolume: user volume name (string) that contains the relativePath. + :param relativePath: remote path (string) of the file to be moved/copied, relative to the user volume level. E.g: path="/newDirectory/myNewFile.txt" + :param destinationFileService: name of fileService (string), or object (dictionary) that defines a destination file service (where the file is moved/copied into). A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param destinationRootVolume: root volume name (string) that contains the userVolume where the file is moved/copied into. A list of root volumes is contained by the fileService object. + :param destinationUserVolume: user volume name (string) that contains the relativePath where the file is moved/copied into. + :param destinationRelativePath: remote path (string) of the destination file, relative to the user volume level. E.g: path="/newDirectory2/myNewFile.txt" + :param replaceExisting: If set to False, it will throw an error if the file already exists, If set to True, it will not throw and eeror in that case. + :param doCopy: if set to True, then it will copy the file or folder. If set to False, then the file or folder will be moved. + :param userVolumeOwner: name (string) of owner of the user volume. Can be left undefined if requester is the owner of the volume. + :param destinationUserVolumeOwner: name (string) of owner of the destination user volume. Can be left undefined if requester is the owner of it. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); isDownloaded = Files.upload("/myUploadedFile.txt","persistent","myUserName", fileServices[0], localFilePath="/myDownloadedFile.txt"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.upload, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.DownloadFile" + else: + taskName = "SciScript-Python.Files.DownloadFile" + + if __getFileServiceAPIUrl(fileService) != __getFileServiceAPIUrl(fileService): + raise Exception("Use of multiple file services is not supported for now."); + + if not relativePath.startswith("/"): + relativePath = "/" + relativePath; + + if not destinationRelativePath.startswith("/"): + destinationRelativePath = "/" + destinationRelativePath; + + if userVolumeOwner is None: + userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + + if destinationUserVolumeOwner is None: + destinationuserVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + + url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName; + headers = {'X-Auth-Token': token} + jsonDict = {'destinationPath': destinationRelativePath, 'destinationRootFolder': destinationRootVolume, 'destinationUserVolume':destinationUserVolume, 'destinationOwner': destinationuserVolumeOwner}; + data = json.dumps(jsonDict).encode() + res = requests.put(url, stream=True, headers=headers, data=data) + + if res.status_code < 200 or res.status_code >= 300: + raise Exception("Error when downloading from remote File System the file " + str(relativePath) + ".\nHttp Response from FileSystem API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + + else: + raise Exception("User token is not defined. First log into SciServer.") + + + +def delete(fileService, rootVolume, userVolume, relativePath, userVolumeOwner=None, quiet=True): + """ + Deletes a directory or file in the File System. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. + :param userVolume: user volume name (string) that contains the relativePath. + :param relativePath: remote path (string) of the directory or file to be deleted, relative to the user volume level. E.g: path="/newDirectory/myNewFile.txt" + :param userVolumeOwner: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. + :param quiet: If set to False, it will throw an error if the file does not exist. If set to True. it will not throw an error. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.delete" + else: + taskName = "SciScript-Python.Files.delete" + + if not relativePath.startswith("/"): + relativePath = "/" + relativePath; + + if userVolumeOwner is None: + userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + + url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName + + headers = {'X-Auth-Token': token} + res = requests.delete(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + pass; + else: + raise Exception("Error when deleting " + relativePath + ".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + + +def shareUserVolume(fileService, rootVolume, userVolume, sharedWith, allowedActions, type="USER", userVolumeOwner=None): + """ + Deletes a directory or file in the File System. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. + :param userVolume: user volume name (string) that contains the relativePath. + :param sharedWith: name (string) of user or group that the userVolume is shared with. + :param allowedActions: array of strings defining actions the user or group is allowed to do with respect to the shared user volume. E.g.: ["read","write","grant","delete"]. The "grant" action means that the user or group can also share the user volume with another user or group. The "delete" action meand ability to delete the user volume (use with care). + :param type: type (string) of the entity defined by the "sharedWith" parameter. Can be set to "USER" or "GROUP". + :param userVolumeOwner: name (string) of owner of the user volume. Can be left undefined if requester is the owner of the volume. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.shareUserVolume(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]); + .. seealso:: Files.getFileServices(), Files.getFilrmsieServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.ShareUserVolume" + else: + taskName = "SciScript-Python.Files.ShareUserVolume" + + if userVolumeOwner is None: + userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + + + data = [{'name': sharedWith, 'type':type, 'allowedActions': allowedActions }] + body = json.dumps(data).encode() + + url = __getFileServiceAPIUrl(fileService) + "api/share/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?TaskName="+taskName + + headers = {'X-Auth-Token': token} + res = requests.patch(url, headers=headers, data=body) + + if res.status_code >= 200 and res.status_code < 300: + pass; + else: + raise Exception("Error when sharing userVolume '" + userVolume + "' in rootVolume '" + rootVolume + "' with '" + sharedWith + "' in file service named '" + fileService.get( + 'name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py index 6315b9a..f141179 100644 --- a/py3/SciServer/Jobs.py +++ b/py3/SciServer/Jobs.py @@ -12,7 +12,11 @@ def getJobDirectory(): """ - Gets the root directory path where the job is being executed. Since its value depends on the value of jobId, the path will be defined only after the job is submitted. + Gets the root directory path where the job is being executed. Since its value depends on the value of jobId, the path will be defined only after the job is submitted. It can be called within Jupyter notebooks or shell commands running as jobs. + :return: a string defining the path to the job directory. + :raises: Throws an exception if job directory cannot be found. + :example: jobDirectory = getJobDirectory(); + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getRDBComputeDomains, Jobs.cancelJob """ computeJobDirectoryFile = Config.ComputeJobDirectoryFile if os.path.isfile(computeJobDirectoryFile): @@ -54,6 +58,51 @@ def getDockerComputeDomains(): raise Exception("User token is not defined. First log into SciServer.") +def getDockerComputeDomainsNames(dockerComputeDomains=None): + """ + Returns the names of the docker compute domains available to the user. + :param dockerComputeDomains: a list of dockerComputeDomain objects (dictionaries), as returned by Jobs.getDockerComputeDomains(). If not set, then an extra internal call to Jobs.getDockerComputeDomains() is made. + :return: an array of strings, each being the name of a docker compute domain available to the user. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: dockerComputeDomainsNames = Files.getDockerComputeDomainsNames(); + .. seealso:: Files.getDockerComputeDomains + """ + if dockerComputeDomains is None: + dockerComputeDomains = getDockerComputeDomains(); + + dockerComputeDomainsNames = []; + for dockerComputeDomain in dockerComputeDomains: + dockerComputeDomainsNames.append(dockerComputeDomain.get('name')) + + return dockerComputeDomainsNames; + + +def getDockerComputeDomainFromName(dockerComputeDomainName, dockerComputeDomains = None): + """ + Returns a DockerComputeDomain object, given its registered name. + :param dockerComputeDomainName: name of the DockerComputeDomainName, as shown within the results of Jobs.getDockerComputeDomains() + :param dockerComputeDomains: a list of dockerComputeDomain objects (dictionaries), as returned by Jobs.getDockerComputeDomains(). If not set, then an internal call to Jobs.getDockerComputeDomains() is made. + :return: a DockerComputeDomain object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: dockerComputeDomain = Jobs.getDockerComputeDomainFromName('dockerComputeDomainAtJHU'); + .. seealso:: Jobs.getDockerComputeDomains, Jobs.getRDBComputeDomains, Jobs.getRDBComputeDomainFromName + """ + if dockerComputeDomainName is None: + raise Exception("dockerComputeDomainName is not defined.") + else: + if dockerComputeDomains is None: + dockerComputeDomains = getDockerComputeDomains(); + + if dockerComputeDomains.__len__() > 0: + for dockerComputeDomain in dockerComputeDomains: + if dockerComputeDomainName == dockerComputeDomain.get('name'): + return dockerComputeDomain; + else: + raise Exception("There are no DockerComputeDomains available for the user."); + + raise Exception("DockerComputeDomain of name '" + dockerComputeDomainName + "' is not available or does not exist."); + + def getRDBComputeDomains(): """ Gets a list of all registered Relational Database (RDB) compute domains that the user has access to. @@ -81,30 +130,23 @@ def getRDBComputeDomains(): raise Exception("User token is not defined. First log into SciServer.") -def getDockerComputeDomainFromName(dockerComputeDomainName, dockerComputeDomains = None): +def getRDBComputeDomainsNames(rdbComputeDomains=None): """ - Returns a DockerComputeDomain object, given its registered name. - :param dockerComputeDomainName: name of the DockerComputeDomainName, as shown within the results of Jobs.getDockerComputeDomains() - :param dockerComputeDomains: a list of DockerComputeDomain objects (dictionaries), as returned by Jobs.getDockerComputeDomains(). If not set, then an internal call to Jobs.getDockerComputeDomains() is made. - :return: a DockerComputeDomain object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. - :example: dockerComputeDomain = Jobs.getDockerComputeDomainFromName('dockerComputeDomainAtJHU'); - .. seealso:: Jobs.getDockerComputeDomains, Jobs.getRDBComputeDomains, Jobs.getRDBComputeDomainFromName + Returns the names of the RDB compute domains available to the user. + :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. + :return: an array of strings, each being the name of a rdb compute domain available to the user. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: dockerComputeDomainsNames = Files.getDockerComputeDomainsNames(); + .. seealso:: Files.getRDBComputeDomains """ - if dockerComputeDomainName is None: - raise Exception("dockerComputeDomainName is not defined.") - else: - if dockerComputeDomains is None: - dockerComputeDomains = getDockerComputeDomains(); + if rdbComputeDomains is None: + rdbComputeDomains = getRDBComputeDomains(); - if dockerComputeDomains.__len__() > 0: - for dockerComputeDomain in dockerComputeDomains: - if dockerComputeDomainName == dockerComputeDomain.get('name'): - return dockerComputeDomain; - else: - raise Exception("There are no DockerComputeDomains available for the user."); + rdbComputeDomainsNames = []; + for rdbComputeDomain in rdbComputeDomains: + rdbComputeDomainsNames.append(rdbComputeDomain.get('name')) - raise Exception("DockerComputeDomain of name '" + dockerComputeDomainName + "' is not available or does not exist."); + return rdbComputeDomainsNames; def getRDBComputeDomainFromName(rdbComputeDomainName, rdbComputeDomains = None): @@ -114,7 +156,7 @@ def getRDBComputeDomainFromName(rdbComputeDomainName, rdbComputeDomains = None): :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. :return: an RDBComputeDomain object (dictionary) that defines an RDB compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. - :example: rdvComputeDomain = Jobs.getRDBComputeDomainFromName('rdbComputeDomainAtJHU'); + :example: rdbComputeDomain = Jobs.getRDBComputeDomainFromName('rdbComputeDomainAtJHU'); .. seealso:: Jobs.getDockerComputeDomains, Jobs.getRDBComputeDomains, Jobs.getDockerComputeDomainFromName """ if rdbComputeDomainName is None: @@ -219,27 +261,27 @@ def getJobStatus(jobId): :param jobId: Id of job (integer). :return: dictionary with the integer value of the job status, as well as its semantic meaning. :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. - :example: job = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); status = Jobs.getJobStatus(job.get('id')); + :example: jobId = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); status = Jobs.getJobStatus(jobId); .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob """ intStatus = getJobDescription(jobId)["status"] if intStatus == 1: - return {'status':"PENDING", 'intStatus':intStatus} + return {'status':intStatus, 'statusMeaning':"PENDING", 'jobId':jobId} elif intStatus == 2: - return {'status': "QUEUED", 'intStatus': intStatus} + return {'status': intStatus, 'statusMeaning': "QUEUED", 'jobId':jobId} elif intStatus == 4: - return {'status': "ACCEPTED", 'intStatus': intStatus} + return {'status': intStatus, 'statusMeaning': "ACCEPTED", 'jobId':jobId} elif intStatus == 8: - return {'status': "STARTED", 'intStatus': intStatus} + return {'status': intStatus, 'statusMeaning': "STARTED", 'jobId':jobId} elif intStatus == 16: - return {'status': "FINISHED", 'intStatus': intStatus} + return {'status': intStatus, 'statusMeaning': "FINISHED", 'jobId':jobId} elif intStatus == 32: - return {'status': "SUCCESS", 'intStatus': intStatus} + return {'status': intStatus, 'statusMeaning': "SUCCESS", 'jobId':jobId} elif intStatus == 64: - return {'status': "ERROR", 'intStatus': intStatus} + return {'status': intStatus, 'statusMeaning': "ERROR", 'jobId':jobId} elif intStatus == 128: - return {'status': "CANCELED", 'intStatus': intStatus} + return {'status': intStatus, 'statusMeaning': "CANCELED", 'jobId':jobId} else: raise Exception("Invalid integer value given to job status.") @@ -258,7 +300,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted. :param parameters: string containing parameters that the notebook might need during its execution. This string is written in the 'parameters.txt' file in the same directory level where the notebook is being executed. :param jobAlias: alias (string) of job, defined by the user. - :return: a dictionary containing the definition of the submitted job. + :return: the job ID (int) :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitNotebookJob('/home/idies/workspace/persistent/Notebook.ipynb', dockerComputeDomain, 'Python (astro)', [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}], [{'name':'SDSS_DAS'}], 'param1=1\nparam2=2\nparam3=3','myNewJob') .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob @@ -351,7 +393,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No if res.status_code != 200: raise Exception("Error when submitting a notebook job to the JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: - return json.loads(res.content.decode()) + return (json.loads(res.content.decode())).get('id') else: raise Exception("User token is not defined. First log into SciServer.") @@ -369,7 +411,7 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN E.g., dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}]. A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted. :param jobAlias: alias (string) of job, defined by the user. - :return: a dictionary containing the definition of the submitted job. + :return: the job ID (int) :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitShellCommandJob('pwd', dockerComputeDomain, 'Python (astro)', [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}], [{'name':'SDSS_DAS'}], 'myNewJob') .. seealso:: Jobs.submitNotebookJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob @@ -462,7 +504,7 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN if res.status_code != 200: raise Exception("Error when submitting a job to the JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: - return json.loads(res.content.decode()) + return (json.loads(res.content.decode())).get('id') else: raise Exception("User token is not defined. First log into SciServer.") @@ -524,7 +566,7 @@ def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = Non if res.status_code != 200: raise Exception("Error when submitting a job to the JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: - return json.loads(res.content.decode()) + return (json.loads(res.content.decode())).get('id') else: raise Exception("User token is not defined. First log into SciServer.") @@ -533,7 +575,6 @@ def cancelJob(jobId): """ Cancels the execution of a job. :param jobId: Id of the job (integer) - :return: True if the job is correctly canceled. :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error. :example: job = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); isCanceled = Jobs.cancelJob(job.get('id')); .. seealso:: Jobs.submitNotebookJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains. @@ -554,7 +595,7 @@ def cancelJob(jobId): if res.status_code != 200: raise Exception("Error when getting from JOBM API the job status of jobId=" + str(jobId) + ".\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: - return True; + pass; else: raise Exception("User token is not defined. First log into SciServer.") @@ -577,13 +618,13 @@ def waitForJob(jobId, verbose=True, pollTime = 5): waitingStr = "Waiting..." back = "\b" * len(waitingStr) if verbose: - print(waitingStr, end="") + print waitingStr, while not complete: if verbose: #print back, - print(waitingStr, end="") - jobDesc = getJobDescription(jobId) + print(waitingStr) + jobDesc = getJobStatus(jobId) jobStatus = jobDesc["status"] if jobStatus >= 32: complete = True From 9d283a73864b6b8e2e9216efa94d813745c1267a Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Mon, 5 Feb 2018 10:39:26 -0500 Subject: [PATCH 20/79] fixed missing parameters in example notebook and tests. --- Examples/Examples_SciScript-Python.ipynb | 6 +++--- Tests/UnitTests.py | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Examples/Examples_SciScript-Python.ipynb b/Examples/Examples_SciScript-Python.ipynb index 8b219a2..e9b0360 100644 --- a/Examples/Examples_SciScript-Python.ipynb +++ b/Examples/Examples_SciScript-Python.ipynb @@ -1538,7 +1538,7 @@ " dockerComputeDomain,\n", " Jobs_DockerImageName, \n", " Jobs_UserVolumes, Jobs_DataVolumes,\n", - " Jobs_Alias)\n", + " Jobs_Parameters, Jobs_Alias)\n", "print(jobId)" ] }, @@ -1584,7 +1584,7 @@ " dockerComputeDomain,\n", " Jobs_DockerImageName, \n", " Jobs_UserVolumes, Jobs_DataVolumes,\n", - " Jobs_Alias)\n", + " Jobs_Parameters, Jobs_Alias)\n", "jobStatus = Jobs.waitForJob(jobId)\n", "print(jobStatus)" ] @@ -1603,7 +1603,7 @@ " dockerComputeDomain,\n", " Jobs_DockerImageName, \n", " Jobs_UserVolumes, Jobs_DataVolumes,\n", - " Jobs_Alias)\n", + " Jobs_Parameters, Jobs_Alias)\n", "Jobs.cancelJob(jobId)" ] }, diff --git a/Tests/UnitTests.py b/Tests/UnitTests.py index bb8eadd..4edbedb 100644 --- a/Tests/UnitTests.py +++ b/Tests/UnitTests.py @@ -608,7 +608,7 @@ def test_Jobs_submitNotebookJob_cancel_waitForJob_getJobStatus_getJobDescription dockerComputeDomain, Jobs_DockerImageName, Jobs_UserVolumes, Jobs_DataVolumes, - Jobs_Alias) + Jobs_Parameters, Jobs_Alias) Jobs.cancelJob(jobId_1) jobStatus = Jobs.getJobStatus(jobId_1) self.assertTrue(jobStatus.get('status') == 128) @@ -618,7 +618,7 @@ def test_Jobs_submitNotebookJob_cancel_waitForJob_getJobStatus_getJobDescription dockerComputeDomain, Jobs_DockerImageName, Jobs_UserVolumes, Jobs_DataVolumes, - Jobs_Alias) + Jobs_Parameters, Jobs_Alias) jobStatus = Jobs.waitForJob(jobId_2) self.assertTrue(jobStatus == Jobs.getJobStatus(jobId_2)) @@ -628,7 +628,7 @@ def test_Jobs_submitNotebookJob_cancel_waitForJob_getJobStatus_getJobDescription self.assertTrue(job.get('username') == Authentication_loginName) self.assertTrue(job.get('dockerImageName') == Jobs_DockerImageName) self.assertTrue(job.get('scriptURI') == Jobs_RemoteNotebookPath) - #self.assertTrue(job.get('submitterDID') == Jobs_Alias) + self.assertTrue(job.get('submitterDID') == Jobs_Alias) jobDirectory = job.get('resultsFolderURI') relativePath = jobDirectory.split('scratch/')[1] + Jobs_NoteBookOutPutFile; @@ -733,12 +733,12 @@ def test_Jobs_getJobDirectory(self): unittest.TestLoader.sortTestMethodsUsing = lambda x, y: cmp(x,y); testLoader = unittest.TestLoader() testLoader.sortTestMethodsUsing = lambda x, y: 0; - suite = testLoader.loadTestsFromTestCase(TestAuthentication); unittest.TextTestRunner(verbosity=2).run(suite) - suite = testLoader.loadTestsFromTestCase(TestLoginPortal); unittest.TextTestRunner(verbosity=2).run(suite) - suite = testLoader.loadTestsFromTestCase(TestCasJobs); unittest.TextTestRunner(verbosity=2).run(suite) - suite = testLoader.loadTestsFromTestCase(TestSkyServer); unittest.TextTestRunner(verbosity=2).run(suite) - suite = testLoader.loadTestsFromTestCase(TestSciDrive); unittest.TextTestRunner(verbosity=2).run(suite) - suite = testLoader.loadTestsFromTestCase(TestSkyQuery); unittest.TextTestRunner(verbosity=2).run(suite) - suite = testLoader.loadTestsFromTestCase(TestFileService); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestAuthentication); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestLoginPortal); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestCasJobs); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestSkyServer); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestSciDrive); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestSkyQuery); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestFileService); unittest.TextTestRunner(verbosity=2).run(suite) suite = testLoader.loadTestsFromTestCase(TestJobs); unittest.TextTestRunner(verbosity=2).run(suite) From 06564046c2edfdbad162f56051532dd5e3af9c59 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Wed, 14 Feb 2018 11:05:53 -0500 Subject: [PATCH 21/79] reset all waitForJob methods to common behavior (no default verbose, default wait time of 5 seconds) --- py2/SciServer/CasJobs.py | 10 ++++++---- py2/SciServer/Jobs.py | 8 ++++---- py2/SciServer/SkyQuery.py | 13 +++++++------ py3/SciServer/CasJobs.py | 10 ++++++---- py3/SciServer/Jobs.py | 8 ++++---- py3/SciServer/SkyQuery.py | 13 +++++++------ 6 files changed, 34 insertions(+), 28 deletions(-) diff --git a/py2/SciServer/CasJobs.py b/py2/SciServer/CasJobs.py index 797d366..6cb5e47 100644 --- a/py2/SciServer/CasJobs.py +++ b/py2/SciServer/CasJobs.py @@ -272,12 +272,13 @@ def cancelJob(jobId): raise Exception("User token is not defined. First log into SciServer.") -def waitForJob(jobId, verbose=True): +def waitForJob(jobId, verbose=False, pollTime = 5): """ - Queries the job status from casjobs every 2 seconds and waits for the casjobs job to return a status of 3, 4, or 5 (Cancelled, Failed or Finished, respectively). + Queries regularly the job status and waits until the job is completed. :param jobId: id of job (integer) - :param verbose: if True, will print "wait" messages on the screen while the job is not done. If False, will suppress printing messages on the screen. + :param verbose: if True, will print "wait" messages on the screen while the job is still running. If False, will suppress the printing of messages on the screen. + :param pollTime: idle time interval (integer, in seconds) before querying again for the job status. Minimum value allowed is 5 seconds. :return: After the job is finished, returns a dictionary object containing the job status and related metadata. The "Status" field can be equal to 0 (Ready), 1 (Started), 2 (Canceling), 3(Canceled), 4 (Failed) or 5 (Finished). :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the CasJobs API returns an error. :example: CasJobs.waitForJob(CasJobs.submitJob("select 1")) @@ -286,6 +287,7 @@ def waitForJob(jobId, verbose=True): """ try: + minPollTime = 5 # in seconds complete = False waitingStr = "Waiting..." @@ -305,7 +307,7 @@ def waitForJob(jobId, verbose=True): #print back, print "Done!" else: - time.sleep(2) + time.sleep(max(minPollTime,pollTime)); return jobDesc except Exception as e: diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py index 935b01f..86fbe31 100644 --- a/py2/SciServer/Jobs.py +++ b/py2/SciServer/Jobs.py @@ -600,13 +600,13 @@ def cancelJob(jobId): raise Exception("User token is not defined. First log into SciServer.") -def waitForJob(jobId, verbose=True, pollTime = 5): +def waitForJob(jobId, verbose=False, pollTime = 5): """ - Queries regularly the job status and waits for the job to be completed. + Queries regularly the job status and waits until the job is completed. :param jobId: id of job (integer) - :param verbose: if True, will print "wait" messages on the screen while the job is not done. If False, will suppress printing messages on the screen. - :param pollTime: idle time interval (integer, in seconds) before a new query for the job status. Minimum value allowed is 5 seconds. + :param verbose: if True, will print "wait" messages on the screen while the job is still running. If False, will suppress the printing of messages on the screen. + :param pollTime: idle time interval (integer, in seconds) before querying again for the job status. Minimum value allowed is 5 seconds. :return: After the job is finished, returns a dictionary object containing the job definition. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitShellCommandJob(dockerComputeDomain,'pwd', 'Python (astro)');Jobs.waitForJob(job.get('id')) diff --git a/py2/SciServer/SkyQuery.py b/py2/SciServer/SkyQuery.py index 5cd5f54..2b18fa4 100644 --- a/py2/SciServer/SkyQuery.py +++ b/py2/SciServer/SkyQuery.py @@ -206,12 +206,13 @@ def submitJob(query, queue='quick'): raise Exception("User token is not defined. First log into SciServer.") -def waitForJob(jobId, verbose=True): +def waitForJob(jobId, verbose=False, pollTime = 5): """ - Queries the job status from SkyQuery every 2 seconds and waits for the SkyQuery job to be completed. + Queries regularly the job status and waits until the job is completed. :param jobId: id of job (integer) - :param verbose: if True, will print "wait" messages on the screen while the job is not done. If False, will suppress printing messages on the screen. + :param verbose: if True, will print "wait" messages on the screen while the job is still running. If False, will suppress the printing of messages on the screen. + :param pollTime: idle time interval (integer, in seconds) before querying again for the job status. Minimum value allowed is 5 seconds. :return: After the job is finished, returns a dictionary object containing the job status and related metadata. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error. :example: SkyQuery.waitForJob(SkyQuery.submitJob("select 1")) @@ -219,8 +220,8 @@ def waitForJob(jobId, verbose=True): .. seealso:: SkyQuery.submitJob, SkyQuery.getJobStatus. """ try: + minPollTime = 5 # in seconds complete = False - waitingStr = "Waiting..." back = "\b" * len(waitingStr) if verbose: @@ -231,13 +232,13 @@ def waitForJob(jobId, verbose=True): #print back, print waitingStr, jobDesc = getJobStatus(jobId) - if jobDesc['status'] == 'completed': + if jobDesc['dateFinished'] is not None: complete = True if verbose: #print back, print "Done!" else: - time.sleep(2) + time.sleep(max(minPollTime,pollTime)); return jobDesc except Exception as e: diff --git a/py3/SciServer/CasJobs.py b/py3/SciServer/CasJobs.py index d02dc71..1039776 100644 --- a/py3/SciServer/CasJobs.py +++ b/py3/SciServer/CasJobs.py @@ -272,12 +272,13 @@ def cancelJob(jobId): raise Exception("User token is not defined. First log into SciServer.") -def waitForJob(jobId, verbose=True): +def waitForJob(jobId, verbose=False, pollTime = 5): """ - Queries the job status from casjobs every 2 seconds and waits for the casjobs job to return a status of 3, 4, or 5 (Cancelled, Failed or Finished, respectively). + Queries regularly the job status and waits until the job is completed. :param jobId: id of job (integer) - :param verbose: if True, will print "wait" messages on the screen while the job is not done. If False, will suppress printing messages on the screen. + :param verbose: if True, will print "wait" messages on the screen while the job is still running. If False, will suppress the printing of messages on the screen. + :param pollTime: idle time interval (integer, in seconds) before querying again for the job status. Minimum value allowed is 5 seconds. :return: After the job is finished, returns a dictionary object containing the job status and related metadata. The "Status" field can be equal to 0 (Ready), 1 (Started), 2 (Canceling), 3(Canceled), 4 (Failed) or 5 (Finished). :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the CasJobs API returns an error. :example: CasJobs.waitForJob(CasJobs.submitJob("select 1")) @@ -286,6 +287,7 @@ def waitForJob(jobId, verbose=True): """ try: + minPollTime = 5 # in seconds complete = False waitingStr = "Waiting..." @@ -305,7 +307,7 @@ def waitForJob(jobId, verbose=True): #print(back, end="") print("Done!") else: - time.sleep(2) + time.sleep(max(minPollTime,pollTime)); return jobDesc except Exception as e: diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py index f141179..bf4ea4d 100644 --- a/py3/SciServer/Jobs.py +++ b/py3/SciServer/Jobs.py @@ -600,13 +600,13 @@ def cancelJob(jobId): raise Exception("User token is not defined. First log into SciServer.") -def waitForJob(jobId, verbose=True, pollTime = 5): +def waitForJob(jobId, verbose=False, pollTime = 5): """ - Queries regularly the job status and waits for the job to be completed. + Queries regularly the job status and waits until the job is completed. :param jobId: id of job (integer) - :param verbose: if True, will print "wait" messages on the screen while the job is not done. If False, will suppress printing messages on the screen. - :param pollTime: idle time interval (integer, in seconds) before a new query for the job status. Minimum value allowed is 5 seconds. + :param verbose: if True, will print "wait" messages on the screen while the job is still running. If False, will suppress the printing of messages on the screen. + :param pollTime: idle time interval (integer, in seconds) before querying again for the job status. Minimum value allowed is 5 seconds. :return: After the job is finished, returns a dictionary object containing the job definition. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitShellCommandJob(dockerComputeDomain,'pwd', 'Python (astro)');Jobs.waitForJob(job.get('id')) diff --git a/py3/SciServer/SkyQuery.py b/py3/SciServer/SkyQuery.py index 9b7b06a..63cd765 100644 --- a/py3/SciServer/SkyQuery.py +++ b/py3/SciServer/SkyQuery.py @@ -206,12 +206,13 @@ def submitJob(query, queue='quick'): raise Exception("User token is not defined. First log into SciServer.") -def waitForJob(jobId, verbose=True): +def waitForJob(jobId, verbose=False, pollTime = 5): """ - Queries the job status from SkyQuery every 2 seconds and waits for the SkyQuery job to be completed. + Queries regularly the job status and waits until the job is completed. :param jobId: id of job (integer) - :param verbose: if True, will print "wait" messages on the screen while the job is not done. If False, will suppress printing messages on the screen. + :param verbose: if True, will print "wait" messages on the screen while the job is still running. If False, will suppress the printing of messages on the screen. + :param pollTime: idle time interval (integer, in seconds) before querying again for the job status. Minimum value allowed is 5 seconds. :return: After the job is finished, returns a dictionary object containing the job status and related metadata. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error. :example: SkyQuery.waitForJob(SkyQuery.submitJob("select 1")) @@ -219,8 +220,8 @@ def waitForJob(jobId, verbose=True): .. seealso:: SkyQuery.submitJob, SkyQuery.getJobStatus. """ try: + minPollTime = 5 # in seconds complete = False - waitingStr = "Waiting..." back = "\b" * len(waitingStr) if verbose: @@ -231,13 +232,13 @@ def waitForJob(jobId, verbose=True): #print(back, end="") print(waitingStr, end="") jobDesc = getJobStatus(jobId) - if jobDesc['status'] == 'completed': + if jobDesc['dateFinished'] is not None: complete = True if verbose: #print(back, end="") print("Done!") else: - time.sleep(2) + time.sleep(max(minPollTime,pollTime)); return jobDesc except Exception as e: From c309e9a3d03cf5ef4cdc19339afe1d33cf41b0e0 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Wed, 14 Feb 2018 21:35:21 -0500 Subject: [PATCH 22/79] added possibility in the FileService request of moving a file from the fileService to another fileService --- py2/SciServer/Files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py2/SciServer/Files.py b/py2/SciServer/Files.py index 6f4c99c..afff80f 100644 --- a/py2/SciServer/Files.py +++ b/py2/SciServer/Files.py @@ -479,7 +479,7 @@ def move(fileService, rootVolume, userVolume, relativePath, destinationFileServi url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName; headers = {'X-Auth-Token': token} - jsonDict = {'destinationPath': destinationRelativePath, 'destinationRootFolder': destinationRootVolume, 'destinationUserVolume':destinationUserVolume, 'destinationOwner': destinationuserVolumeOwner}; + jsonDict = {'destinationPath': destinationRelativePath, 'destinationRootFolder': destinationRootVolume, 'destinationUserVolume':destinationUserVolume, 'destinationOwner': destinationuserVolumeOwner, 'destinationFileService': destinationFileService}; data = json.dumps(jsonDict).encode() res = requests.put(url, stream=True, headers=headers, data=data) From cbbc20595d2ebd01050616fdd330e01faefc4230 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Wed, 14 Feb 2018 21:47:14 -0500 Subject: [PATCH 23/79] remove unnecessary check of secondary fileService --- py2/SciServer/Files.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/py2/SciServer/Files.py b/py2/SciServer/Files.py index afff80f..2343b72 100644 --- a/py2/SciServer/Files.py +++ b/py2/SciServer/Files.py @@ -462,9 +462,6 @@ def move(fileService, rootVolume, userVolume, relativePath, destinationFileServi else: taskName = "SciScript-Python.Files.DownloadFile" - if __getFileServiceAPIUrl(fileService) != __getFileServiceAPIUrl(fileService): - raise Exception("Use of multiple file services is not supported for now."); - if not relativePath.startswith("/"): relativePath = "/" + relativePath; From a674955332391225769694b80c30129e49a15fa0 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Wed, 14 Feb 2018 21:48:12 -0500 Subject: [PATCH 24/79] unmask rest of unit tests --- Tests/UnitTests.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Tests/UnitTests.py b/Tests/UnitTests.py index 4edbedb..c51e610 100644 --- a/Tests/UnitTests.py +++ b/Tests/UnitTests.py @@ -733,12 +733,12 @@ def test_Jobs_getJobDirectory(self): unittest.TestLoader.sortTestMethodsUsing = lambda x, y: cmp(x,y); testLoader = unittest.TestLoader() testLoader.sortTestMethodsUsing = lambda x, y: 0; - #suite = testLoader.loadTestsFromTestCase(TestAuthentication); unittest.TextTestRunner(verbosity=2).run(suite) - #suite = testLoader.loadTestsFromTestCase(TestLoginPortal); unittest.TextTestRunner(verbosity=2).run(suite) - #suite = testLoader.loadTestsFromTestCase(TestCasJobs); unittest.TextTestRunner(verbosity=2).run(suite) - #suite = testLoader.loadTestsFromTestCase(TestSkyServer); unittest.TextTestRunner(verbosity=2).run(suite) - #suite = testLoader.loadTestsFromTestCase(TestSciDrive); unittest.TextTestRunner(verbosity=2).run(suite) - #suite = testLoader.loadTestsFromTestCase(TestSkyQuery); unittest.TextTestRunner(verbosity=2).run(suite) - #suite = testLoader.loadTestsFromTestCase(TestFileService); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestAuthentication); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestLoginPortal); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestCasJobs); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestSkyServer); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestSciDrive); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestSkyQuery); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestFileService); unittest.TextTestRunner(verbosity=2).run(suite) suite = testLoader.loadTestsFromTestCase(TestJobs); unittest.TextTestRunner(verbosity=2).run(suite) From f8c63cd44decd3cc843e6b3022d9b2182656d613 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Wed, 14 Feb 2018 21:52:46 -0500 Subject: [PATCH 25/79] replicate lst chenges now in py3 --- py3/SciServer/Files.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/py3/SciServer/Files.py b/py3/SciServer/Files.py index 6f4c99c..2343b72 100644 --- a/py3/SciServer/Files.py +++ b/py3/SciServer/Files.py @@ -462,9 +462,6 @@ def move(fileService, rootVolume, userVolume, relativePath, destinationFileServi else: taskName = "SciScript-Python.Files.DownloadFile" - if __getFileServiceAPIUrl(fileService) != __getFileServiceAPIUrl(fileService): - raise Exception("Use of multiple file services is not supported for now."); - if not relativePath.startswith("/"): relativePath = "/" + relativePath; @@ -479,7 +476,7 @@ def move(fileService, rootVolume, userVolume, relativePath, destinationFileServi url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName; headers = {'X-Auth-Token': token} - jsonDict = {'destinationPath': destinationRelativePath, 'destinationRootFolder': destinationRootVolume, 'destinationUserVolume':destinationUserVolume, 'destinationOwner': destinationuserVolumeOwner}; + jsonDict = {'destinationPath': destinationRelativePath, 'destinationRootFolder': destinationRootVolume, 'destinationUserVolume':destinationUserVolume, 'destinationOwner': destinationuserVolumeOwner, 'destinationFileService': destinationFileService}; data = json.dumps(jsonDict).encode() res = requests.put(url, stream=True, headers=headers, data=data) From 2eda1fae1bee2bb7babe448b053b34034f91d16f Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Sat, 17 Feb 2018 17:30:44 -0500 Subject: [PATCH 26/79] Fixed typo Fixed typo --- py2/SciServer/Files.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py2/SciServer/Files.py b/py2/SciServer/Files.py index 2343b72..ffa66a0 100644 --- a/py2/SciServer/Files.py +++ b/py2/SciServer/Files.py @@ -472,11 +472,11 @@ def move(fileService, rootVolume, userVolume, relativePath, destinationFileServi userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; if destinationUserVolumeOwner is None: - destinationuserVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + destinationUserVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName; headers = {'X-Auth-Token': token} - jsonDict = {'destinationPath': destinationRelativePath, 'destinationRootFolder': destinationRootVolume, 'destinationUserVolume':destinationUserVolume, 'destinationOwner': destinationuserVolumeOwner, 'destinationFileService': destinationFileService}; + jsonDict = {'destinationPath': destinationRelativePath, 'destinationRootFolder': destinationRootVolume, 'destinationUserVolume':destinationUserVolume, 'destinationOwner': destinationUserVolumeOwner, 'destinationFileService': destinationFileService}; data = json.dumps(jsonDict).encode() res = requests.put(url, stream=True, headers=headers, data=data) From b5ee73f293b87992c587540f2b44995d3c1e3d1a Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Sat, 17 Feb 2018 17:31:48 -0500 Subject: [PATCH 27/79] Fixed typo Fixed typo --- py3/SciServer/Files.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py3/SciServer/Files.py b/py3/SciServer/Files.py index 2343b72..ffa66a0 100644 --- a/py3/SciServer/Files.py +++ b/py3/SciServer/Files.py @@ -472,11 +472,11 @@ def move(fileService, rootVolume, userVolume, relativePath, destinationFileServi userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; if destinationUserVolumeOwner is None: - destinationuserVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + destinationUserVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName; headers = {'X-Auth-Token': token} - jsonDict = {'destinationPath': destinationRelativePath, 'destinationRootFolder': destinationRootVolume, 'destinationUserVolume':destinationUserVolume, 'destinationOwner': destinationuserVolumeOwner, 'destinationFileService': destinationFileService}; + jsonDict = {'destinationPath': destinationRelativePath, 'destinationRootFolder': destinationRootVolume, 'destinationUserVolume':destinationUserVolume, 'destinationOwner': destinationUserVolumeOwner, 'destinationFileService': destinationFileService}; data = json.dumps(jsonDict).encode() res = requests.put(url, stream=True, headers=headers, data=data) From d72b0d590e982a7edbb4b527e0cea797a1c318e6 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Sun, 18 Feb 2018 02:07:30 -0500 Subject: [PATCH 28/79] removed csv file ending in target location --- py2/SciServer/Jobs.py | 2 +- py3/SciServer/Jobs.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py index 86fbe31..4d7b0a3 100644 --- a/py2/SciServer/Jobs.py +++ b/py2/SciServer/Jobs.py @@ -547,7 +547,7 @@ def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = Non if resultsName is None: resultsName = 'results' - targets = [{'location':resultsName+'.csv', 'type':'FILE_CSV', 'resultNumber':1}] + targets = [{'location':resultsName, 'type':'FILE_CSV', 'resultNumber':1}] rdbDomainId = rdbComputeDomain.get('id'); dockerJobModel = { diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py index bf4ea4d..3861c56 100644 --- a/py3/SciServer/Jobs.py +++ b/py3/SciServer/Jobs.py @@ -547,7 +547,7 @@ def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = Non if resultsName is None: resultsName = 'results' - targets = [{'location':resultsName+'.csv', 'type':'FILE_CSV', 'resultNumber':1}] + targets = [{'location':resultsName, 'type':'FILE_CSV', 'resultNumber':1}] rdbDomainId = rdbComputeDomain.get('id'); dockerJobModel = { From ba3ccb4cf6bab6d0808797442eca9216fa058a58 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Sun, 18 Feb 2018 07:21:12 -0500 Subject: [PATCH 29/79] made rdb job to support outputs of multiple sql statements --- py2/SciServer/Jobs.py | 23 ++++++++++++++++++----- py3/SciServer/Jobs.py | 23 ++++++++++++++++++----- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py index 4d7b0a3..1d6b3a4 100644 --- a/py2/SciServer/Jobs.py +++ b/py2/SciServer/Jobs.py @@ -508,13 +508,13 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN else: raise Exception("User token is not defined. First log into SciServer.") -def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = None, resultsName=None, jobAlias = ""): +def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = None, resultsName='queryResults', jobAlias = ""): """ Submits a sql query for execution (as an asynchronous job) inside a relational database (RDB) compute domain. :param sqlQuery: sql query (string) :param rdbComputeDomain: object (dictionary) that defines a relational database (RDB) compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). :param databaseContextName: database context name (string) on which the sql query is executed. - :param resultsName: name (string) of the table or file containing the query result. + :param resultsName: name (string) of the table or file (without file type ending) that contains the query result. In case the sql query has multiple statements, should be set to a list of names (e.g., ['result1','result2']). :param jobAlias: alias (string) of job, defined by the user. :return: a dictionary containing the definition of the submitted job. :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. @@ -544,10 +544,23 @@ def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = Non else: raise Exception("rbdComputeDomain has no database contexts available for the user."); - if resultsName is None: - resultsName = 'results' + targets = []; + if type(resultsName) == str: + targets.append({'location': resultsName, 'type': 'FILE_CSV', 'resultNumber': 1}); + elif type(resultsName) == list: + if len(set(resultsName)) != len(resultsName): + raise Exception("Elements of parameter 'resultsName' must be unique"); + + for i in range(len(resultsName)): + if type(resultsName[i]) == str: + targets.append({'location': resultsName[i], 'type': 'FILE_CSV', 'resultNumber': i+1}); + else: + raise Exception("Elements of array 'resultsName' are not strings"); + + else: + raise Exception("Type of parameter 'resultsName' is not supported"); + - targets = [{'location':resultsName, 'type':'FILE_CSV', 'resultNumber':1}] rdbDomainId = rdbComputeDomain.get('id'); dockerJobModel = { diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py index 3861c56..9e80a3a 100644 --- a/py3/SciServer/Jobs.py +++ b/py3/SciServer/Jobs.py @@ -508,13 +508,13 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN else: raise Exception("User token is not defined. First log into SciServer.") -def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = None, resultsName=None, jobAlias = ""): +def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = None, resultsName='queryResults', jobAlias = ""): """ Submits a sql query for execution (as an asynchronous job) inside a relational database (RDB) compute domain. :param sqlQuery: sql query (string) :param rdbComputeDomain: object (dictionary) that defines a relational database (RDB) compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). :param databaseContextName: database context name (string) on which the sql query is executed. - :param resultsName: name (string) of the table or file containing the query result. + :param resultsName: name (string) of the table or file (without file type ending) that contains the query result. In case the sql query has multiple statements, should be set to a list of names (e.g., ['result1','result2']). :param jobAlias: alias (string) of job, defined by the user. :return: a dictionary containing the definition of the submitted job. :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. @@ -544,10 +544,23 @@ def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = Non else: raise Exception("rbdComputeDomain has no database contexts available for the user."); - if resultsName is None: - resultsName = 'results' + targets = []; + if type(resultsName) == str: + targets.append({'location': resultsName, 'type': 'FILE_CSV', 'resultNumber': 1}); + elif type(resultsName) == list: + if len(set(resultsName)) != len(resultsName): + raise Exception("Elements of parameter 'resultsName' must be unique"); + + for i in range(len(resultsName)): + if type(resultsName[i]) == str: + targets.append({'location': resultsName[i], 'type': 'FILE_CSV', 'resultNumber': i+1}); + else: + raise Exception("Elements of array 'resultsName' are not strings"); + + else: + raise Exception("Type of parameter 'resultsName' is not supported"); + - targets = [{'location':resultsName, 'type':'FILE_CSV', 'resultNumber':1}] rdbDomainId = rdbComputeDomain.get('id'); dockerJobModel = { From a4d2dde079ee45de6f02c6a5191807a9eac4f6c8 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Wed, 14 Mar 2018 09:59:14 -0400 Subject: [PATCH 30/79] changed way of referring to a file or folder (unique path instead of separate path elements). Improved descriptions --- py2/SciServer/Files.py | 274 +++++++++++++++++++++++------------------ py3/SciServer/Files.py | 274 +++++++++++++++++++++++------------------ 2 files changed, 312 insertions(+), 236 deletions(-) diff --git a/py2/SciServer/Files.py b/py2/SciServer/Files.py index ffa66a0..b12142e 100644 --- a/py2/SciServer/Files.py +++ b/py2/SciServer/Files.py @@ -59,10 +59,10 @@ def getFileServices(verbose=True): def getFileServicesNames(fileServices=None, verbose=True): """ - Returns the names of the fileServices available to the user. - :param fileServices: a list of FileService objects (dictionaries), as returned by Jobs.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. + Returns the names and description of the fileServices available to the user.l + :param fileServices: a list of FileService objects (dictionaries), as returned by Files.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). - :return: an array of strings, each being the name of a file service available to the user. + :return: an array of dicts, where each dict has the name and description of a file service available to the user. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. :example: fileServiceNames = Files.getFileServicesNames(); .. seealso:: Files.getFileServices @@ -72,16 +72,10 @@ def getFileServicesNames(fileServices=None, verbose=True): fileServices = getFileServices(verbose); fileServiceNames = []; - if fileServices.__len__() > 0: - for fileService in fileServices: - fileServiceNames.append(fileService.get('name')) + for fileService in fileServices: + fileServiceNames.append({"name":fileService.get('name'),"description":fileService.get('description')}) - return fileServiceNames; - else: - #raise Exception("There are no FileServices available for the user."); - if verbose: - warnings.warn("There are no FileServices available for the user.", Warning, stacklevel=2) - return fileServiceNames + return fileServiceNames @@ -90,8 +84,8 @@ def getFileServicesNames(fileServices=None, verbose=True): def getFileServiceFromName(fileServiceName, fileServices=None, verbose=True): """ Returns a FileService object, given its registered name. - :param fileServiceName: name of the FileService, as shown within the results of Jobs.getFileServices() - :param fileServices: a list of FileService objects (dictionaries), as returned by Jobs.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. + :param fileServiceName: name of the FileService, as shown within the results of Files.getFileServices() + :param fileServices: a list of FileService objects (dictionaries), as returned by Files.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). :return a FileService object (dictionary) that defines a FileService. A list of these kind of objects available to the user is returned by the function Jobs.getFileServices(). If no fileService can be found, then returns None. :raises Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. @@ -110,12 +104,10 @@ def getFileServiceFromName(fileServiceName, fileServices=None, verbose=True): if fileServiceName == fileService.get('name'): return fileService; - #raise Exception("FileService of name '" + fileServiceName + "' is not available or does not exist."); if verbose: - warnings.warn("FileService of name '" + fileServiceName + "' is not available or does not exist.", Warning, stacklevel=2) + warnings.warn("FileService of name '" + fileServiceName + "' is not available to the user or does not exist.", Warning, stacklevel=2) return None else: - #raise Exception("There are no FileServices available for the user."); if verbose: warnings.warn("There are no FileServices available for the user.", Warning, stacklevel=2) return None @@ -147,25 +139,84 @@ def __getFileServiceAPIUrl(fileService): return url -def getRootVolumes(fileService): +def getRootVolumesInfo(fileService, verbose=True): """ - Gets the definitions the RootVolumes available in a particular FileService, and of the file services that a user is able to access. + Gets the names and descriptions of root volumes available to the user in a particular FileService. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :return: list of dictionaries, where each dictionary contains the definition of a FileService that a user is able to access. + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: list of dictionaries, where each dictionary contains the name and description of a root volume. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. - :example: fileServices = Files.getFileServices(); rootVolumes = getRootVolumes(fileServices[0]) - .. seealso:: Files.getFileServiceFromName + :example: fileServices = Files.getFileServices(); rootVolumesInfo = getRootVolumesInfo(fileServices[0]) + .. seealso:: Files.getUserVolumesInfo + """ + if type(fileService) == str: + fileService = getFileServiceFromName(fileService, verbose=verbose) + + rootVolumes = [] + for rootVolume in fileService.get("rootVolumes"): + rootVolumes.append({"rootVolumeName":rootVolume.get("name"), "rootVolumeDescription":rootVolume.get("description")}) + + return rootVolumes + + + +def getUserVolumesInfo(fileService, rootVolumeName = None, verbose=True): + """ + Gets the names definitions the RootVolumes available in a particular FileService, and of the file services that a user is able to access. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolumeName: name of root Volume (string) for which the user volumes are fetched. If set to None, then user volumes in all root folders are fetched. + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: list of dictionaries, where each dictionary contains the name and description of a root volumes that a user is able to access. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServices = Files.getFileServices(); rootVolumeNames = Files.getUserVolumesInfo(fileServices[0]) + .. seealso:: Files.getRootVolumes, + """ + if type(fileService) == str: + fileService = getFileServiceFromName(fileService, verbose=verbose) + + userVolumes = []; + for rootVolume in fileService.get("rootVolumes"): + for userVolume in rootVolume.get('userVolumes'): + if rootVolumeName is not None: + if rootVolume.get('name') == rootVolumeName: + userVolumes.append({"userVolumeName": userVolume.get('name'), "userVolumeDescription": userVolume.get('description'), "rootVolumeName":rootVolume.get("name"), "rootVolumeDescription":rootVolume.get("description")}) + + else: + userVolumes.append({"userVolumeName": userVolume.get('name'), "userVolumeDescription": userVolume.get('description'),"rootVolumeName": rootVolume.get("name"), "rootVolumeDescription": rootVolume.get("description")}) + + + return userVolumes + + + +def splitPath(path): + """ + Splits a path of the form rootVolume/userVolumeOwner/userVolume/relativePath/... into its 4 components: rootVolume, userVolumeOwner, userVolume, and relativePath. + :param path: file system path (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/relativePath... + :return: a tuple containing the four components: (rootVolume, userVolumeOwner, userVolume, relativePath) + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.createUserVolume(fileServices[0], "volumes","newUserVolume"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName """ - return fileService.get("rootVolumes"); + if path.startswith("/"): + path = path[1:] + + path = path.split("/") + if len(path) < 3: + raise Exception("path variable does not conform with the format 'rootVolume/userVolumeOwner/userVolume/relativePath...'") + + rootVolume = path[0] + userVolumeOwner = path[1] + userVolume = path[2] + relativePath = "/".join(path[3:]) + return (rootVolume, userVolumeOwner, userVolume, relativePath) -def createUserVolume(fileService, rootVolume, userVolume, userVolumeOwner=None, quiet=True): +def createUserVolume(fileService, path, quiet=True): """ Create a user volume. - :param fileService: name of fileService (string), or object (dictionary) that defines the file service that contains the root volume. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param rootVolume: name of root volume (string) that contains the user volume to be created. A list of user volumes is given by getRootVolumes(fileService) - :param userVolume: name of user volume (string) to be created, one level below the root volume level. - :param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume. + :param fileService: name of fileService (string), or object (dictionary) that defines the file service that contains the user volume. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: path (in the remote file service) to the user volume (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume :param quiet: if set to False, will throw an error if the User Volume already exists. If True, won't throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); Files.createUserVolume(fileServices[0], "volumes","newUserVolume"); @@ -179,8 +230,10 @@ def createUserVolume(fileService, rootVolume, userVolume, userVolumeOwner=None, else: taskName = "SciScript-Python.Files.createUserVolume" - if userVolumeOwner is None: - userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); url = __getFileServiceAPIUrl(fileService) + "api/volume/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?quiet="+str(quiet) + "&TaskName="+taskName; @@ -190,18 +243,16 @@ def createUserVolume(fileService, rootVolume, userVolume, userVolumeOwner=None, if res.status_code >= 200 and res.status_code < 300: pass; else: - raise Exception("Error when creating user volume '" + userVolume + "' in root volume '" + rootVolume + "' in file service named '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when creating user volume '" + str(path) + "' in file service '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") -def deleteUserVolume(fileService, rootVolume, userVolume, userVolumeOwner=None, quiet=True): +def deleteUserVolume(fileService, path, quiet=True): """ Delete a user volume. :param fileService: name of fileService (string), or object (dictionary) that defines the file service that contains the root volume. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param rootVolume: root volume name (string) that contains the user volume to be deleted. A list of user volumes is given by getRootVolumes(fileService) - :param userVolume: user volume name (string) to be deleted. - :param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume. + :param path: path (in the remote file service) to the user volume (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume :param quiet: If set to False, it will throw an error if user volume does not exists. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); Files.deleteUserVolume("volumes","newUserVolume",fileServices[0]); @@ -215,8 +266,10 @@ def deleteUserVolume(fileService, rootVolume, userVolume, userVolumeOwner=None, else: taskName = "SciScript-Python.Files.createUserVolume" - if userVolumeOwner is None: - userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) url = __getFileServiceAPIUrl(fileService) + "api/volume/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?quiet="+str(quiet)+"&TaskName="+taskName; @@ -226,19 +279,16 @@ def deleteUserVolume(fileService, rootVolume, userVolume, userVolumeOwner=None, if res.status_code >= 200 and res.status_code < 300: pass else: - raise Exception("Error when deleting user volume '" + userVolume + "' in root volume '" + rootVolume + "' in file service named '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when deleting user volume '" + str(path) + "' in file service '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") -def createDir(fileService, rootVolume, userVolume, relativePath, userVolumeOwner=None, quiet=True): +def createDir(fileService, path, quiet=True): """ Create a directory. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. - :param userVolume: user volume name (string) that contains the relativePath. - :param relativePath: path (string) to the directory, relative to the userVolume level. E.g: path="/newDirectory" - :param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume. + :param path: path (in the remote file service) to the directory (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/directory :param quiet: If set to False, it will throw an error if the directory already exists. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); Files.createDir(fileServices[0], "myRootVolume","myUserVolume", "myNewDir"); @@ -252,13 +302,19 @@ def createDir(fileService, rootVolume, userVolume, relativePath, userVolumeOwner else: taskName = "SciScript-Python.Files.createDir" + + (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); + if not relativePath.startswith("/"): relativePath = "/" + relativePath; if userVolumeOwner is None: userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - url = __getFileServiceAPIUrl(fileService) + "api/folder/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName; + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + url = __getFileServiceAPIUrl(fileService) + "api/folder/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName; headers = {'X-Auth-Token': token} res = requests.put(url, headers=headers) @@ -266,18 +322,16 @@ def createDir(fileService, rootVolume, userVolume, relativePath, userVolumeOwner if res.status_code >= 200 and res.status_code < 300: pass; else: - raise Exception("Error when creating directory '" + relativePath + "' in user volume '" + userVolume + "' in root volume '" + rootVolume + "' in file service named '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when creating directory '" + str(path) + "' in file service '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") -def upload(fileService, rootVolume, userVolume, relativePath, data="", localFilePath=None, userVolumeOwner=None, quiet=True): +def upload(fileService, path, data="", localFilePath=None, quiet=True): """ Uploads data or a local file into a path defined in the file system. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. - :param userVolume: user volume name (string) that contains the relativePath. - :param relativePath: destination path (string) of the file to be uploaded, relative to the user volume level. E.g: path="/newDirectory/myNewFile.txt". + :param path: path (in the remote file service) to the destination file (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/destinationFile.txt :param data: string containing data to be uploaded, in case localFilePath is not set. :param localFilePath: path to a local file to be uploaded (string), :param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume. @@ -294,13 +348,12 @@ def upload(fileService, rootVolume, userVolume, relativePath, data="", localFile else: taskName = "SciScript-Python.Files.UploadFile" - if not relativePath.startswith("/"): - relativePath = "/" + relativePath; + (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); - if userVolumeOwner is None: - userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) - url = __getFileServiceAPIUrl(fileService) + "api/file/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName + url = __getFileServiceAPIUrl(fileService) + "api/file/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName headers = {'X-Auth-Token': token} @@ -316,18 +369,16 @@ def upload(fileService, rootVolume, userVolume, relativePath, data="", localFile if res.status_code >= 200 and res.status_code < 300: pass; else: - raise Exception("Error when uploading file '" + relativePath + "' in user volume '" + userVolume + "' in root volume '" + rootVolume + "' in file service named '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when uploading file to '" + str(path) + "' in file service '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") -def download(fileService, rootVolume, userVolume, relativePath, localFilePath=None, format="txt", userVolumeOwner=None, quiet=True): +def download(fileService, path, localFilePath=None, format="txt", quiet=True): """ Downloads a file from the remote file system into the local file system, or returns the file content as an object in several formats. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. - :param userVolume: user volume name (string) that contains the relativePath. - :param relativePath: remote path (string) of the file to be downloaded, relative to the user volume level. E.g: path="/newDirectory/myNewFile.txt" + :param path: String defining the path (in the remote file service) of the file to be downloaded, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/fileToBeDownloaded.txt. :param localFilePath: local destination path of the file to be downloaded. If set to None, then an object of format 'format' will be returned. :param format: name (string) of the returned object's type (if localFilePath is not defined). This parameter can be "StringIO" (io.StringIO object containing readable text), "BytesIO" (io.BytesIO object containing readable binary data), "response" ( the HTTP response as an object of class requests.Response) or "txt" (a text string). If the parameter 'localFilePath' is defined, then the 'format' parameter is not used and the file is downloaded to the local file system instead. :param userVolumeOwner: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. @@ -340,29 +391,27 @@ def download(fileService, rootVolume, userVolume, relativePath, localFilePath=No token = Authentication.getToken() if token is not None and token != "": + (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) if localFilePath is not None: if os.path.isfile(localFilePath) and not quiet: - raise Exception("Error when downloading from remote File Syatem the file " + str(relativePath) + ".\nLocal file '" + localFilePath + "' already exists."); + raise Exception("Error when downloading '" + str(path) + "' from file service '" + fileService.get("name") + "'. Local file '" + localFilePath + "' already exists."); if Config.isSciServerComputeEnvironment(): taskName = "Compute.SciScript-Python.Files.DownloadFile" else: taskName = "SciScript-Python.Files.DownloadFile" - if not relativePath.startswith("/"): - relativePath = "/" + relativePath; - - if userVolumeOwner is None: - userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - - url = __getFileServiceAPIUrl(fileService) + "api/file/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?TaskName=" + taskName; + url = __getFileServiceAPIUrl(fileService) + "api/file/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?TaskName=" + taskName; headers = {'X-Auth-Token': token} res = requests.get(url, stream=True, headers=headers) if res.status_code < 200 or res.status_code >= 300: - raise Exception("Error when downloading from remote File Syatem the file " + str(relativePath) + ".\nHttp Response from FileSystem API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when downloading '" + str(path) + "' from file service '" + fileService.get("name") + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); if localFilePath is not None and localFilePath != "": @@ -393,16 +442,13 @@ def download(fileService, rootVolume, userVolume, relativePath, localFilePath=No -def dirList(fileService, rootVolume, userVolume, relativePath, level=1, options='', userVolumeOwner=None): +def dirList(fileService, path, level=1, options=''): """ Lists the contents of a directory. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. - :param userVolume: user volume name (string) that contains the relativePath. - :param relativePath: remote path (string) of the directory, relative to the user volume level. E.g: path="/newDirectory/myNewFile.txt" + :param path: String defining the path (in the remote file service) of the directory to be listed, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/directoryToBeListed. :param level: amount (int) of listed directory levels that are below or at the same level to that of the relativePath. :param options: string of file filtering options. - :param userVolumeOwner: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. :return: dictionary containing the directory listing. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); dirs = Files.dirList("/","persistent","myUserName", fileServices[0], 2); @@ -416,13 +462,12 @@ def dirList(fileService, rootVolume, userVolume, relativePath, level=1, options= else: taskName = "SciScript-Python.Files.dirList" - if userVolumeOwner is None: - userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); - if not relativePath.startswith("/"): - path = "/" + relativePath; + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) - url = __getFileServiceAPIUrl(fileService) + "api/jsontree/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + path + "?options=" + options + "&level=" + str(level) + "&TaskName=" + taskName; + url = __getFileServiceAPIUrl(fileService) + "api/jsontree/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?options=" + options + "&level=" + str(level) + "&TaskName=" + taskName; headers = {'X-Auth-Token': token} res = requests.get(url, headers=headers) @@ -430,26 +475,20 @@ def dirList(fileService, rootVolume, userVolume, relativePath, level=1, options= if res.status_code >= 200 and res.status_code < 300: return json.loads(res.content.decode()); else: - raise Exception("Error when listing directory.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when listing contents of '" + str(path) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") -def move(fileService, rootVolume, userVolume, relativePath, destinationFileService, destinationRootVolume, destinationUserVolume, destinationRelativePath, replaceExisting=True, doCopy=True, userVolumeOwner=None, destinationUserVolumeOwner=None): +def move(fileService, path, destinationFileService, destinationPath, replaceExisting=True, doCopy=True): """ Moves or copies a file or folder. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileService object. - :param userVolume: user volume name (string) that contains the relativePath. - :param relativePath: remote path (string) of the file to be moved/copied, relative to the user volume level. E.g: path="/newDirectory/myNewFile.txt" + :param path: String defining the origin path (in the remote fileService) of the file or directory to be copied/moved, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/fileToBeMoved.txt. :param destinationFileService: name of fileService (string), or object (dictionary) that defines a destination file service (where the file is moved/copied into). A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param destinationRootVolume: root volume name (string) that contains the userVolume where the file is moved/copied into. A list of root volumes is contained by the fileService object. - :param destinationUserVolume: user volume name (string) that contains the relativePath where the file is moved/copied into. - :param destinationRelativePath: remote path (string) of the destination file, relative to the user volume level. E.g: path="/newDirectory2/myNewFile.txt" + :param destinationRelativePath: String defining the destination path (in the remote destinationFileService) of the file or directory to be copied/moved, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/recentlyMovedFile.txt. :param replaceExisting: If set to False, it will throw an error if the file already exists, If set to True, it will not throw and eeror in that case. :param doCopy: if set to True, then it will copy the file or folder. If set to False, then the file or folder will be moved. - :param userVolumeOwner: name (string) of owner of the user volume. Can be left undefined if requester is the owner of the volume. - :param destinationUserVolumeOwner: name (string) of owner of the destination user volume. Can be left undefined if requester is the owner of it. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); isDownloaded = Files.upload("/myUploadedFile.txt","persistent","myUserName", fileServices[0], localFilePath="/myDownloadedFile.txt"); .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.upload, Files.dirList @@ -458,15 +497,12 @@ def move(fileService, rootVolume, userVolume, relativePath, destinationFileServi if token is not None and token != "": if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.Files.DownloadFile" + taskName = "Compute.SciScript-Python.Files.Move" else: - taskName = "SciScript-Python.Files.DownloadFile" + taskName = "SciScript-Python.Files.Move" - if not relativePath.startswith("/"): - relativePath = "/" + relativePath; - - if not destinationRelativePath.startswith("/"): - destinationRelativePath = "/" + destinationRelativePath; + (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); + (destinationRootVolume, destinationUserVolumeOwner, destinationUserVolume, destinationRelativePath) = splitPath(destinationPath); if userVolumeOwner is None: userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; @@ -474,28 +510,36 @@ def move(fileService, rootVolume, userVolume, relativePath, destinationFileServi if destinationUserVolumeOwner is None: destinationUserVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName; - headers = {'X-Auth-Token': token} - jsonDict = {'destinationPath': destinationRelativePath, 'destinationRootFolder': destinationRootVolume, 'destinationUserVolume':destinationUserVolume, 'destinationOwner': destinationUserVolumeOwner, 'destinationFileService': destinationFileService}; + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + if type(destinationFileService) == str: + destinationFileService = getFileServiceFromName(destinationFileService) + + url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName; + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + + if type(destinationFileService) == dict: + destinationFileService = destinationFileService['name'] + + + jsonDict = {'destinationPath': destinationRelativePath, 'destinationRootVolume': destinationRootVolume, 'destinationUserVolume':destinationUserVolume, 'destinationOwnerName': destinationUserVolumeOwner, 'destinationFileService': destinationFileService}; data = json.dumps(jsonDict).encode() - res = requests.put(url, stream=True, headers=headers, data=data) + res = requests.put(url, stream=True, headers=headers, json=jsonDict) if res.status_code < 200 or res.status_code >= 300: - raise Exception("Error when downloading from remote File System the file " + str(relativePath) + ".\nHttp Response from FileSystem API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when moving '" + str(path) + "' in file service '" + fileService.get("name") + "' to '" + str(path) + "' in file service '" + destinationFileService.get("name)") + "'. \nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") -def delete(fileService, rootVolume, userVolume, relativePath, userVolumeOwner=None, quiet=True): +def delete(fileService, path, quiet=True): """ Deletes a directory or file in the File System. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. - :param userVolume: user volume name (string) that contains the relativePath. - :param relativePath: remote path (string) of the directory or file to be deleted, relative to the user volume level. E.g: path="/newDirectory/myNewFile.txt" - :param userVolumeOwner: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. + :param path: String defining the path (in the remote fileService) of the file or directory to be deleted, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/fileToBeDeleted.txt. :param quiet: If set to False, it will throw an error if the file does not exist. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]); @@ -509,13 +553,9 @@ def delete(fileService, rootVolume, userVolume, relativePath, userVolumeOwner=No else: taskName = "SciScript-Python.Files.delete" - if not relativePath.startswith("/"): - relativePath = "/" + relativePath; - - if userVolumeOwner is None: - userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); - url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName + url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName headers = {'X-Auth-Token': token} res = requests.delete(url, headers=headers) @@ -523,17 +563,17 @@ def delete(fileService, rootVolume, userVolume, relativePath, userVolumeOwner=No if res.status_code >= 200 and res.status_code < 300: pass; else: - raise Exception("Error when deleting " + relativePath + ".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when deleting '" + str(path) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") -def shareUserVolume(fileService, rootVolume, userVolume, sharedWith, allowedActions, type="USER", userVolumeOwner=None): +def shareUserVolume(fileService, path, sharedWith, allowedActions, type="USER"): """ Deletes a directory or file in the File System. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. + :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileService object. :param userVolume: user volume name (string) that contains the relativePath. :param sharedWith: name (string) of user or group that the userVolume is shared with. :param allowedActions: array of strings defining actions the user or group is allowed to do with respect to the shared user volume. E.g.: ["read","write","grant","delete"]. The "grant" action means that the user or group can also share the user volume with another user or group. The "delete" action meand ability to delete the user volume (use with care). @@ -551,8 +591,7 @@ def shareUserVolume(fileService, rootVolume, userVolume, sharedWith, allowedActi else: taskName = "SciScript-Python.Files.ShareUserVolume" - if userVolumeOwner is None: - userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); data = [{'name': sharedWith, 'type':type, 'allowedActions': allowedActions }] @@ -566,7 +605,6 @@ def shareUserVolume(fileService, rootVolume, userVolume, sharedWith, allowedActi if res.status_code >= 200 and res.status_code < 300: pass; else: - raise Exception("Error when sharing userVolume '" + userVolume + "' in rootVolume '" + rootVolume + "' with '" + sharedWith + "' in file service named '" + fileService.get( - 'name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()) + raise Exception("Error when sharing userVolume '" + str(path) + "' in file service '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()) else: raise Exception("User token is not defined. First log into SciServer.") diff --git a/py3/SciServer/Files.py b/py3/SciServer/Files.py index ffa66a0..b12142e 100644 --- a/py3/SciServer/Files.py +++ b/py3/SciServer/Files.py @@ -59,10 +59,10 @@ def getFileServices(verbose=True): def getFileServicesNames(fileServices=None, verbose=True): """ - Returns the names of the fileServices available to the user. - :param fileServices: a list of FileService objects (dictionaries), as returned by Jobs.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. + Returns the names and description of the fileServices available to the user.l + :param fileServices: a list of FileService objects (dictionaries), as returned by Files.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). - :return: an array of strings, each being the name of a file service available to the user. + :return: an array of dicts, where each dict has the name and description of a file service available to the user. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. :example: fileServiceNames = Files.getFileServicesNames(); .. seealso:: Files.getFileServices @@ -72,16 +72,10 @@ def getFileServicesNames(fileServices=None, verbose=True): fileServices = getFileServices(verbose); fileServiceNames = []; - if fileServices.__len__() > 0: - for fileService in fileServices: - fileServiceNames.append(fileService.get('name')) + for fileService in fileServices: + fileServiceNames.append({"name":fileService.get('name'),"description":fileService.get('description')}) - return fileServiceNames; - else: - #raise Exception("There are no FileServices available for the user."); - if verbose: - warnings.warn("There are no FileServices available for the user.", Warning, stacklevel=2) - return fileServiceNames + return fileServiceNames @@ -90,8 +84,8 @@ def getFileServicesNames(fileServices=None, verbose=True): def getFileServiceFromName(fileServiceName, fileServices=None, verbose=True): """ Returns a FileService object, given its registered name. - :param fileServiceName: name of the FileService, as shown within the results of Jobs.getFileServices() - :param fileServices: a list of FileService objects (dictionaries), as returned by Jobs.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. + :param fileServiceName: name of the FileService, as shown within the results of Files.getFileServices() + :param fileServices: a list of FileService objects (dictionaries), as returned by Files.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). :return a FileService object (dictionary) that defines a FileService. A list of these kind of objects available to the user is returned by the function Jobs.getFileServices(). If no fileService can be found, then returns None. :raises Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. @@ -110,12 +104,10 @@ def getFileServiceFromName(fileServiceName, fileServices=None, verbose=True): if fileServiceName == fileService.get('name'): return fileService; - #raise Exception("FileService of name '" + fileServiceName + "' is not available or does not exist."); if verbose: - warnings.warn("FileService of name '" + fileServiceName + "' is not available or does not exist.", Warning, stacklevel=2) + warnings.warn("FileService of name '" + fileServiceName + "' is not available to the user or does not exist.", Warning, stacklevel=2) return None else: - #raise Exception("There are no FileServices available for the user."); if verbose: warnings.warn("There are no FileServices available for the user.", Warning, stacklevel=2) return None @@ -147,25 +139,84 @@ def __getFileServiceAPIUrl(fileService): return url -def getRootVolumes(fileService): +def getRootVolumesInfo(fileService, verbose=True): """ - Gets the definitions the RootVolumes available in a particular FileService, and of the file services that a user is able to access. + Gets the names and descriptions of root volumes available to the user in a particular FileService. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :return: list of dictionaries, where each dictionary contains the definition of a FileService that a user is able to access. + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: list of dictionaries, where each dictionary contains the name and description of a root volume. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. - :example: fileServices = Files.getFileServices(); rootVolumes = getRootVolumes(fileServices[0]) - .. seealso:: Files.getFileServiceFromName + :example: fileServices = Files.getFileServices(); rootVolumesInfo = getRootVolumesInfo(fileServices[0]) + .. seealso:: Files.getUserVolumesInfo + """ + if type(fileService) == str: + fileService = getFileServiceFromName(fileService, verbose=verbose) + + rootVolumes = [] + for rootVolume in fileService.get("rootVolumes"): + rootVolumes.append({"rootVolumeName":rootVolume.get("name"), "rootVolumeDescription":rootVolume.get("description")}) + + return rootVolumes + + + +def getUserVolumesInfo(fileService, rootVolumeName = None, verbose=True): + """ + Gets the names definitions the RootVolumes available in a particular FileService, and of the file services that a user is able to access. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolumeName: name of root Volume (string) for which the user volumes are fetched. If set to None, then user volumes in all root folders are fetched. + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: list of dictionaries, where each dictionary contains the name and description of a root volumes that a user is able to access. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServices = Files.getFileServices(); rootVolumeNames = Files.getUserVolumesInfo(fileServices[0]) + .. seealso:: Files.getRootVolumes, + """ + if type(fileService) == str: + fileService = getFileServiceFromName(fileService, verbose=verbose) + + userVolumes = []; + for rootVolume in fileService.get("rootVolumes"): + for userVolume in rootVolume.get('userVolumes'): + if rootVolumeName is not None: + if rootVolume.get('name') == rootVolumeName: + userVolumes.append({"userVolumeName": userVolume.get('name'), "userVolumeDescription": userVolume.get('description'), "rootVolumeName":rootVolume.get("name"), "rootVolumeDescription":rootVolume.get("description")}) + + else: + userVolumes.append({"userVolumeName": userVolume.get('name'), "userVolumeDescription": userVolume.get('description'),"rootVolumeName": rootVolume.get("name"), "rootVolumeDescription": rootVolume.get("description")}) + + + return userVolumes + + + +def splitPath(path): + """ + Splits a path of the form rootVolume/userVolumeOwner/userVolume/relativePath/... into its 4 components: rootVolume, userVolumeOwner, userVolume, and relativePath. + :param path: file system path (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/relativePath... + :return: a tuple containing the four components: (rootVolume, userVolumeOwner, userVolume, relativePath) + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.createUserVolume(fileServices[0], "volumes","newUserVolume"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName """ - return fileService.get("rootVolumes"); + if path.startswith("/"): + path = path[1:] + + path = path.split("/") + if len(path) < 3: + raise Exception("path variable does not conform with the format 'rootVolume/userVolumeOwner/userVolume/relativePath...'") + + rootVolume = path[0] + userVolumeOwner = path[1] + userVolume = path[2] + relativePath = "/".join(path[3:]) + return (rootVolume, userVolumeOwner, userVolume, relativePath) -def createUserVolume(fileService, rootVolume, userVolume, userVolumeOwner=None, quiet=True): +def createUserVolume(fileService, path, quiet=True): """ Create a user volume. - :param fileService: name of fileService (string), or object (dictionary) that defines the file service that contains the root volume. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param rootVolume: name of root volume (string) that contains the user volume to be created. A list of user volumes is given by getRootVolumes(fileService) - :param userVolume: name of user volume (string) to be created, one level below the root volume level. - :param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume. + :param fileService: name of fileService (string), or object (dictionary) that defines the file service that contains the user volume. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: path (in the remote file service) to the user volume (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume :param quiet: if set to False, will throw an error if the User Volume already exists. If True, won't throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); Files.createUserVolume(fileServices[0], "volumes","newUserVolume"); @@ -179,8 +230,10 @@ def createUserVolume(fileService, rootVolume, userVolume, userVolumeOwner=None, else: taskName = "SciScript-Python.Files.createUserVolume" - if userVolumeOwner is None: - userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); url = __getFileServiceAPIUrl(fileService) + "api/volume/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?quiet="+str(quiet) + "&TaskName="+taskName; @@ -190,18 +243,16 @@ def createUserVolume(fileService, rootVolume, userVolume, userVolumeOwner=None, if res.status_code >= 200 and res.status_code < 300: pass; else: - raise Exception("Error when creating user volume '" + userVolume + "' in root volume '" + rootVolume + "' in file service named '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when creating user volume '" + str(path) + "' in file service '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") -def deleteUserVolume(fileService, rootVolume, userVolume, userVolumeOwner=None, quiet=True): +def deleteUserVolume(fileService, path, quiet=True): """ Delete a user volume. :param fileService: name of fileService (string), or object (dictionary) that defines the file service that contains the root volume. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param rootVolume: root volume name (string) that contains the user volume to be deleted. A list of user volumes is given by getRootVolumes(fileService) - :param userVolume: user volume name (string) to be deleted. - :param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume. + :param path: path (in the remote file service) to the user volume (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume :param quiet: If set to False, it will throw an error if user volume does not exists. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); Files.deleteUserVolume("volumes","newUserVolume",fileServices[0]); @@ -215,8 +266,10 @@ def deleteUserVolume(fileService, rootVolume, userVolume, userVolumeOwner=None, else: taskName = "SciScript-Python.Files.createUserVolume" - if userVolumeOwner is None: - userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) url = __getFileServiceAPIUrl(fileService) + "api/volume/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?quiet="+str(quiet)+"&TaskName="+taskName; @@ -226,19 +279,16 @@ def deleteUserVolume(fileService, rootVolume, userVolume, userVolumeOwner=None, if res.status_code >= 200 and res.status_code < 300: pass else: - raise Exception("Error when deleting user volume '" + userVolume + "' in root volume '" + rootVolume + "' in file service named '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when deleting user volume '" + str(path) + "' in file service '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") -def createDir(fileService, rootVolume, userVolume, relativePath, userVolumeOwner=None, quiet=True): +def createDir(fileService, path, quiet=True): """ Create a directory. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. - :param userVolume: user volume name (string) that contains the relativePath. - :param relativePath: path (string) to the directory, relative to the userVolume level. E.g: path="/newDirectory" - :param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume. + :param path: path (in the remote file service) to the directory (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/directory :param quiet: If set to False, it will throw an error if the directory already exists. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); Files.createDir(fileServices[0], "myRootVolume","myUserVolume", "myNewDir"); @@ -252,13 +302,19 @@ def createDir(fileService, rootVolume, userVolume, relativePath, userVolumeOwner else: taskName = "SciScript-Python.Files.createDir" + + (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); + if not relativePath.startswith("/"): relativePath = "/" + relativePath; if userVolumeOwner is None: userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - url = __getFileServiceAPIUrl(fileService) + "api/folder/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName; + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + url = __getFileServiceAPIUrl(fileService) + "api/folder/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName; headers = {'X-Auth-Token': token} res = requests.put(url, headers=headers) @@ -266,18 +322,16 @@ def createDir(fileService, rootVolume, userVolume, relativePath, userVolumeOwner if res.status_code >= 200 and res.status_code < 300: pass; else: - raise Exception("Error when creating directory '" + relativePath + "' in user volume '" + userVolume + "' in root volume '" + rootVolume + "' in file service named '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when creating directory '" + str(path) + "' in file service '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") -def upload(fileService, rootVolume, userVolume, relativePath, data="", localFilePath=None, userVolumeOwner=None, quiet=True): +def upload(fileService, path, data="", localFilePath=None, quiet=True): """ Uploads data or a local file into a path defined in the file system. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. - :param userVolume: user volume name (string) that contains the relativePath. - :param relativePath: destination path (string) of the file to be uploaded, relative to the user volume level. E.g: path="/newDirectory/myNewFile.txt". + :param path: path (in the remote file service) to the destination file (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/destinationFile.txt :param data: string containing data to be uploaded, in case localFilePath is not set. :param localFilePath: path to a local file to be uploaded (string), :param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume. @@ -294,13 +348,12 @@ def upload(fileService, rootVolume, userVolume, relativePath, data="", localFile else: taskName = "SciScript-Python.Files.UploadFile" - if not relativePath.startswith("/"): - relativePath = "/" + relativePath; + (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); - if userVolumeOwner is None: - userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) - url = __getFileServiceAPIUrl(fileService) + "api/file/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName + url = __getFileServiceAPIUrl(fileService) + "api/file/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName headers = {'X-Auth-Token': token} @@ -316,18 +369,16 @@ def upload(fileService, rootVolume, userVolume, relativePath, data="", localFile if res.status_code >= 200 and res.status_code < 300: pass; else: - raise Exception("Error when uploading file '" + relativePath + "' in user volume '" + userVolume + "' in root volume '" + rootVolume + "' in file service named '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when uploading file to '" + str(path) + "' in file service '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") -def download(fileService, rootVolume, userVolume, relativePath, localFilePath=None, format="txt", userVolumeOwner=None, quiet=True): +def download(fileService, path, localFilePath=None, format="txt", quiet=True): """ Downloads a file from the remote file system into the local file system, or returns the file content as an object in several formats. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. - :param userVolume: user volume name (string) that contains the relativePath. - :param relativePath: remote path (string) of the file to be downloaded, relative to the user volume level. E.g: path="/newDirectory/myNewFile.txt" + :param path: String defining the path (in the remote file service) of the file to be downloaded, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/fileToBeDownloaded.txt. :param localFilePath: local destination path of the file to be downloaded. If set to None, then an object of format 'format' will be returned. :param format: name (string) of the returned object's type (if localFilePath is not defined). This parameter can be "StringIO" (io.StringIO object containing readable text), "BytesIO" (io.BytesIO object containing readable binary data), "response" ( the HTTP response as an object of class requests.Response) or "txt" (a text string). If the parameter 'localFilePath' is defined, then the 'format' parameter is not used and the file is downloaded to the local file system instead. :param userVolumeOwner: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. @@ -340,29 +391,27 @@ def download(fileService, rootVolume, userVolume, relativePath, localFilePath=No token = Authentication.getToken() if token is not None and token != "": + (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) if localFilePath is not None: if os.path.isfile(localFilePath) and not quiet: - raise Exception("Error when downloading from remote File Syatem the file " + str(relativePath) + ".\nLocal file '" + localFilePath + "' already exists."); + raise Exception("Error when downloading '" + str(path) + "' from file service '" + fileService.get("name") + "'. Local file '" + localFilePath + "' already exists."); if Config.isSciServerComputeEnvironment(): taskName = "Compute.SciScript-Python.Files.DownloadFile" else: taskName = "SciScript-Python.Files.DownloadFile" - if not relativePath.startswith("/"): - relativePath = "/" + relativePath; - - if userVolumeOwner is None: - userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - - url = __getFileServiceAPIUrl(fileService) + "api/file/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?TaskName=" + taskName; + url = __getFileServiceAPIUrl(fileService) + "api/file/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?TaskName=" + taskName; headers = {'X-Auth-Token': token} res = requests.get(url, stream=True, headers=headers) if res.status_code < 200 or res.status_code >= 300: - raise Exception("Error when downloading from remote File Syatem the file " + str(relativePath) + ".\nHttp Response from FileSystem API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when downloading '" + str(path) + "' from file service '" + fileService.get("name") + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); if localFilePath is not None and localFilePath != "": @@ -393,16 +442,13 @@ def download(fileService, rootVolume, userVolume, relativePath, localFilePath=No -def dirList(fileService, rootVolume, userVolume, relativePath, level=1, options='', userVolumeOwner=None): +def dirList(fileService, path, level=1, options=''): """ Lists the contents of a directory. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. - :param userVolume: user volume name (string) that contains the relativePath. - :param relativePath: remote path (string) of the directory, relative to the user volume level. E.g: path="/newDirectory/myNewFile.txt" + :param path: String defining the path (in the remote file service) of the directory to be listed, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/directoryToBeListed. :param level: amount (int) of listed directory levels that are below or at the same level to that of the relativePath. :param options: string of file filtering options. - :param userVolumeOwner: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. :return: dictionary containing the directory listing. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); dirs = Files.dirList("/","persistent","myUserName", fileServices[0], 2); @@ -416,13 +462,12 @@ def dirList(fileService, rootVolume, userVolume, relativePath, level=1, options= else: taskName = "SciScript-Python.Files.dirList" - if userVolumeOwner is None: - userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); - if not relativePath.startswith("/"): - path = "/" + relativePath; + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) - url = __getFileServiceAPIUrl(fileService) + "api/jsontree/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + path + "?options=" + options + "&level=" + str(level) + "&TaskName=" + taskName; + url = __getFileServiceAPIUrl(fileService) + "api/jsontree/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?options=" + options + "&level=" + str(level) + "&TaskName=" + taskName; headers = {'X-Auth-Token': token} res = requests.get(url, headers=headers) @@ -430,26 +475,20 @@ def dirList(fileService, rootVolume, userVolume, relativePath, level=1, options= if res.status_code >= 200 and res.status_code < 300: return json.loads(res.content.decode()); else: - raise Exception("Error when listing directory.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when listing contents of '" + str(path) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") -def move(fileService, rootVolume, userVolume, relativePath, destinationFileService, destinationRootVolume, destinationUserVolume, destinationRelativePath, replaceExisting=True, doCopy=True, userVolumeOwner=None, destinationUserVolumeOwner=None): +def move(fileService, path, destinationFileService, destinationPath, replaceExisting=True, doCopy=True): """ Moves or copies a file or folder. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileService object. - :param userVolume: user volume name (string) that contains the relativePath. - :param relativePath: remote path (string) of the file to be moved/copied, relative to the user volume level. E.g: path="/newDirectory/myNewFile.txt" + :param path: String defining the origin path (in the remote fileService) of the file or directory to be copied/moved, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/fileToBeMoved.txt. :param destinationFileService: name of fileService (string), or object (dictionary) that defines a destination file service (where the file is moved/copied into). A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param destinationRootVolume: root volume name (string) that contains the userVolume where the file is moved/copied into. A list of root volumes is contained by the fileService object. - :param destinationUserVolume: user volume name (string) that contains the relativePath where the file is moved/copied into. - :param destinationRelativePath: remote path (string) of the destination file, relative to the user volume level. E.g: path="/newDirectory2/myNewFile.txt" + :param destinationRelativePath: String defining the destination path (in the remote destinationFileService) of the file or directory to be copied/moved, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/recentlyMovedFile.txt. :param replaceExisting: If set to False, it will throw an error if the file already exists, If set to True, it will not throw and eeror in that case. :param doCopy: if set to True, then it will copy the file or folder. If set to False, then the file or folder will be moved. - :param userVolumeOwner: name (string) of owner of the user volume. Can be left undefined if requester is the owner of the volume. - :param destinationUserVolumeOwner: name (string) of owner of the destination user volume. Can be left undefined if requester is the owner of it. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); isDownloaded = Files.upload("/myUploadedFile.txt","persistent","myUserName", fileServices[0], localFilePath="/myDownloadedFile.txt"); .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.upload, Files.dirList @@ -458,15 +497,12 @@ def move(fileService, rootVolume, userVolume, relativePath, destinationFileServi if token is not None and token != "": if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.Files.DownloadFile" + taskName = "Compute.SciScript-Python.Files.Move" else: - taskName = "SciScript-Python.Files.DownloadFile" + taskName = "SciScript-Python.Files.Move" - if not relativePath.startswith("/"): - relativePath = "/" + relativePath; - - if not destinationRelativePath.startswith("/"): - destinationRelativePath = "/" + destinationRelativePath; + (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); + (destinationRootVolume, destinationUserVolumeOwner, destinationUserVolume, destinationRelativePath) = splitPath(destinationPath); if userVolumeOwner is None: userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; @@ -474,28 +510,36 @@ def move(fileService, rootVolume, userVolume, relativePath, destinationFileServi if destinationUserVolumeOwner is None: destinationUserVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName; - headers = {'X-Auth-Token': token} - jsonDict = {'destinationPath': destinationRelativePath, 'destinationRootFolder': destinationRootVolume, 'destinationUserVolume':destinationUserVolume, 'destinationOwner': destinationUserVolumeOwner, 'destinationFileService': destinationFileService}; + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + if type(destinationFileService) == str: + destinationFileService = getFileServiceFromName(destinationFileService) + + url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName; + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + + if type(destinationFileService) == dict: + destinationFileService = destinationFileService['name'] + + + jsonDict = {'destinationPath': destinationRelativePath, 'destinationRootVolume': destinationRootVolume, 'destinationUserVolume':destinationUserVolume, 'destinationOwnerName': destinationUserVolumeOwner, 'destinationFileService': destinationFileService}; data = json.dumps(jsonDict).encode() - res = requests.put(url, stream=True, headers=headers, data=data) + res = requests.put(url, stream=True, headers=headers, json=jsonDict) if res.status_code < 200 or res.status_code >= 300: - raise Exception("Error when downloading from remote File System the file " + str(relativePath) + ".\nHttp Response from FileSystem API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when moving '" + str(path) + "' in file service '" + fileService.get("name") + "' to '" + str(path) + "' in file service '" + destinationFileService.get("name)") + "'. \nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") -def delete(fileService, rootVolume, userVolume, relativePath, userVolumeOwner=None, quiet=True): +def delete(fileService, path, quiet=True): """ Deletes a directory or file in the File System. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. - :param userVolume: user volume name (string) that contains the relativePath. - :param relativePath: remote path (string) of the directory or file to be deleted, relative to the user volume level. E.g: path="/newDirectory/myNewFile.txt" - :param userVolumeOwner: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. + :param path: String defining the path (in the remote fileService) of the file or directory to be deleted, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/fileToBeDeleted.txt. :param quiet: If set to False, it will throw an error if the file does not exist. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]); @@ -509,13 +553,9 @@ def delete(fileService, rootVolume, userVolume, relativePath, userVolumeOwner=No else: taskName = "SciScript-Python.Files.delete" - if not relativePath.startswith("/"): - relativePath = "/" + relativePath; - - if userVolumeOwner is None: - userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); - url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName + url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName headers = {'X-Auth-Token': token} res = requests.delete(url, headers=headers) @@ -523,17 +563,17 @@ def delete(fileService, rootVolume, userVolume, relativePath, userVolumeOwner=No if res.status_code >= 200 and res.status_code < 300: pass; else: - raise Exception("Error when deleting " + relativePath + ".\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when deleting '" + str(path) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") -def shareUserVolume(fileService, rootVolume, userVolume, sharedWith, allowedActions, type="USER", userVolumeOwner=None): +def shareUserVolume(fileService, path, sharedWith, allowedActions, type="USER"): """ Deletes a directory or file in the File System. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileSerive object. + :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileService object. :param userVolume: user volume name (string) that contains the relativePath. :param sharedWith: name (string) of user or group that the userVolume is shared with. :param allowedActions: array of strings defining actions the user or group is allowed to do with respect to the shared user volume. E.g.: ["read","write","grant","delete"]. The "grant" action means that the user or group can also share the user volume with another user or group. The "delete" action meand ability to delete the user volume (use with care). @@ -551,8 +591,7 @@ def shareUserVolume(fileService, rootVolume, userVolume, sharedWith, allowedActi else: taskName = "SciScript-Python.Files.ShareUserVolume" - if userVolumeOwner is None: - userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; + (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); data = [{'name': sharedWith, 'type':type, 'allowedActions': allowedActions }] @@ -566,7 +605,6 @@ def shareUserVolume(fileService, rootVolume, userVolume, sharedWith, allowedActi if res.status_code >= 200 and res.status_code < 300: pass; else: - raise Exception("Error when sharing userVolume '" + userVolume + "' in rootVolume '" + rootVolume + "' with '" + sharedWith + "' in file service named '" + fileService.get( - 'name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()) + raise Exception("Error when sharing userVolume '" + str(path) + "' in file service '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()) else: raise Exception("User token is not defined. First log into SciServer.") From faf45638798c49a039320a646bb3396596497d9b Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Wed, 14 Mar 2018 10:06:34 -0400 Subject: [PATCH 31/79] update example notebook to the new way of referring to file system path --- Examples/Examples_SciScript-Python.ipynb | 143 +++++++++++++++-------- 1 file changed, 96 insertions(+), 47 deletions(-) diff --git a/Examples/Examples_SciScript-Python.ipynb b/Examples/Examples_SciScript-Python.ipynb index e9b0360..150bd55 100644 --- a/Examples/Examples_SciScript-Python.ipynb +++ b/Examples/Examples_SciScript-Python.ipynb @@ -1121,10 +1121,10 @@ "# defining the FileService name\n", "\n", "Files_FileServiceName = \"FileServiceJHU\"\n", - "Files_RootVolumeName1 = \"volumes\"\n", - "Files_UserVolumeName1 = Authentication_loginName + \"_UserVolume555\"\n", - "Files_RootVolumeName2 = \"volumes\"\n", - "Files_UserVolumeName2 = Authentication_loginName + \"_UserVolume999\"\n", + "Files_RootVolumeName1 = \"persistent\"\n", + "Files_UserVolumeName1 = \"UserVolume555\"\n", + "Files_RootVolumeName2 = \"persistent\"\n", + "Files_UserVolumeName2 = \"UserVolume999\"\n", "Files_NewDirectoryName1 = \"myNewDirectory555\"\n", "Files_NewDirectoryName2 = \"myNewDirectory999\"\n", "Files_LocalFileName = \"MyNewFile.txt\"\n", @@ -1179,9 +1179,9 @@ "outputs": [], "source": [ "# get FileService from Name\n", - "\n", + "Files_FileServiceName = 'NewFileServiceScitest12_1 INSECURE'\n", "fileService = Files.getFileServiceFromName(Files_FileServiceName);\n", - "print(fileService)" + "fileService" ] }, { @@ -1208,8 +1208,22 @@ "source": [ "# get root volumes\n", "\n", - "rootVolumes = Files.getRootVolumes(fileService)\n", - "print(rootVolumes)" + "rootVolumes = Files.getRootVolumesInfo(fileService)\n", + "rootVolumes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# get root volumes\n", + "\n", + "userVolumes = Files.getUserVolumesInfo(fileService)\n", + "userVolumes" ] }, { @@ -1222,8 +1236,10 @@ "source": [ "# create user volume\n", "\n", - "Files.createUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName1)\n", - "Files.createUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName2)" + "Files.createUserVolume(fileService,\n", + " \"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1]))\n", + "Files.createUserVolume(fileService, \n", + " \"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName2]))" ] }, { @@ -1236,8 +1252,10 @@ "source": [ "# create a directory in the persistent volume\n", "\n", - "Files.createDir(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1);\n", - "Files.createDir(fileService, Files_RootVolumeName2, Files_UserVolumeName2, Files_NewDirectoryName2);" + "Files.createDir(fileService, \n", + " \"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1]));\n", + "Files.createDir(fileService, \n", + " \"/\".join([Files_RootVolumeName2,Authentication_loginName,Files_UserVolumeName2,Files_NewDirectoryName2]));" ] }, { @@ -1250,8 +1268,8 @@ "source": [ "# upload a text string into a file in the remote directory\n", "\n", - "Files.upload(fileService, Files_RootVolumeName1, Files_UserVolumeName1, \n", - " Files_NewDirectoryName1 + \"/\" + Files_LocalFileName, \n", + "Files.upload(fileService, \n", + " \"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1,Files_LocalFileName]), \n", " data=Files_LocalFileContent);" ] }, @@ -1265,8 +1283,10 @@ "source": [ "# List content of remote directory\n", "\n", - "dirList = Files.dirList(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1, level=2)\n", - "print(dirList)" + "dirList = Files.dirList(fileService, \n", + " \"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1]),\n", + " level=4)\n", + "dirList" ] }, { @@ -1279,22 +1299,23 @@ "source": [ "# download a remote text file into a local directory\n", "\n", - "Files.download(fileService, Files_RootVolumeName1, Files_UserVolumeName1, \n", - " Files_NewDirectoryName1 + \"/\" + Files_LocalFileName, \n", - " localFilePath=Files_LocalFileName);" + "Files.download(fileService, \n", + " \"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1,Files_LocalFileName]) \n", + " ,localFilePath=Files_LocalFileName);" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": false }, "outputs": [], "source": [ "# Delete remote file\n", "\n", - "Files.delete(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1 + \"/\" + Files_LocalFileName)" + "Files.delete(fileService, \n", + "\"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1,Files_LocalFileName]))" ] }, { @@ -1307,8 +1328,9 @@ "source": [ "# upload a local file into a remote directory\n", "\n", - "Files.upload(fileService, Files_RootVolumeName1, Files_UserVolumeName1,\n", - " Files_NewDirectoryName1 + \"/\" + Files_LocalFileName, localFilePath=Files_LocalFileName, quiet=False);" + "Files.upload(fileService, \n", + "\"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1,Files_LocalFileName])\n", + ", localFilePath=Files_LocalFileName, quiet=False);" ] }, { @@ -1321,9 +1343,11 @@ "source": [ "# copy remote file into remote directory\n", "\n", - "Files.move(fileService, Files_RootVolumeName1, Files_UserVolumeName1,Files_NewDirectoryName1 + \"/\" + Files_LocalFileName, \n", - " fileService, Files_RootVolumeName2, Files_UserVolumeName2,Files_UserVolumeName2 + \"/\" + Files_NewDirectoryName2 + \"/\" + Files_LocalFileName\n", - " );" + "Files.move(fileService, \n", + "\"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1,Files_LocalFileName]), \n", + " fileService, \n", + "\"/\".join([Files_RootVolumeName2,Authentication_loginName,Files_UserVolumeName2,Files_NewDirectoryName2 ,Files_LocalFileName])\n", + ");" ] }, { @@ -1336,10 +1360,9 @@ "source": [ "# sharing user volume with another user\n", "\n", - "Files.shareUserVolume(fileService, Files_RootVolumeName2, Files_UserVolumeName2, \n", - " sharedWith=Authentication_login_sharedWithName, type=\"USER\", \n", - " userVolumeOwner=Authentication_loginName,\n", - " allowedActions=[\"read\"])" + "Files.shareUserVolume(fileService, \n", + "\"/\".join([Files_RootVolumeName2,Authentication_loginName,Files_UserVolumeName2]), \n", + " sharedWith=Authentication_login_sharedWithName, type=\"USER\",allowedActions=[\"read\"])" ] }, { @@ -1365,9 +1388,9 @@ "source": [ "# the other user downloads the remote text file (from the shared user volume) into a local string variable\n", "\n", - "string = Files.download(fileService, Files_RootVolumeName2, Files_UserVolumeName2, \n", - " Files_NewDirectoryName2 + \"/\" + Files_LocalFileName, \n", - " format=\"txt\", userVolumeOwner=Authentication_loginName);\n", + "string = Files.download(fileService, \n", + " \"/\".join([Files_RootVolumeName2,Authentication_loginName,Files_UserVolumeName2,Files_NewDirectoryName2,Files_LocalFileName]), \n", + " format=\"txt\");\n", "print(string)" ] }, @@ -1394,8 +1417,10 @@ "source": [ "# delete folders\n", "\n", - "Files.delete(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1)\n", - "Files.delete(fileService, Files_RootVolumeName2, Files_UserVolumeName2, Files_NewDirectoryName2)" + "Files.delete(fileService, \n", + " \"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1]))\n", + "Files.delete(fileService, \n", + " \"/\".join([Files_RootVolumeName2,Authentication_loginName,Files_UserVolumeName2,Files_NewDirectoryName2]))" ] }, { @@ -1408,8 +1433,10 @@ "source": [ "# delete user volumes\n", "\n", - "Files.deleteUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName1)\n", - "Files.deleteUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName2)" + "Files.deleteUserVolume(fileService, \n", + " \"/\".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1]))\n", + "Files.deleteUserVolume(fileService, \n", + " \"/\".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName2]))" ] }, { @@ -1446,9 +1473,9 @@ "source": [ "Jobs_DockerComputeDomainName = 'Small Jobs Batch Domain'\n", "\n", - "Jobs_FileServiceName = \"FileServiceJHU\"\n", - "Jobs_RootVolumeName = \"volumes\"\n", - "Jobs_UserVolumeName = Authentication_loginName + \"_JobsTestVolume\"\n", + "Jobs_FileServiceName = \"NewFileServiceScitest12_1 INSECURE\"\n", + "Jobs_RootVolumeName = \"persistent\"\n", + "Jobs_UserVolumeName = \"JobsTestVolume\"\n", "Jobs_DirectoryName = \"JobsTestDirectory\"\n", "Jobs_NotebookName = 'TestNotebook.ipynb'\n", "Jobs_ShellCommand = 'ls -laht'\n", @@ -1518,10 +1545,11 @@ "# uploading Jupyter notebook to remote directory\n", "\n", "fileService = Files.getFileServiceFromName(Jobs_FileServiceName);\n", - "Files.createUserVolume(fileService, Jobs_RootVolumeName, Jobs_UserVolumeName)\n", - "Files.upload(fileService, Jobs_RootVolumeName, Jobs_UserVolumeName, \n", - " Jobs_DirectoryName + \"/\" + Jobs_NotebookName, \n", - " localFilePath=Jobs_NotebookName);" + "Files.createUserVolume(fileService, \n", + " \"/\".join([Jobs_RootVolumeName, Authentication_loginName, Jobs_UserVolumeName]))\n", + "Files.upload(fileService, \n", + " \"/\".join([Jobs_RootVolumeName,Authentication_loginName,Jobs_UserVolumeName,Jobs_DirectoryName,Jobs_NotebookName]), \n", + " localFilePath=Jobs_NotebookName);" ] }, { @@ -1534,7 +1562,8 @@ "source": [ "# submit a Jupyter notebook job.\n", "\n", - "jobId = Jobs.submitNotebookJob('/home/idies/workspace/' + Jobs_UserVolumeName + '/' + Jobs_DirectoryName + '/' + Jobs_NotebookName,\n", + "jobId = Jobs.submitNotebookJob('/home/idies/workspace/' + \n", + "\"/\".join([Jobs_RootVolumeName,Authentication_loginName,Jobs_UserVolumeName,Jobs_DirectoryName,Jobs_NotebookName]),\n", " dockerComputeDomain,\n", " Jobs_DockerImageName, \n", " Jobs_UserVolumes, Jobs_DataVolumes,\n", @@ -1542,6 +1571,17 @@ "print(jobId)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "'/home/idies/workspace/' + \"/\".join([Jobs_RootVolumeName,Authentication_loginName,Jobs_UserVolumeName,Jobs_DirectoryName,Jobs_NotebookName])" + ] + }, { "cell_type": "code", "execution_count": null, @@ -1578,7 +1618,7 @@ }, "outputs": [], "source": [ - "# wait until job is finsihed and get job status\n", + "# wait until job is finished and get job status\n", "\n", "jobId = Jobs.submitNotebookJob('/home/idies/workspace/' + Jobs_UserVolumeName + '/' + Jobs_DirectoryName + '/' + Jobs_NotebookName,\n", " dockerComputeDomain,\n", @@ -1615,7 +1655,7 @@ }, "outputs": [], "source": [ - "Files.deleteUserVolume(fileService, Jobs_RootVolumeName, Jobs_UserVolumeName, quiet=True)" + "Files.deleteUserVolume(fileService, \"/\".join([Jobs_RootVolumeName,Authentication_loginName,Jobs_UserVolumeName]), quiet=True)" ] }, { @@ -1690,6 +1730,15 @@ "jobId = Jobs.submitRDBQueryJob(Jobs_SqlQuery, rdbComputeDomain, Jobs_DatabaseContextName, Jobs_QueryResultsFile, Jobs_Alias)\n", "print(jobId)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] } ], "metadata": { From 98ff9c3ae8d2d7d479361fefdb8c36f2661fd1ba Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Thu, 15 Mar 2018 12:29:22 -0400 Subject: [PATCH 32/79] updated Files section of unit tests with latest file service schema --- Tests/UnitTests.py | 98 ++++++++++++++++++++++------------------------ 1 file changed, 46 insertions(+), 52 deletions(-) diff --git a/Tests/UnitTests.py b/Tests/UnitTests.py index c51e610..8ab71b7 100644 --- a/Tests/UnitTests.py +++ b/Tests/UnitTests.py @@ -41,19 +41,17 @@ SciDrive_FileName = "TestFile.csv" SciDrive_FileContent = "Column1,Column2\n4.5,5.5\n" - -Files_FileServiceName = "FileServiceJHU" -Files_RootVolumeName1 = "volumes" -Files_UserVolumeName1 = Authentication_loginName + "_UserVolume555" -Files_RootVolumeName2 = "volumes" -Files_UserVolumeName2 = Authentication_loginName + "_UserVolume999" +Files_FileServiceName = "NewFileServiceScitest12_1 INSECURE" +Files_RootVolumeName1 = "persistent" +Files_UserVolumeName1 = "UserVolume555" +Files_RootVolumeName2 = "persistent" +Files_UserVolumeName2 = "UserVolume999" Files_NewDirectoryName1 = "myNewDirectory555" Files_NewDirectoryName2 = "myNewDirectory999" Files_LocalFileName = "MyNewFile.txt" Files_LocalFileContent = "#ID,Column1,Column2\n1,4.5,5.5" - Jobs_DockerComputeDomainName = 'Small Jobs Batch Domain' Jobs_FileServiceName = "FileServiceJHU" Jobs_RootVolumeName = "volumes" @@ -456,32 +454,42 @@ def test_Files_getFileServicesNames(self): def test_Files_getFileServicesNames(self): fileServiceNames = Files.getFileServicesNames(); - self.assertTrue(Files_FileServiceName in fileServiceNames) + found = False; + for fileService in fileServiceNames: + if fileService.get('name') == Files_FileServiceName: + found=True; + + self.assertTrue(found) def test_Files_getFileServiceFromName(self): fileService = Files.getFileServiceFromName(Files_FileServiceName); self.assertTrue(fileService.get('name') == Files_FileServiceName); - def test_Files_getRootVolumes(self): + def test_Files_getRootVolumesInfo(self): fileService = Files.getFileServiceFromName(Files_FileServiceName); - rootVolumes = Files.getRootVolumes(fileService) + rootVolumes = Files.getRootVolumesInfo(fileService) self.assertTrue(rootVolumes.__len__() > 0) found = False for rootVolume in rootVolumes: - if rootVolume.get('name') == Files_RootVolumeName1: + if rootVolume.get('rootVolumeName') == Files_RootVolumeName1: found = True self.assertTrue(found) found = False for rootVolume in rootVolumes: - if rootVolume.get('name') == Files_RootVolumeName2: + if rootVolume.get('rootVolumeName') == Files_RootVolumeName2: found = True self.assertTrue(found) + def test_Files_getUserVolumesInfo(self): + fileService = Files.getFileServiceFromName(Files_FileServiceName); + userVolumesInfo = Files.getUserVolumesInfo(fileService) + self.assertTrue(userVolumesInfo.__len__() > 0) + def test_Files_createUserVolume_deleteUserVolume(self): fileService = Files.getFileServiceFromName(Files_FileServiceName); - Files.createUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName1,quiet=False) - Files.deleteUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName1,quiet=False) + Files.createUserVolume(fileService,"/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1]),quiet=False) + Files.deleteUserVolume(fileService,"/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1]),quiet=False) def test_Files_createDir_upload_dirList_download_download_shareUserVolume(self): @@ -495,55 +503,41 @@ def test_Files_createDir_upload_dirList_download_download_shareUserVolume(self): try: fileService = Files.getFileServiceFromName(Files_FileServiceName); - Files.createUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName1,quiet=False) - Files.createUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName2,quiet=False) - Files.createDir(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1); - Files.createDir(fileService, Files_RootVolumeName2, Files_UserVolumeName2, Files_NewDirectoryName2); + Files.createUserVolume(fileService,"/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1]),quiet=False) + Files.createUserVolume(fileService,"/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName2]),quiet=False) + + Files.createDir(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1, Files_NewDirectoryName1])); + Files.createDir(fileService, "/".join([Files_RootVolumeName2, Authentication_loginName, Files_UserVolumeName2, Files_NewDirectoryName2])); - dirList = Files.dirList(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1,level=2) + dirList = Files.dirList(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1, Files_NewDirectoryName1]),level=2) self.assertTrue(dirList.get('root').get('name') == Files_NewDirectoryName1) - Files.upload(fileService, Files_RootVolumeName1, Files_UserVolumeName1, - Files_NewDirectoryName1 + "/" + Files_LocalFileName, - data=Files_LocalFileContent); + Files.upload(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1, Files_NewDirectoryName1, Files_LocalFileName]),data=Files_LocalFileContent); - dirList = Files.dirList(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1,level=2) + dirList = Files.dirList(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1, Files_NewDirectoryName1]) ,level=2) self.assertTrue(dirList.get('root').get('files')[0].get('name') == Files_LocalFileName) - Files.download(fileService, Files_RootVolumeName1, Files_UserVolumeName1, - Files_NewDirectoryName1 + "/" + Files_LocalFileName, - localFilePath=Files_LocalFileName); + Files.download(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1,Files_NewDirectoryName1, Files_LocalFileName]),localFilePath=Files_LocalFileName); with open(Files_LocalFileName, 'r') as myfile: downloadedFileContent = myfile.read() assert(downloadedFileContent == Files_LocalFileContent) - Files.delete(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1 + "/" + Files_LocalFileName) + Files.delete(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1, Files_NewDirectoryName1, Files_LocalFileName])) - dirList = Files.dirList(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1,level=2) + dirList = Files.dirList(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1, Files_NewDirectoryName1]),level=2) self.assertIsNone(dirList.get('root').get('files')) - Files.upload(fileService, Files_RootVolumeName1, Files_UserVolumeName1, - Files_NewDirectoryName1 + "/" + Files_LocalFileName, localFilePath=Files_LocalFileName, - quiet=False); + Files.upload(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1, Files_NewDirectoryName1, Files_LocalFileName]), localFilePath=Files_LocalFileName,quiet=False); - Files.move(fileService, Files_RootVolumeName1, Files_UserVolumeName1, - Files_NewDirectoryName1 + "/" + Files_LocalFileName, - fileService, Files_RootVolumeName2, Files_UserVolumeName2, - Files_UserVolumeName2 + "/" + Files_NewDirectoryName2 + "/" + Files_LocalFileName - ); + Files.move(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1, Files_NewDirectoryName1, Files_LocalFileName]), fileService, "/".join([Files_RootVolumeName2, Authentication_loginName, Files_UserVolumeName2, Files_NewDirectoryName2, Files_LocalFileName])); - Files.shareUserVolume(fileService, Files_RootVolumeName2, Files_UserVolumeName2, - sharedWith=Authentication_login_sharedWithName, type="USER", - userVolumeOwner=Authentication_loginName, - allowedActions=["read"]) + Files.shareUserVolume(fileService, "/".join([Files_RootVolumeName2, Authentication_loginName, Files_UserVolumeName2]), sharedWith=Authentication_login_sharedWithName, type="USER",allowedActions=["read"]) token1 = Authentication.login(Authentication_login_sharedWithName, Authentication_login_sharedWithPassword); - string = Files.download(fileService, Files_RootVolumeName2, Files_UserVolumeName2, - Files_NewDirectoryName2 + "/" + Files_LocalFileName, - format="txt", userVolumeOwner=Authentication_loginName); + string = Files.download(fileService, "/".join([Files_RootVolumeName2, Authentication_loginName, Files_UserVolumeName2, Files_NewDirectoryName2, Files_LocalFileName]), format="txt"); self.assertTrue(string, Files_LocalFileContent) @@ -552,8 +546,8 @@ def test_Files_createDir_upload_dirList_download_download_shareUserVolume(self): finally: try: os.remove(Files_LocalFileName); - Files.deleteUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName1,quiet=True) - Files.deleteUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName2,quiet=True) + Files.deleteUserVolume(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1]),quiet=True) + Files.deleteUserVolume(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName2]),quiet=True) except: pass; @@ -733,12 +727,12 @@ def test_Jobs_getJobDirectory(self): unittest.TestLoader.sortTestMethodsUsing = lambda x, y: cmp(x,y); testLoader = unittest.TestLoader() testLoader.sortTestMethodsUsing = lambda x, y: 0; - suite = testLoader.loadTestsFromTestCase(TestAuthentication); unittest.TextTestRunner(verbosity=2).run(suite) - suite = testLoader.loadTestsFromTestCase(TestLoginPortal); unittest.TextTestRunner(verbosity=2).run(suite) - suite = testLoader.loadTestsFromTestCase(TestCasJobs); unittest.TextTestRunner(verbosity=2).run(suite) - suite = testLoader.loadTestsFromTestCase(TestSkyServer); unittest.TextTestRunner(verbosity=2).run(suite) - suite = testLoader.loadTestsFromTestCase(TestSciDrive); unittest.TextTestRunner(verbosity=2).run(suite) - suite = testLoader.loadTestsFromTestCase(TestSkyQuery); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestAuthentication); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestLoginPortal); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestCasJobs); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestSkyServer); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestSciDrive); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestSkyQuery); unittest.TextTestRunner(verbosity=2).run(suite) suite = testLoader.loadTestsFromTestCase(TestFileService); unittest.TextTestRunner(verbosity=2).run(suite) - suite = testLoader.loadTestsFromTestCase(TestJobs); unittest.TextTestRunner(verbosity=2).run(suite) + #suite = testLoader.loadTestsFromTestCase(TestJobs); unittest.TextTestRunner(verbosity=2).run(suite) From 535573c8df74cbfe7321d734a488e3531822e240 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Fri, 16 Mar 2018 15:21:18 -0400 Subject: [PATCH 33/79] improve docs --- py2/SciServer/Files.py | 8 +++----- py3/SciServer/Files.py | 8 +++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/py2/SciServer/Files.py b/py2/SciServer/Files.py index b12142e..24edaf4 100644 --- a/py2/SciServer/Files.py +++ b/py2/SciServer/Files.py @@ -571,14 +571,12 @@ def delete(fileService, path, quiet=True): def shareUserVolume(fileService, path, sharedWith, allowedActions, type="USER"): """ - Deletes a directory or file in the File System. + Shares a user volume with another user or group :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileService object. - :param userVolume: user volume name (string) that contains the relativePath. - :param sharedWith: name (string) of user or group that the userVolume is shared with. + :param path: String defining the path (in the remote fileService) of the user volume to be shared, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume. + :param sharedWith: name (string) of user or group that the user volume is shared with. :param allowedActions: array of strings defining actions the user or group is allowed to do with respect to the shared user volume. E.g.: ["read","write","grant","delete"]. The "grant" action means that the user or group can also share the user volume with another user or group. The "delete" action meand ability to delete the user volume (use with care). :param type: type (string) of the entity defined by the "sharedWith" parameter. Can be set to "USER" or "GROUP". - :param userVolumeOwner: name (string) of owner of the user volume. Can be left undefined if requester is the owner of the volume. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.shareUserVolume(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]); .. seealso:: Files.getFileServices(), Files.getFilrmsieServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList diff --git a/py3/SciServer/Files.py b/py3/SciServer/Files.py index b12142e..24edaf4 100644 --- a/py3/SciServer/Files.py +++ b/py3/SciServer/Files.py @@ -571,14 +571,12 @@ def delete(fileService, path, quiet=True): def shareUserVolume(fileService, path, sharedWith, allowedActions, type="USER"): """ - Deletes a directory or file in the File System. + Shares a user volume with another user or group :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param rootVolume: root volume name (string) that contains the userVolume. A list of root volumes is contained by the fileService object. - :param userVolume: user volume name (string) that contains the relativePath. - :param sharedWith: name (string) of user or group that the userVolume is shared with. + :param path: String defining the path (in the remote fileService) of the user volume to be shared, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume. + :param sharedWith: name (string) of user or group that the user volume is shared with. :param allowedActions: array of strings defining actions the user or group is allowed to do with respect to the shared user volume. E.g.: ["read","write","grant","delete"]. The "grant" action means that the user or group can also share the user volume with another user or group. The "delete" action meand ability to delete the user volume (use with care). :param type: type (string) of the entity defined by the "sharedWith" parameter. Can be set to "USER" or "GROUP". - :param userVolumeOwner: name (string) of owner of the user volume. Can be left undefined if requester is the owner of the volume. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.shareUserVolume(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]); .. seealso:: Files.getFileServices(), Files.getFilrmsieServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList From 077e7710cae028c6389feebf3c07f88f44b5da63 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Fri, 16 Mar 2018 15:21:33 -0400 Subject: [PATCH 34/79] improve docs --- py2/SciServer/Jobs.py | 16 +++++++++++----- py3/SciServer/Jobs.py | 18 ++++++++++++------ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py index 1d6b3a4..2a97fb1 100644 --- a/py2/SciServer/Jobs.py +++ b/py2/SciServer/Jobs.py @@ -286,7 +286,7 @@ def getJobStatus(jobId): raise Exception("Invalid integer value given to job status.") -def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=None, userVolumes=None, dataVolumes=None, parameters="", jobAlias= ""): +def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=None, userVolumes=None, dataVolumes=None, resultsFolderPath="", parameters="", jobAlias= ""): """ Submits a Jupyter Notebook for execution (as an asynchronous job) inside a Docker compute domain, :param notebookPath: path of the notebook within the filesystem mounted in SciServer-Compute (string). Example: notebookPath = '/home/idies/worskpace/persistent/JupyterNotebook.ipynb' @@ -298,6 +298,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No :param dataVolumes: a list with the names of data volumes that will be mounted to the docker Image. E.g., dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}]. A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted. + :param resultsFolderPath: full path to results folder (string) where the original notebook is copied to and executed. E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder. If not set, then a default folder will be set automatically. :param parameters: string containing parameters that the notebook might need during its execution. This string is written in the 'parameters.txt' file in the same directory level where the notebook is being executed. :param jobAlias: alias (string) of job, defined by the user. :return: the job ID (int) @@ -382,6 +383,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No "submitterDID": jobAlias, "dockerComputeEndpoint": dockerComputeEndpoint, "dockerImageName": dockerImageName, + "resultsFolderURI": resultsFolderPath, "volumeContainers": datVols, "userVolumes": uVols } @@ -398,7 +400,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No raise Exception("User token is not defined. First log into SciServer.") -def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageName = None, userVolumes = None, dataVolumes = None, jobAlias = ""): +def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageName = None, userVolumes = None, dataVolumes = None, resultsFolderPath = "", jobAlias = ""): """ Submits a shell command for execution (as an asynchronous job) inside a Docker compute domain. :param shellCommand: shell command (string) defined by the user. @@ -410,6 +412,7 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN :param dataVolumes: a list with the names of data volumes that will be mounted to the docker Image. E.g., dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}]. A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted. + :param resultsFolderPath: full path to results folder (string) where the shell command is executed. E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder. If not set, then a default folder will be set automatically. :param jobAlias: alias (string) of job, defined by the user. :return: the job ID (int) :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. @@ -494,7 +497,8 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN "dockerComputeEndpoint": dockerComputeEndpoint, "dockerImageName": dockerImageName, "volumeContainers": datVols, - "userVolumes": uVols + "userVolumes": uVols, + "resultsFolderURI": resultsFolderPath } data = json.dumps(dockerJobModel).encode() url = Config.RacmApiURL + "/jobm/rest/jobs/docker?TaskName="+taskName; @@ -508,13 +512,14 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN else: raise Exception("User token is not defined. First log into SciServer.") -def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = None, resultsName='queryResults', jobAlias = ""): +def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = None, resultsName='queryResults', resultsFolderPath="", jobAlias = ""): """ Submits a sql query for execution (as an asynchronous job) inside a relational database (RDB) compute domain. :param sqlQuery: sql query (string) :param rdbComputeDomain: object (dictionary) that defines a relational database (RDB) compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). :param databaseContextName: database context name (string) on which the sql query is executed. :param resultsName: name (string) of the table or file (without file type ending) that contains the query result. In case the sql query has multiple statements, should be set to a list of names (e.g., ['result1','result2']). + :param resultsFolderPath: full path to results folder (string) where query output tables are written into. E.g.: /home/idies/workspace/rootVOlume/username/userVolume/jobsFolder . If not set, then a default folder will be set automatically. :param jobAlias: alias (string) of job, defined by the user. :return: a dictionary containing the definition of the submitted job. :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. @@ -568,7 +573,8 @@ def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = Non "submitterDID": jobAlias, "databaseContextName": databaseContextName, "rdbDomainId": rdbDomainId, - "targets": targets + "targets": targets, + "resultsFolderURI":resultsFolderPath } data = json.dumps(dockerJobModel).encode() diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py index 9e80a3a..06b9fe5 100644 --- a/py3/SciServer/Jobs.py +++ b/py3/SciServer/Jobs.py @@ -286,7 +286,7 @@ def getJobStatus(jobId): raise Exception("Invalid integer value given to job status.") -def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=None, userVolumes=None, dataVolumes=None, parameters="", jobAlias= ""): +def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=None, userVolumes=None, dataVolumes=None, resultsFolderPath="", parameters="", jobAlias= ""): """ Submits a Jupyter Notebook for execution (as an asynchronous job) inside a Docker compute domain, :param notebookPath: path of the notebook within the filesystem mounted in SciServer-Compute (string). Example: notebookPath = '/home/idies/worskpace/persistent/JupyterNotebook.ipynb' @@ -298,6 +298,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No :param dataVolumes: a list with the names of data volumes that will be mounted to the docker Image. E.g., dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}]. A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted. + :param resultsFolderPath: full path to results folder (string) where the original notebook is copied to and executed. E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder. If not set, then a default folder will be set automatically. :param parameters: string containing parameters that the notebook might need during its execution. This string is written in the 'parameters.txt' file in the same directory level where the notebook is being executed. :param jobAlias: alias (string) of job, defined by the user. :return: the job ID (int) @@ -382,6 +383,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No "submitterDID": jobAlias, "dockerComputeEndpoint": dockerComputeEndpoint, "dockerImageName": dockerImageName, + "resultsFolderURI": resultsFolderPath, "volumeContainers": datVols, "userVolumes": uVols } @@ -398,7 +400,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No raise Exception("User token is not defined. First log into SciServer.") -def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageName = None, userVolumes = None, dataVolumes = None, jobAlias = ""): +def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageName = None, userVolumes = None, dataVolumes = None, resultsFolderPath = "", jobAlias = ""): """ Submits a shell command for execution (as an asynchronous job) inside a Docker compute domain. :param shellCommand: shell command (string) defined by the user. @@ -410,6 +412,7 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN :param dataVolumes: a list with the names of data volumes that will be mounted to the docker Image. E.g., dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}]. A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted. + :param resultsFolderPath: full path to results folder (string) where the shell command is executed. E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder. If not set, then a default folder will be set automatically. :param jobAlias: alias (string) of job, defined by the user. :return: the job ID (int) :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. @@ -494,7 +497,8 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN "dockerComputeEndpoint": dockerComputeEndpoint, "dockerImageName": dockerImageName, "volumeContainers": datVols, - "userVolumes": uVols + "userVolumes": uVols, + "resultsFolderURI": resultsFolderPath } data = json.dumps(dockerJobModel).encode() url = Config.RacmApiURL + "/jobm/rest/jobs/docker?TaskName="+taskName; @@ -508,13 +512,14 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN else: raise Exception("User token is not defined. First log into SciServer.") -def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = None, resultsName='queryResults', jobAlias = ""): +def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = None, resultsName='queryResults', resultsFolderPath="", jobAlias = ""): """ Submits a sql query for execution (as an asynchronous job) inside a relational database (RDB) compute domain. :param sqlQuery: sql query (string) :param rdbComputeDomain: object (dictionary) that defines a relational database (RDB) compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). :param databaseContextName: database context name (string) on which the sql query is executed. :param resultsName: name (string) of the table or file (without file type ending) that contains the query result. In case the sql query has multiple statements, should be set to a list of names (e.g., ['result1','result2']). + :param resultsFolderPath: full path to results folder (string) where query output tables are written into. E.g.: /home/idies/workspace/rootVOlume/username/userVolume/jobsFolder . If not set, then a default folder will be set automatically. :param jobAlias: alias (string) of job, defined by the user. :return: a dictionary containing the definition of the submitted job. :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. @@ -568,7 +573,8 @@ def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = Non "submitterDID": jobAlias, "databaseContextName": databaseContextName, "rdbDomainId": rdbDomainId, - "targets": targets + "targets": targets, + "resultsFolderURI":resultsFolderPath } data = json.dumps(dockerJobModel).encode() @@ -631,7 +637,7 @@ def waitForJob(jobId, verbose=False, pollTime = 5): waitingStr = "Waiting..." back = "\b" * len(waitingStr) if verbose: - print waitingStr, + print(waitingStr) while not complete: if verbose: From 923fd0a987bbe0977a3c02f6fc93e0ae5e5003f2 Mon Sep 17 00:00:00 2001 From: glemson Date: Mon, 28 May 2018 15:11:31 +0000 Subject: [PATCH 35/79] Updates to FIles.py --- py3/SciServer/Files.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/py3/SciServer/Files.py b/py3/SciServer/Files.py index 24edaf4..5379f8f 100644 --- a/py3/SciServer/Files.py +++ b/py3/SciServer/Files.py @@ -177,12 +177,13 @@ def getUserVolumesInfo(fileService, rootVolumeName = None, verbose=True): userVolumes = []; for rootVolume in fileService.get("rootVolumes"): for userVolume in rootVolume.get('userVolumes'): + path=os.path.join(rootVolume.get('name'),userVolume.get('owner'),userVolume.get('name')) if rootVolumeName is not None: if rootVolume.get('name') == rootVolumeName: - userVolumes.append({"userVolumeName": userVolume.get('name'), "userVolumeDescription": userVolume.get('description'), "rootVolumeName":rootVolume.get("name"), "rootVolumeDescription":rootVolume.get("description")}) + userVolumes.append({"userVolumeName": userVolume.get('name'), "path":path, "userVolumeDescription": userVolume.get('description'), "rootVolumeName":rootVolume.get("name"), "rootVolumeDescription":rootVolume.get("description")}) else: - userVolumes.append({"userVolumeName": userVolume.get('name'), "userVolumeDescription": userVolume.get('description'),"rootVolumeName": rootVolume.get("name"), "rootVolumeDescription": rootVolume.get("description")}) + userVolumes.append({"userVolumeName": userVolume.get('name'), "path":path,"userVolumeDescription": userVolume.get('description'),"rootVolumeName": rootVolume.get("name"), "rootVolumeDescription": rootVolume.get("description")}) return userVolumes @@ -597,7 +598,7 @@ def shareUserVolume(fileService, path, sharedWith, allowedActions, type="USER"): url = __getFileServiceAPIUrl(fileService) + "api/share/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?TaskName="+taskName - headers = {'X-Auth-Token': token} + headers = {'X-Auth-Token': token,'Content-Type':'application/json'} res = requests.patch(url, headers=headers, data=body) if res.status_code >= 200 and res.status_code < 300: From 129ee03f29c4518dbe923d17587bf8c936b0ff8e Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Tue, 5 Jun 2018 17:23:34 -0400 Subject: [PATCH 36/79] use "get " method for dict in py3 --- py3/SciServer/Jobs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py index 06b9fe5..80731a9 100644 --- a/py3/SciServer/Jobs.py +++ b/py3/SciServer/Jobs.py @@ -344,7 +344,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No for vol in dockerComputeDomain.get('userVolumes'): if vol.get('name') == uVol.get('name'): found = True; - if (uVol.has_key('needsWriteAccess')): + if (uVol.get('needsWriteAccess')): if uVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'): uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); else: @@ -458,7 +458,7 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN for vol in dockerComputeDomain.get('userVolumes'): if vol.get('name') == uVol.get('name'): found = True; - if (uVol.has_key('needsWriteAccess')): + if (uVol.get('needsWriteAccess')): if uVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'): uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); else: From fcdd00ffa011148d7aff437d941d6fdad8609ba7 Mon Sep 17 00:00:00 2001 From: Joseph Jon Booker Date: Thu, 7 Jun 2018 13:47:25 -0400 Subject: [PATCH 37/79] Allow overriding the config from a json file This json file is usually in ~/.config/sciserver/sciscript.json, but this should work for Windows config files in the user profile --- py2/SciServer/Config.py | 19 ++++++++++++++++++- py3/SciServer/Config.py | 19 ++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/py2/SciServer/Config.py b/py2/SciServer/Config.py index f922b4a..504342e 100644 --- a/py2/SciServer/Config.py +++ b/py2/SciServer/Config.py @@ -1,3 +1,5 @@ +import json +import os """ The SciServer.Config module contains important parameters for the correct functioning of the SciServer package.\n Although these parameters must be set/defined by the admin or user before the installation of the package, they can also be accessed and changed on-the-fly while on the python session.\n @@ -32,8 +34,23 @@ version = "sciserver-v1.10.0" #sciserver release version ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. +_CONFIG_DIR = os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')) +_SCISERVER_CONFIG_FILE = os.path.join(_CONFIG_DIR, 'sciserver', 'sciscript.json') +if os.path.exists(_SCISERVER_CONFIG_FILE): + with open(_SCISERVER_CONFIG_FILE) as f: + _config_data = json.load(f) + CasJobsRESTUri = _config_data.get('CasJobsRESTUri', CasJobsRESTUri) + AuthenticationURL = _config_data.get('AuthenticationURL', AuthenticationURL) + SciDriveHost = _config_data.get('SciDriveHost', SciDriveHost) + SkyQueryUrl = _config_data.get('SkyQueryUrl', SkyQueryUrl) + SkyServerWSurl = _config_data.get('SkyServerWSurl', SkyServerWSurl) + RacmApiURL = _config_data.get('RacmApiURL', RacmApiURL) + DataRelease = _config_data.get('DataRelease', DataRelease) + KeystoneTokenPath = _config_data.get('KeystoneTokenPath', KeystoneTokenPath) + version = _config_data.get('version', version) + ComputeJobDirectoryFile = _config_data.get('ComputeJobDirectoryFile', ComputeJobDirectoryFile) + # returns TRUE if the library is running inside the SciServer-Compute, and FALSE if not -import os; def isSciServerComputeEnvironment(): """ Checks whether the library is being run within the SciServer-Compute environment. diff --git a/py3/SciServer/Config.py b/py3/SciServer/Config.py index f922b4a..504342e 100644 --- a/py3/SciServer/Config.py +++ b/py3/SciServer/Config.py @@ -1,3 +1,5 @@ +import json +import os """ The SciServer.Config module contains important parameters for the correct functioning of the SciServer package.\n Although these parameters must be set/defined by the admin or user before the installation of the package, they can also be accessed and changed on-the-fly while on the python session.\n @@ -32,8 +34,23 @@ version = "sciserver-v1.10.0" #sciserver release version ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. +_CONFIG_DIR = os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')) +_SCISERVER_CONFIG_FILE = os.path.join(_CONFIG_DIR, 'sciserver', 'sciscript.json') +if os.path.exists(_SCISERVER_CONFIG_FILE): + with open(_SCISERVER_CONFIG_FILE) as f: + _config_data = json.load(f) + CasJobsRESTUri = _config_data.get('CasJobsRESTUri', CasJobsRESTUri) + AuthenticationURL = _config_data.get('AuthenticationURL', AuthenticationURL) + SciDriveHost = _config_data.get('SciDriveHost', SciDriveHost) + SkyQueryUrl = _config_data.get('SkyQueryUrl', SkyQueryUrl) + SkyServerWSurl = _config_data.get('SkyServerWSurl', SkyServerWSurl) + RacmApiURL = _config_data.get('RacmApiURL', RacmApiURL) + DataRelease = _config_data.get('DataRelease', DataRelease) + KeystoneTokenPath = _config_data.get('KeystoneTokenPath', KeystoneTokenPath) + version = _config_data.get('version', version) + ComputeJobDirectoryFile = _config_data.get('ComputeJobDirectoryFile', ComputeJobDirectoryFile) + # returns TRUE if the library is running inside the SciServer-Compute, and FALSE if not -import os; def isSciServerComputeEnvironment(): """ Checks whether the library is being run within the SciServer-Compute environment. From b2c8c436ad477ae7c94962698a22e8ee7d0475ef Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Mon, 18 Jun 2018 10:52:27 -0400 Subject: [PATCH 38/79] add checking of rootVolume and owner to match user volumes --- py2/SciServer/Jobs.py | 2 +- py3/SciServer/Jobs.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py index 2a97fb1..69549d1 100644 --- a/py2/SciServer/Jobs.py +++ b/py2/SciServer/Jobs.py @@ -342,7 +342,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No for uVol in userVolumes: found = False; for vol in dockerComputeDomain.get('userVolumes'): - if vol.get('name') == uVol.get('name'): + if vol.get('name') == uVol.get('name') and vol.get('rootVolumeName') == uVol.get('rootVolumeName') and vol.get('owner') == uVol.get('owner'): found = True; if (uVol.has_key('needsWriteAccess')): if uVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'): diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py index 80731a9..c12b497 100644 --- a/py3/SciServer/Jobs.py +++ b/py3/SciServer/Jobs.py @@ -342,7 +342,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No for uVol in userVolumes: found = False; for vol in dockerComputeDomain.get('userVolumes'): - if vol.get('name') == uVol.get('name'): + if vol.get('name') == uVol.get('name') and vol.get('rootVolumeName') == uVol.get('rootVolumeName') and vol.get('owner') == uVol.get('owner'): found = True; if (uVol.get('needsWriteAccess')): if uVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'): From a8a5418842930df95eca3d6ca5c908c7ac1931dd Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Mon, 25 Jun 2018 13:47:59 -0400 Subject: [PATCH 39/79] update sciserver version --- docs_sphinx/SciServer.rst | 17 +++++++++++++++++ docs_sphinx/conf.py | 8 ++++---- py2/SciServer/Config.py | 4 ++-- py2/SciServer/__init__.py | 2 +- py2/setup.py | 2 +- py3/SciServer/Config.py | 4 ++-- py3/SciServer/__init__.py | 2 +- py3/setup.py | 2 +- 8 files changed, 29 insertions(+), 12 deletions(-) diff --git a/docs_sphinx/SciServer.rst b/docs_sphinx/SciServer.rst index 27d717d..d06fdf3 100644 --- a/docs_sphinx/SciServer.rst +++ b/docs_sphinx/SciServer.rst @@ -61,3 +61,20 @@ SciServer.SkyQuery module :members: :undoc-members: :show-inheritance: + +SciServer.Files module +------------------------ + +.. automodule:: SciServer.Files + :members: + :undoc-members: + :show-inheritance: + +SciServer.Jobs module +------------------------ + +.. automodule:: SciServer.Jobs + :members: + :undoc-members: + :show-inheritance: + diff --git a/docs_sphinx/conf.py b/docs_sphinx/conf.py index 0d13936..ba2c023 100644 --- a/docs_sphinx/conf.py +++ b/docs_sphinx/conf.py @@ -48,16 +48,16 @@ # General information about the project. project = 'SciScript-Python' -copyright = '2017, SciServer' +copyright = '2018, SciServer' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '1.10.0' +version = '2.0.0' # The full version, including alpha/beta/rc tags. -release = '1.10.0' +release = '2.0.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -268,7 +268,7 @@ epub_title = '.' epub_author = 'Gerard Lemson, Manuchehr Taghizadeh-Popp' epub_publisher = 'Gerard Lemson, Manuchehr Taghizadeh-Popp' -epub_copyright = '2016, SciServer' +epub_copyright = '2018, SciServer' # The basename for the epub file. It defaults to the project name. #epub_basename = '.' diff --git a/py2/SciServer/Config.py b/py2/SciServer/Config.py index 504342e..e32d601 100644 --- a/py2/SciServer/Config.py +++ b/py2/SciServer/Config.py @@ -29,9 +29,9 @@ SkyQueryUrl = "http://voservices.net/skyquery/Api/V1" SkyServerWSurl = "https://skyserver.sdss.org" RacmApiURL = "http://scitest12.pha.jhu.edu/racm" -DataRelease = "DR13" +DataRelease = "DR14" KeystoneTokenPath = "/home/idies/keystone.token" #the path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment -version = "sciserver-v1.10.0" #sciserver release version +version = "sciserver-v2.0.0" #sciserver release version ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. _CONFIG_DIR = os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')) diff --git a/py2/SciServer/__init__.py b/py2/SciServer/__init__.py index a7f4476..b6dc013 100644 --- a/py2/SciServer/__init__.py +++ b/py2/SciServer/__init__.py @@ -31,7 +31,7 @@ **Authors**: Gerard Lemson , Manuchehr Taghizadeh-Popp -**Version**: sciserver-v1.10.0 +**Version**: sciserver-v2.0.0 """ diff --git a/py2/setup.py b/py2/setup.py index 33d3959..a968c68 100644 --- a/py2/setup.py +++ b/py2/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages setup( name = "SciServer", - version = "1.10.0", + version = "2.0.0", packages = find_packages(), ) \ No newline at end of file diff --git a/py3/SciServer/Config.py b/py3/SciServer/Config.py index 504342e..e32d601 100644 --- a/py3/SciServer/Config.py +++ b/py3/SciServer/Config.py @@ -29,9 +29,9 @@ SkyQueryUrl = "http://voservices.net/skyquery/Api/V1" SkyServerWSurl = "https://skyserver.sdss.org" RacmApiURL = "http://scitest12.pha.jhu.edu/racm" -DataRelease = "DR13" +DataRelease = "DR14" KeystoneTokenPath = "/home/idies/keystone.token" #the path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment -version = "sciserver-v1.10.0" #sciserver release version +version = "sciserver-v2.0.0" #sciserver release version ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. _CONFIG_DIR = os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')) diff --git a/py3/SciServer/__init__.py b/py3/SciServer/__init__.py index a7f4476..b6dc013 100644 --- a/py3/SciServer/__init__.py +++ b/py3/SciServer/__init__.py @@ -31,7 +31,7 @@ **Authors**: Gerard Lemson , Manuchehr Taghizadeh-Popp -**Version**: sciserver-v1.10.0 +**Version**: sciserver-v2.0.0 """ diff --git a/py3/setup.py b/py3/setup.py index 33d3959..a968c68 100644 --- a/py3/setup.py +++ b/py3/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages setup( name = "SciServer", - version = "1.10.0", + version = "2.0.0", packages = find_packages(), ) \ No newline at end of file From 22a933a0ed31a104679610c19d9a0c05c84e6713 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Mon, 25 Jun 2018 13:54:26 -0400 Subject: [PATCH 40/79] update volume names in example notebook --- Examples/Examples_SciScript-Python.ipynb | 576 +++++++---------------- 1 file changed, 180 insertions(+), 396 deletions(-) diff --git a/Examples/Examples_SciScript-Python.ipynb b/Examples/Examples_SciScript-Python.ipynb index 150bd55..9783054 100644 --- a/Examples/Examples_SciScript-Python.ipynb +++ b/Examples/Examples_SciScript-Python.ipynb @@ -4,7 +4,6 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": false, "scrolled": true }, "outputs": [], @@ -32,9 +31,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#help(SciServer)" @@ -43,9 +40,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# *******************************************************************************************************\n", @@ -56,9 +51,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#help(Authentication)" @@ -67,9 +60,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#logging in and getting current token from different ways\n", @@ -87,9 +78,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#getting curent user info\n", @@ -104,9 +93,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#reseting the current token to another value:\n", @@ -123,9 +110,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "#logging-in again\n", @@ -136,9 +121,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# *******************************************************************************************************\n", @@ -149,9 +132,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "#help(LoginPortal)" @@ -160,9 +141,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#logging in and getting current token from different ways\n", @@ -178,9 +157,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#getting curent user info\n", @@ -195,9 +172,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#reseting the current token to another value:\n", @@ -210,9 +185,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#logging-in again\n", @@ -223,9 +196,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# *******************************************************************************************************\n", @@ -236,9 +207,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "#help(CasJobs)" @@ -247,9 +216,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "#Defining databse context and query, and other variables\n", @@ -266,9 +233,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#get user schema info\n", @@ -280,9 +245,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#get info about tables inside MyDB database context:\n", @@ -294,9 +257,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#execute a quick SQL query:\n", @@ -308,9 +269,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#submit a job, which inserts the query results into a table in the MyDB database context. \n", @@ -325,9 +284,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# drop or delete table in MyDB database context\n", @@ -339,9 +296,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#get job status\n", @@ -355,9 +310,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#cancel a job\n", @@ -371,9 +324,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#execute a query and write a local Fits file containing the query results:\n", @@ -385,9 +336,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#delete local FITS file just created:\n", @@ -398,9 +347,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#get a Pandas dataframe containing the results of a query\n", @@ -412,9 +359,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# get numpy array containing the results of a query\n", @@ -426,9 +371,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#uploads a Pandas dataframe into a Database table\n", @@ -443,9 +386,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# drop or delete table just created:\n", @@ -457,9 +398,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#upload csv data string into a database table:\n", @@ -473,9 +412,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# drop or delete table just created:\n", @@ -487,9 +424,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# *******************************************************************************************************\n", @@ -500,9 +435,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#help(SkyServer)" @@ -511,9 +444,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "#defining sql query and SDSS data relelease:\n", @@ -525,9 +456,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#Exectute sql query\n", @@ -539,9 +468,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#get an image cutout\n", @@ -555,9 +482,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# do a radial search of objects:\n", @@ -569,9 +494,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#do rectangular search of objects:\n", @@ -583,9 +506,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#do an object search based on RA,Dec coordinates:\n", @@ -597,9 +518,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# *******************************************************************************************************\n", @@ -610,9 +529,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "#help(SciDrive)" @@ -621,9 +538,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#list content and metadata of top level directory in SciDrive\n", @@ -635,9 +550,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "#define name of directory to be created in SciDrive:\n", @@ -651,9 +564,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#create a folder or container in SciDrive\n", @@ -665,9 +576,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#list content and metadata of directory in SciDrive\n", @@ -679,9 +588,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#get the public url to access the directory content in SciDrive\n", @@ -693,9 +600,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#Delete folder or container in SciDrive:\n", @@ -707,9 +612,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#create a local file:\n", @@ -722,9 +625,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#uploading a file to SciDrive:\n", @@ -736,9 +637,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#download file:\n", @@ -751,9 +650,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#upload string data:\n", @@ -766,9 +663,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#delete folder in SciDrive:\n", @@ -780,9 +675,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#delete local file:\n", @@ -793,9 +686,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# *******************************************************************************************************\n", @@ -806,9 +697,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "#help(SkyQuery)" @@ -817,9 +706,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword);" @@ -828,9 +715,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#list all databses or datasets available\n", @@ -842,9 +727,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#get info about the user's personal database or dataset\n", @@ -856,9 +739,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#list tables inside dataset\n", @@ -870,9 +751,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#list available job queues\n", @@ -884,9 +763,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#list available job queues and related info\n", @@ -900,9 +777,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "#Define query\n", @@ -913,9 +788,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#submit a query as a job\n", @@ -927,9 +800,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#get status of a submitted job\n", @@ -942,9 +813,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# wait for a job to be finished and then get the status\n", @@ -958,9 +827,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# cancel a job that is running, and then get its status\n", @@ -975,9 +842,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# get list of jobs\n", @@ -991,9 +856,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "#define csv table to be uploaded to into MyDB in SkyQuery\n", @@ -1005,9 +868,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#uploading the csv table:\n", @@ -1019,9 +880,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#downloading table:\n", @@ -1033,9 +892,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#list tables inside dataset\n", @@ -1047,9 +904,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#get dataset table info:\n", @@ -1061,9 +916,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#get dataset table columns info\n", @@ -1075,9 +928,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#drop (or delete) table from dataset.\n", @@ -1089,9 +940,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# *******************************************************************************************************\n", @@ -1102,9 +951,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#help(Files)" @@ -1113,17 +960,15 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# defining the FileService name\n", "\n", "Files_FileServiceName = \"FileServiceJHU\"\n", - "Files_RootVolumeName1 = \"persistent\"\n", + "Files_RootVolumeName1 = \"Storage\"\n", "Files_UserVolumeName1 = \"UserVolume555\"\n", - "Files_RootVolumeName2 = \"persistent\"\n", + "Files_RootVolumeName2 = \"Storage\"\n", "Files_UserVolumeName2 = \"UserVolume999\"\n", "Files_NewDirectoryName1 = \"myNewDirectory555\"\n", "Files_NewDirectoryName2 = \"myNewDirectory999\"\n", @@ -1134,9 +979,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword);" @@ -1145,9 +988,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# get available FileServices\n", @@ -1159,9 +1000,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# get names of file services available to the user.\n", @@ -1173,13 +1012,11 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# get FileService from Name\n", - "Files_FileServiceName = 'NewFileServiceScitest12_1 INSECURE'\n", + "Files_FileServiceName = 'NewFileServiceScitest12 INSECURE'\n", "fileService = Files.getFileServiceFromName(Files_FileServiceName);\n", "fileService" ] @@ -1187,9 +1024,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# get the API endpoint URL of a FileService\n", @@ -1201,9 +1036,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# get root volumes\n", @@ -1215,9 +1048,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# get root volumes\n", @@ -1229,9 +1060,16 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, + "outputs": [], + "source": [ + "str(True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "# create user volume\n", @@ -1245,9 +1083,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# create a directory in the persistent volume\n", @@ -1261,9 +1097,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# upload a text string into a file in the remote directory\n", @@ -1276,9 +1110,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# List content of remote directory\n", @@ -1292,9 +1124,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# download a remote text file into a local directory\n", @@ -1307,9 +1137,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# Delete remote file\n", @@ -1321,9 +1149,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# upload a local file into a remote directory\n", @@ -1336,9 +1162,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# copy remote file into remote directory\n", @@ -1353,9 +1177,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# sharing user volume with another user\n", @@ -1368,9 +1190,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# let the other user log-in\n", @@ -1381,9 +1201,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# the other user downloads the remote text file (from the shared user volume) into a local string variable\n", @@ -1397,9 +1215,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# the original user logs in again\n", @@ -1410,9 +1226,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# delete folders\n", @@ -1426,9 +1240,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# delete user volumes\n", @@ -1442,9 +1254,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# *******************************************************************************************************\n", @@ -1455,9 +1265,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#help(Jobs)" @@ -1466,22 +1274,21 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "Jobs_DockerComputeDomainName = 'Small Jobs Batch Domain'\n", "\n", - "Jobs_FileServiceName = \"NewFileServiceScitest12_1 INSECURE\"\n", - "Jobs_RootVolumeName = \"persistent\"\n", + "Jobs_FileServiceName = \"NewFileServiceScitest12 INSECURE\"\n", + "Jobs_RootVolumeName = \"Storage\"\n", "Jobs_UserVolumeName = \"JobsTestVolume\"\n", "Jobs_DirectoryName = \"JobsTestDirectory\"\n", "Jobs_NotebookName = 'TestNotebook.ipynb'\n", "Jobs_ShellCommand = 'ls -laht'\n", "Jobs_DockerImageName = 'Python (astro)'\n", "\n", - "Jobs_UserVolumes = [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}]\n", + "Jobs_UserVolumes = [{'name':Jobs_UserVolumeName, 'rootVolumeName':Jobs_RootVolumeName, 'owner':Authentication_loginName, 'needsWriteAccess':True},\n", + " {'name':'scratch', 'rootVolumeName':'Temporary', 'owner':Authentication_loginName, 'needsWriteAccess':True}]\n", "Jobs_DataVolumes = [{'name':'SDSS_DAS'}]\n", "Jobs_Parameters = 'param1=1\\nparam2=2\\nparam3=3'\n", "Jobs_Alias = 'MyNewJob'\n", @@ -1495,9 +1302,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# get docker compute domains\n", @@ -1509,9 +1314,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# get names of docker compute domains available to the user\n", @@ -1523,23 +1326,19 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# get docker compute domain from name\n", "\n", "dockerComputeDomain = Jobs.getDockerComputeDomainFromName(Jobs_DockerComputeDomainName)\n", - "print(dockerComputeDomain)" + "dockerComputeDomain" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# uploading Jupyter notebook to remote directory\n", @@ -1555,39 +1354,27 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# submit a Jupyter notebook job.\n", "\n", - "jobId = Jobs.submitNotebookJob('/home/idies/workspace/' + \n", - "\"/\".join([Jobs_RootVolumeName,Authentication_loginName,Jobs_UserVolumeName,Jobs_DirectoryName,Jobs_NotebookName]),\n", - " dockerComputeDomain,\n", - " Jobs_DockerImageName, \n", - " Jobs_UserVolumes, Jobs_DataVolumes,\n", - " Jobs_Parameters, Jobs_Alias)\n", + "jobId = Jobs.submitNotebookJob(notebookPath='',#'/home/idies/workspace/' + \n", + "#\"/\".join([Jobs_RootVolumeName,Authentication_loginName,Jobs_UserVolumeName,Jobs_DirectoryName,Jobs_NotebookName]),\n", + " dockerComputeDomain=dockerComputeDomain,\n", + " dockerImageName=Jobs_DockerImageName,\n", + " userVolumes=Jobs_UserVolumes,\n", + " dataVolumes=Jobs_DataVolumes,\n", + " resultsFolderPath=\"\",\n", + " parameters=Jobs_Parameters,\n", + " jobAlias=Jobs_Alias)\n", "print(jobId)" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "'/home/idies/workspace/' + \"/\".join([Jobs_RootVolumeName,Authentication_loginName,Jobs_UserVolumeName,Jobs_DirectoryName,Jobs_NotebookName])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# get job status\n", @@ -1599,9 +1386,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# get job description\n", @@ -1613,18 +1398,17 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# wait until job is finished and get job status\n", "\n", - "jobId = Jobs.submitNotebookJob('/home/idies/workspace/' + Jobs_UserVolumeName + '/' + Jobs_DirectoryName + '/' + Jobs_NotebookName,\n", - " dockerComputeDomain,\n", - " Jobs_DockerImageName, \n", - " Jobs_UserVolumes, Jobs_DataVolumes,\n", - " Jobs_Parameters, Jobs_Alias)\n", + "jobId = Jobs.submitNotebookJob(notebookPath='/home/idies/workspace/' + Jobs_UserVolumeName + '/' + Jobs_DirectoryName + '/' + Jobs_NotebookName,\n", + " dockerComputeDomain=dockerComputeDomain,\n", + " dockerImageName=Jobs_DockerImageName, \n", + " userVolumes=Jobs_UserVolumes, dataVolumes=Jobs_DataVolumes,\n", + " resultsFolderPath=\"\",\n", + " parameters=Jobs_Parameters, jobAlias=Jobs_Alias)\n", "jobStatus = Jobs.waitForJob(jobId)\n", "print(jobStatus)" ] @@ -1632,27 +1416,24 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# cancel job\n", "\n", "jobId = Jobs.submitNotebookJob('/home/idies/workspace/' + Jobs_UserVolumeName + '/' + Jobs_DirectoryName + '/' + Jobs_NotebookName,\n", - " dockerComputeDomain,\n", - " Jobs_DockerImageName, \n", - " Jobs_UserVolumes, Jobs_DataVolumes,\n", - " Jobs_Parameters, Jobs_Alias)\n", + " dockerComputeDomain=dockerComputeDomain,\n", + " dockerImageName=Jobs_DockerImageName, \n", + " userVolumes=Jobs_UserVolumes, dataVolumes=Jobs_DataVolumes,\n", + " resultsFolderPath=\"\",\n", + " parameters=Jobs_Parameters, jobAlias=Jobs_Alias)\n", "Jobs.cancelJob(jobId)" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "Files.deleteUserVolume(fileService, \"/\".join([Jobs_RootVolumeName,Authentication_loginName,Jobs_UserVolumeName]), quiet=True)" @@ -1661,26 +1442,25 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# submit shell command job\n", "\n", - "jobId = Jobs.submitShellCommandJob(Jobs_ShellCommand,\n", - " dockerComputeDomain,\n", - " Jobs_DockerImageName,\n", - " Jobs_UserVolumes, Jobs_DataVolumes,\n", - " Jobs_Alias)" + "jobId = Jobs.submitShellCommandJob(shellCommand=Jobs_ShellCommand,\n", + " dockerComputeDomain=dockerComputeDomain,\n", + " dockerImageName=Jobs_DockerImageName,\n", + " userVolumes=Jobs_UserVolumes, dataVolumes=Jobs_DataVolumes,\n", + " resultsFolderPath = \"\",\n", + " jobAlias=Jobs_Alias)\n", + "jobStatus = Jobs.waitForJob(jobId)\n", + "print(jobStatus)" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# get relational database (RDB) compute domains\n", @@ -1692,9 +1472,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# get names of relational database (RDB) compute domains\n", @@ -1706,9 +1484,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# get relational database (RDB) compute domain from name\n", @@ -1720,9 +1496,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# submit RDB (relatinal database) query job\n", @@ -1734,32 +1508,42 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, + "outputs": [], + "source": [ + "# get job description\n", + "\n", + "job = Jobs.getJobDescription(jobId)\n", + "print(job)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 2", + "display_name": "Python 3", "language": "python", - "name": "python2" + "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" + "pygments_lexer": "ipython3", + "version": "3.6.5" } }, "nbformat": 4, - "nbformat_minor": 0 + "nbformat_minor": 1 } From 70fbcaca65acc9ed92e832ff9085928cf8ae1faa Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Wed, 11 Jul 2018 09:20:36 -0400 Subject: [PATCH 41/79] remove deprecated getJobDirectory method in Jobs module --- py2/SciServer/Jobs.py | 19 ------------------- py3/SciServer/Jobs.py | 19 ------------------- 2 files changed, 38 deletions(-) diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py index 69549d1..41fa533 100644 --- a/py2/SciServer/Jobs.py +++ b/py2/SciServer/Jobs.py @@ -10,25 +10,6 @@ -def getJobDirectory(): - """ - Gets the root directory path where the job is being executed. Since its value depends on the value of jobId, the path will be defined only after the job is submitted. It can be called within Jupyter notebooks or shell commands running as jobs. - :return: a string defining the path to the job directory. - :raises: Throws an exception if job directory cannot be found. - :example: jobDirectory = getJobDirectory(); - .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getRDBComputeDomains, Jobs.cancelJob - """ - computeJobDirectoryFile = Config.ComputeJobDirectoryFile - if os.path.isfile(computeJobDirectoryFile): - jobDirectory = None; - with open(computeJobDirectoryFile, 'r') as f: - jobDirectory = f.read().rstrip('\n') - - return jobDirectory - else: - raise OSError("Cannot find job execution directory") - - def getDockerComputeDomains(): """ Gets a list of all registered Docker compute domains that the user has access to. diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py index c12b497..9548dd6 100644 --- a/py3/SciServer/Jobs.py +++ b/py3/SciServer/Jobs.py @@ -10,25 +10,6 @@ -def getJobDirectory(): - """ - Gets the root directory path where the job is being executed. Since its value depends on the value of jobId, the path will be defined only after the job is submitted. It can be called within Jupyter notebooks or shell commands running as jobs. - :return: a string defining the path to the job directory. - :raises: Throws an exception if job directory cannot be found. - :example: jobDirectory = getJobDirectory(); - .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getRDBComputeDomains, Jobs.cancelJob - """ - computeJobDirectoryFile = Config.ComputeJobDirectoryFile - if os.path.isfile(computeJobDirectoryFile): - jobDirectory = None; - with open(computeJobDirectoryFile, 'r') as f: - jobDirectory = f.read().rstrip('\n') - - return jobDirectory - else: - raise OSError("Cannot find job execution directory") - - def getDockerComputeDomains(): """ Gets a list of all registered Docker compute domains that the user has access to. From 438a6153c41056632a33588cdf683c601c94b07a Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Thu, 12 Jul 2018 19:22:07 -0400 Subject: [PATCH 42/79] added missing field to the user volume info in py2 --- py2/SciServer/Files.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/py2/SciServer/Files.py b/py2/SciServer/Files.py index 24edaf4..80fbfe1 100644 --- a/py2/SciServer/Files.py +++ b/py2/SciServer/Files.py @@ -177,12 +177,13 @@ def getUserVolumesInfo(fileService, rootVolumeName = None, verbose=True): userVolumes = []; for rootVolume in fileService.get("rootVolumes"): for userVolume in rootVolume.get('userVolumes'): + path = os.path.join(rootVolume.get('name'), userVolume.get('owner'), userVolume.get('name')) if rootVolumeName is not None: if rootVolume.get('name') == rootVolumeName: - userVolumes.append({"userVolumeName": userVolume.get('name'), "userVolumeDescription": userVolume.get('description'), "rootVolumeName":rootVolume.get("name"), "rootVolumeDescription":rootVolume.get("description")}) + userVolumes.append({"userVolumeName": userVolume.get('name'), "path":path, "userVolumeDescription": userVolume.get('description'), "rootVolumeName":rootVolume.get("name"), "rootVolumeDescription":rootVolume.get("description")}) else: - userVolumes.append({"userVolumeName": userVolume.get('name'), "userVolumeDescription": userVolume.get('description'),"rootVolumeName": rootVolume.get("name"), "rootVolumeDescription": rootVolume.get("description")}) + userVolumes.append({"userVolumeName": userVolume.get('name'), "path":path, "userVolumeDescription": userVolume.get('description'),"rootVolumeName": rootVolume.get("name"), "rootVolumeDescription": rootVolume.get("description")}) return userVolumes From 269450749c20f48a92587a4f17c9f4df605132c3 Mon Sep 17 00:00:00 2001 From: Joseph Jon Booker Date: Fri, 13 Jul 2018 10:52:51 -0400 Subject: [PATCH 43/79] Load configs from /etc/sciserver/sciscript.json if it exists This file overrides any values built into Config.py, but is overridable by ~/.config/sciserver/sciscript.json (if it exists) or at runtime --- py2/SciServer/Config.py | 20 ++++++++++++++++++++ py3/SciServer/Config.py | 32 ++++++++++++++++++-------------- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/py2/SciServer/Config.py b/py2/SciServer/Config.py index e32d601..28a35ea 100644 --- a/py2/SciServer/Config.py +++ b/py2/SciServer/Config.py @@ -50,6 +50,26 @@ version = _config_data.get('version', version) ComputeJobDirectoryFile = _config_data.get('ComputeJobDirectoryFile', ComputeJobDirectoryFile) +def _load_config(filename): + if os.path.exists(filename): + with open(filename) as f: + _config_data = json.load(f) + CasJobsRESTUri = _config_data.get('CasJobsRESTUri', CasJobsRESTUri) + AuthenticationURL = _config_data.get('AuthenticationURL', AuthenticationURL) + SciDriveHost = _config_data.get('SciDriveHost', SciDriveHost) + SkyQueryUrl = _config_data.get('SkyQueryUrl', SkyQueryUrl) + SkyServerWSurl = _config_data.get('SkyServerWSurl', SkyServerWSurl) + RacmApiURL = _config_data.get('RacmApiURL', RacmApiURL) + DataRelease = _config_data.get('DataRelease', DataRelease) + KeystoneTokenPath = _config_data.get('KeystoneTokenPath', KeystoneTokenPath) + version = _config_data.get('version', version) + ComputeJobDirectoryFile = _config_data.get('ComputeJobDirectoryFile', ComputeJobDirectoryFile) + +_CONFIG_DIR = os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')) +_SCISERVER_SYSTEM_CONFIG_DIR = '/etc/' # will not likely exist on non *nix systems +for config_dir in [_SCISERVER_SYSTEM_CONFIG_DIR, _CONFIG_DIR]: + _load_config(os.path.join(config_dir, 'sciserver', 'sciscript.json')) + # returns TRUE if the library is running inside the SciServer-Compute, and FALSE if not def isSciServerComputeEnvironment(): """ diff --git a/py3/SciServer/Config.py b/py3/SciServer/Config.py index e32d601..0a81abc 100644 --- a/py3/SciServer/Config.py +++ b/py3/SciServer/Config.py @@ -34,21 +34,25 @@ version = "sciserver-v2.0.0" #sciserver release version ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. +def _load_config(filename): + if os.path.exists(filename): + with open(filename) as f: + _config_data = json.load(f) + CasJobsRESTUri = _config_data.get('CasJobsRESTUri', CasJobsRESTUri) + AuthenticationURL = _config_data.get('AuthenticationURL', AuthenticationURL) + SciDriveHost = _config_data.get('SciDriveHost', SciDriveHost) + SkyQueryUrl = _config_data.get('SkyQueryUrl', SkyQueryUrl) + SkyServerWSurl = _config_data.get('SkyServerWSurl', SkyServerWSurl) + RacmApiURL = _config_data.get('RacmApiURL', RacmApiURL) + DataRelease = _config_data.get('DataRelease', DataRelease) + KeystoneTokenPath = _config_data.get('KeystoneTokenPath', KeystoneTokenPath) + version = _config_data.get('version', version) + ComputeJobDirectoryFile = _config_data.get('ComputeJobDirectoryFile', ComputeJobDirectoryFile) + _CONFIG_DIR = os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')) -_SCISERVER_CONFIG_FILE = os.path.join(_CONFIG_DIR, 'sciserver', 'sciscript.json') -if os.path.exists(_SCISERVER_CONFIG_FILE): - with open(_SCISERVER_CONFIG_FILE) as f: - _config_data = json.load(f) - CasJobsRESTUri = _config_data.get('CasJobsRESTUri', CasJobsRESTUri) - AuthenticationURL = _config_data.get('AuthenticationURL', AuthenticationURL) - SciDriveHost = _config_data.get('SciDriveHost', SciDriveHost) - SkyQueryUrl = _config_data.get('SkyQueryUrl', SkyQueryUrl) - SkyServerWSurl = _config_data.get('SkyServerWSurl', SkyServerWSurl) - RacmApiURL = _config_data.get('RacmApiURL', RacmApiURL) - DataRelease = _config_data.get('DataRelease', DataRelease) - KeystoneTokenPath = _config_data.get('KeystoneTokenPath', KeystoneTokenPath) - version = _config_data.get('version', version) - ComputeJobDirectoryFile = _config_data.get('ComputeJobDirectoryFile', ComputeJobDirectoryFile) +_SCISERVER_SYSTEM_CONFIG_DIR = '/etc/' # will not likely exist on non *nix systems +for config_dir in [_SCISERVER_SYSTEM_CONFIG_DIR, _CONFIG_DIR]: + _load_config(os.path.join(config_dir, 'sciserver', 'sciscript.json')) # returns TRUE if the library is running inside the SciServer-Compute, and FALSE if not def isSciServerComputeEnvironment(): From 4753a077020e854e45b15ec646c15aff955385d9 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Mon, 16 Jul 2018 13:03:35 -0400 Subject: [PATCH 44/79] allow for multiple query statements to be stored as array of Pandas dataframes in CasJobs.executeQuery --- py2/SciServer/CasJobs.py | 12 ++++++++++-- py3/SciServer/CasJobs.py | 10 +++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/py2/SciServer/CasJobs.py b/py2/SciServer/CasJobs.py index 6cb5e47..6937684 100644 --- a/py2/SciServer/CasJobs.py +++ b/py2/SciServer/CasJobs.py @@ -151,8 +151,16 @@ def executeQuery(sql, context="MyDB", format="pandas"): if (format == "readable") or (format == "StringIO"): return StringIO(postResponse.content.decode()) elif format == "pandas": - r=json.loads(postResponse.content.decode()) - return pandas.DataFrame(r['Result'][0]['Data'],columns=r['Result'][0]['Columns']) + r = json.loads(postResponse.content.decode()) + if len(r['Result']) > 1: + res = [] + for result in r['Result']: + res.append(pandas.DataFrame(result['Data'], columns=result['Columns'])) + + return res + else: + return pandas.DataFrame(r['Result'][0]['Data'], columns=r['Result'][0]['Columns']) + elif format == "csv": return postResponse.content.decode() elif format == "dict": diff --git a/py3/SciServer/CasJobs.py b/py3/SciServer/CasJobs.py index 1039776..896455c 100644 --- a/py3/SciServer/CasJobs.py +++ b/py3/SciServer/CasJobs.py @@ -152,7 +152,15 @@ def executeQuery(sql, context="MyDB", format="pandas"): return StringIO(postResponse.content.decode()) elif format == "pandas": r=json.loads(postResponse.content.decode()) - return pandas.DataFrame(r['Result'][0]['Data'],columns=r['Result'][0]['Columns']) + if len(r['Result']) > 1: + res = [] + for result in r['Result']: + res.append(pandas.DataFrame(result['Data'],columns=result['Columns'])) + + return res + else: + return pandas.DataFrame(r['Result'][0]['Data'],columns=r['Result'][0]['Columns']) + elif format == "csv": return postResponse.content.decode() elif format == "dict": From a5549cad424930a130cd1871caae9f932c83757b Mon Sep 17 00:00:00 2001 From: Joseph Jon Booker Date: Fri, 20 Jul 2018 12:06:39 -0400 Subject: [PATCH 45/79] Handle scoping correctly for overriding configs With the move to a function for loading seperate files, python's treatment of assignments as being only for the local function scope caused this code to throw errors --- py2/SciServer/Config.py | 2 ++ py3/SciServer/Config.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/py2/SciServer/Config.py b/py2/SciServer/Config.py index 28a35ea..abbc3d1 100644 --- a/py2/SciServer/Config.py +++ b/py2/SciServer/Config.py @@ -54,6 +54,8 @@ def _load_config(filename): if os.path.exists(filename): with open(filename) as f: _config_data = json.load(f) + global CasJobsRESTUri, AuthenticationURL, SciDriveHost, SkyQueryUrl, SkyServerWSurl + global RacmApiURL, DataRelease, KeystoneTokenPath, version, ComputeJobDirectoryFile CasJobsRESTUri = _config_data.get('CasJobsRESTUri', CasJobsRESTUri) AuthenticationURL = _config_data.get('AuthenticationURL', AuthenticationURL) SciDriveHost = _config_data.get('SciDriveHost', SciDriveHost) diff --git a/py3/SciServer/Config.py b/py3/SciServer/Config.py index 0a81abc..9bcbd2d 100644 --- a/py3/SciServer/Config.py +++ b/py3/SciServer/Config.py @@ -38,6 +38,8 @@ def _load_config(filename): if os.path.exists(filename): with open(filename) as f: _config_data = json.load(f) + global CasJobsRESTUri, AuthenticationURL, SciDriveHost, SkyQueryUrl, SkyServerWSurl + global RacmApiURL, DataRelease, KeystoneTokenPath, version, ComputeJobDirectoryFile CasJobsRESTUri = _config_data.get('CasJobsRESTUri', CasJobsRESTUri) AuthenticationURL = _config_data.get('AuthenticationURL', AuthenticationURL) SciDriveHost = _config_data.get('SciDriveHost', SciDriveHost) From 99ba54f4c86e6cb35f82985374f4fe4ced092f11 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Mon, 23 Jul 2018 09:12:34 -0400 Subject: [PATCH 46/79] add short description of new modules in README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index bc1b760..e7c6865 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,10 @@ Some SciServer tools you can access with this package: * [SkyServer](http://skyserver.sdss.org/): Access to the SDSS astronomical survey. * [SkyQuery](http://www.voservices.net/skyquery): Cross-match of astronomical source catalogs. + + * [Jobs](http://apps.sciserver.org/compute/jobs , http://apps.sciserver.org/sciquery-ui/): Synch and asynch submission of Jupyter notebooks, shell commands and SQL queries. + + * [Files](http://apps.sciserver.org/dashboard/files): Interact with the SciServer file system. Maintainer: Manuchehr Taghizadeh-Popp. From 4f2f954abc6938646236e695f2bec76481da482c Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Mon, 23 Jul 2018 09:22:17 -0400 Subject: [PATCH 47/79] fix URLs --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e7c6865..fa7a80f 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Some SciServer tools you can access with this package: * [SkyQuery](http://www.voservices.net/skyquery): Cross-match of astronomical source catalogs. - * [Jobs](http://apps.sciserver.org/compute/jobs , http://apps.sciserver.org/sciquery-ui/): Synch and asynch submission of Jupyter notebooks, shell commands and SQL queries. + * [Compute Jobs](http://apps.sciserver.org/compute/jobs) and [SciQuery Jobs](http://apps.sciserver.org/sciquery-ui/): Synch and asynch submission of Jupyter notebooks, shell commands and SQL queries. * [Files](http://apps.sciserver.org/dashboard/files): Interact with the SciServer file system. From 11470e011493c7974550ba03a57d8d94bb5a7b49 Mon Sep 17 00:00:00 2001 From: Joseph Jon Booker Date: Tue, 24 Jul 2018 14:18:13 -0400 Subject: [PATCH 48/79] Fix py2/py3 differences Extra code was left in python2's Config.py The content-type was missing for one api call in py2 --- py2/SciServer/Config.py | 16 ---------------- py2/SciServer/Files.py | 2 +- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/py2/SciServer/Config.py b/py2/SciServer/Config.py index abbc3d1..9bcbd2d 100644 --- a/py2/SciServer/Config.py +++ b/py2/SciServer/Config.py @@ -34,22 +34,6 @@ version = "sciserver-v2.0.0" #sciserver release version ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. -_CONFIG_DIR = os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')) -_SCISERVER_CONFIG_FILE = os.path.join(_CONFIG_DIR, 'sciserver', 'sciscript.json') -if os.path.exists(_SCISERVER_CONFIG_FILE): - with open(_SCISERVER_CONFIG_FILE) as f: - _config_data = json.load(f) - CasJobsRESTUri = _config_data.get('CasJobsRESTUri', CasJobsRESTUri) - AuthenticationURL = _config_data.get('AuthenticationURL', AuthenticationURL) - SciDriveHost = _config_data.get('SciDriveHost', SciDriveHost) - SkyQueryUrl = _config_data.get('SkyQueryUrl', SkyQueryUrl) - SkyServerWSurl = _config_data.get('SkyServerWSurl', SkyServerWSurl) - RacmApiURL = _config_data.get('RacmApiURL', RacmApiURL) - DataRelease = _config_data.get('DataRelease', DataRelease) - KeystoneTokenPath = _config_data.get('KeystoneTokenPath', KeystoneTokenPath) - version = _config_data.get('version', version) - ComputeJobDirectoryFile = _config_data.get('ComputeJobDirectoryFile', ComputeJobDirectoryFile) - def _load_config(filename): if os.path.exists(filename): with open(filename) as f: diff --git a/py2/SciServer/Files.py b/py2/SciServer/Files.py index 80fbfe1..c9a4c8e 100644 --- a/py2/SciServer/Files.py +++ b/py2/SciServer/Files.py @@ -598,7 +598,7 @@ def shareUserVolume(fileService, path, sharedWith, allowedActions, type="USER"): url = __getFileServiceAPIUrl(fileService) + "api/share/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?TaskName="+taskName - headers = {'X-Auth-Token': token} + headers = {'X-Auth-Token': token,'Content-Type':'application/json'} res = requests.patch(url, headers=headers, data=body) if res.status_code >= 200 and res.status_code < 300: From 6042253004f8098edb0a07deeb3ab3af07365648 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Fri, 3 Aug 2018 10:42:01 -0400 Subject: [PATCH 49/79] update URLs in Config file --- py2/SciServer/Config.py | 4 ++-- py3/SciServer/Config.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/py2/SciServer/Config.py b/py2/SciServer/Config.py index 9bcbd2d..0ca212b 100644 --- a/py2/SciServer/Config.py +++ b/py2/SciServer/Config.py @@ -24,11 +24,11 @@ """ # URLs for accessing SciServer web services (API endpoints) CasJobsRESTUri = "https://skyserver.sdss.org/CasJobs/RestApi" -AuthenticationURL = "http://scitest12.pha.jhu.edu/login-portal/keystone/v3/tokens" +AuthenticationURL = "https://apps.sciserver.org/login-portal/keystone/v3/tokens" SciDriveHost = "https://www.scidrive.org" SkyQueryUrl = "http://voservices.net/skyquery/Api/V1" SkyServerWSurl = "https://skyserver.sdss.org" -RacmApiURL = "http://scitest12.pha.jhu.edu/racm" +RacmApiURL = "https://apps.sciserver.org/racm" DataRelease = "DR14" KeystoneTokenPath = "/home/idies/keystone.token" #the path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment version = "sciserver-v2.0.0" #sciserver release version diff --git a/py3/SciServer/Config.py b/py3/SciServer/Config.py index 9bcbd2d..0ca212b 100644 --- a/py3/SciServer/Config.py +++ b/py3/SciServer/Config.py @@ -24,11 +24,11 @@ """ # URLs for accessing SciServer web services (API endpoints) CasJobsRESTUri = "https://skyserver.sdss.org/CasJobs/RestApi" -AuthenticationURL = "http://scitest12.pha.jhu.edu/login-portal/keystone/v3/tokens" +AuthenticationURL = "https://apps.sciserver.org/login-portal/keystone/v3/tokens" SciDriveHost = "https://www.scidrive.org" SkyQueryUrl = "http://voservices.net/skyquery/Api/V1" SkyServerWSurl = "https://skyserver.sdss.org" -RacmApiURL = "http://scitest12.pha.jhu.edu/racm" +RacmApiURL = "https://apps.sciserver.org/racm" DataRelease = "DR14" KeystoneTokenPath = "/home/idies/keystone.token" #the path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment version = "sciserver-v2.0.0" #sciserver release version From 6c6e374fbb40ec28ec96ea6d7c9d9d8ef43c9866 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Thu, 9 Aug 2018 11:07:05 -0400 Subject: [PATCH 50/79] update gitignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f7b1acc..d75e6eb 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,6 @@ /py3/dist /py3/SciServer.egg-info /py3/.idea -/Examples/.ipynb_checkpoints/ \ No newline at end of file +/Examples/.ipynb_checkpoints/ +/docs_sphinx/SciServer.egg-info +/docs_sphinx/dist \ No newline at end of file From cca899a50bc9fa9cabfdc238ffa3c87c3b3824bc Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Thu, 9 Aug 2018 11:07:28 -0400 Subject: [PATCH 51/79] update docuemntation --- docs_sphinx/SciServer.rst | 17 +++++++------- docs_sphinx/index.rst | 2 +- py2/SciServer/Authentication.py | 1 + py2/SciServer/Files.py | 38 +++++++++++++++++++++++++++++--- py2/SciServer/Jobs.py | 39 ++++++++++++++++++++++++++------- py2/SciServer/SkyQuery.py | 4 ++-- py2/SciServer/__init__.py | 20 ++++++++++++----- py3/SciServer/Authentication.py | 1 + py3/SciServer/Files.py | 38 +++++++++++++++++++++++++++++--- py3/SciServer/Jobs.py | 39 ++++++++++++++++++++++++++------- py3/SciServer/SkyQuery.py | 4 ++-- py3/SciServer/__init__.py | 20 ++++++++++++----- 12 files changed, 178 insertions(+), 45 deletions(-) diff --git a/docs_sphinx/SciServer.rst b/docs_sphinx/SciServer.rst index d06fdf3..bb3bc03 100644 --- a/docs_sphinx/SciServer.rst +++ b/docs_sphinx/SciServer.rst @@ -15,7 +15,7 @@ SciServer.CasJobs module :show-inheritance: SciServer.Config module ------------------------- +----------------------- .. automodule:: SciServer.Config :members: @@ -23,7 +23,7 @@ SciServer.Config module :show-inheritance: SciServer.Authentication module ------------------------- +------------------------------- .. automodule:: SciServer.Authentication :members: @@ -31,7 +31,7 @@ SciServer.Authentication module :show-inheritance: SciServer.LoginPortal module ------------------------- +---------------------------- .. automodule:: SciServer.LoginPortal :members: @@ -39,7 +39,7 @@ SciServer.LoginPortal module :show-inheritance: SciServer.SciDrive module ------------------------- +------------------------- .. automodule:: SciServer.SciDrive :members: @@ -47,7 +47,7 @@ SciServer.SciDrive module :show-inheritance: SciServer.SkyServer module ------------------------- +-------------------------- .. automodule:: SciServer.SkyServer :members: @@ -55,7 +55,7 @@ SciServer.SkyServer module :show-inheritance: SciServer.SkyQuery module ------------------------- +------------------------- .. automodule:: SciServer.SkyQuery :members: @@ -63,7 +63,7 @@ SciServer.SkyQuery module :show-inheritance: SciServer.Files module ------------------------- +---------------------- .. automodule:: SciServer.Files :members: @@ -71,10 +71,11 @@ SciServer.Files module :show-inheritance: SciServer.Jobs module ------------------------- +--------------------- .. automodule:: SciServer.Jobs :members: :undoc-members: :show-inheritance: + diff --git a/docs_sphinx/index.rst b/docs_sphinx/index.rst index 8ff9b57..e1ec1f7 100644 --- a/docs_sphinx/index.rst +++ b/docs_sphinx/index.rst @@ -4,7 +4,7 @@ contain the root `toctree` directive. Welcome to SciScript Python documentation! -============================= +========================================== Contents: diff --git a/py2/SciServer/Authentication.py b/py2/SciServer/Authentication.py index 73e7ca0..abe07cd 100644 --- a/py2/SciServer/Authentication.py +++ b/py2/SciServer/Authentication.py @@ -86,6 +86,7 @@ def login(UserName=None, Password=None): :return: authentication token (string) :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. :example: token1 = Authentication.login(); token2 = Authentication.login('loginName','loginPassword'); + .. seealso:: Authentication.getKeystoneUserWithToken, Authentication.getToken, Authentication.setToken, Authentication.token. """ taskName = "" diff --git a/py2/SciServer/Files.py b/py2/SciServer/Files.py index c9a4c8e..ee4384c 100644 --- a/py2/SciServer/Files.py +++ b/py2/SciServer/Files.py @@ -12,10 +12,12 @@ def getFileServices(verbose=True): """ Gets the definitions of file services that a user is able to access. A FileService represents a file system that contains root volumes accessible to the user for public/private data storage. Within each rootVolume, users can create sharable userVolumes for storing files. + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). :return: list of dictionaries, where each dictionary represents the description of a FileService that the user is able to access. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. :example: fileServices = Files.getFileServices(); + .. seealso:: Files.getFileServiceFromName """ token = Authentication.getToken() @@ -59,12 +61,14 @@ def getFileServices(verbose=True): def getFileServicesNames(fileServices=None, verbose=True): """ - Returns the names and description of the fileServices available to the user.l + Returns the names and description of the fileServices available to the user. + :param fileServices: a list of FileService objects (dictionaries), as returned by Files.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). :return: an array of dicts, where each dict has the name and description of a file service available to the user. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. :example: fileServiceNames = Files.getFileServicesNames(); + .. seealso:: Files.getFileServices """ @@ -84,12 +88,14 @@ def getFileServicesNames(fileServices=None, verbose=True): def getFileServiceFromName(fileServiceName, fileServices=None, verbose=True): """ Returns a FileService object, given its registered name. + :param fileServiceName: name of the FileService, as shown within the results of Files.getFileServices() :param fileServices: a list of FileService objects (dictionaries), as returned by Files.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). - :return a FileService object (dictionary) that defines a FileService. A list of these kind of objects available to the user is returned by the function Jobs.getFileServices(). If no fileService can be found, then returns None. - :raises Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :return: a FileService object (dictionary) that defines a FileService. A list of these kind of objects available to the user is returned by the function Jobs.getFileServices(). If no fileService can be found, then returns None. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. :example: fileService = Files.getFileServiceFromName('FileServiceAtJHU'); + .. seealso:: Files.getFileServices """ @@ -118,9 +124,11 @@ def getFileServiceFromName(fileServiceName, fileServices=None, verbose=True): def __getFileServiceAPIUrl(fileService): """ Gets the API endpoint URL of a FileService. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :return: API endpoint URL of the FileService (string). :example: fileServiceAPIUrl = Files.__getFileServiceAPIUrl(); + .. seealso:: Files.getFileServiceFromName """ @@ -142,11 +150,13 @@ def __getFileServiceAPIUrl(fileService): def getRootVolumesInfo(fileService, verbose=True): """ Gets the names and descriptions of root volumes available to the user in a particular FileService. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). :return: list of dictionaries, where each dictionary contains the name and description of a root volume. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. :example: fileServices = Files.getFileServices(); rootVolumesInfo = getRootVolumesInfo(fileServices[0]) + .. seealso:: Files.getUserVolumesInfo """ if type(fileService) == str: @@ -163,12 +173,14 @@ def getRootVolumesInfo(fileService, verbose=True): def getUserVolumesInfo(fileService, rootVolumeName = None, verbose=True): """ Gets the names definitions the RootVolumes available in a particular FileService, and of the file services that a user is able to access. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param rootVolumeName: name of root Volume (string) for which the user volumes are fetched. If set to None, then user volumes in all root folders are fetched. :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). :return: list of dictionaries, where each dictionary contains the name and description of a root volumes that a user is able to access. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. :example: fileServices = Files.getFileServices(); rootVolumeNames = Files.getUserVolumesInfo(fileServices[0]) + .. seealso:: Files.getRootVolumes, """ if type(fileService) == str: @@ -193,10 +205,12 @@ def getUserVolumesInfo(fileService, rootVolumeName = None, verbose=True): def splitPath(path): """ Splits a path of the form rootVolume/userVolumeOwner/userVolume/relativePath/... into its 4 components: rootVolume, userVolumeOwner, userVolume, and relativePath. + :param path: file system path (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/relativePath... :return: a tuple containing the four components: (rootVolume, userVolumeOwner, userVolume, relativePath) :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); Files.createUserVolume(fileServices[0], "volumes","newUserVolume"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName """ if path.startswith("/"): @@ -216,11 +230,13 @@ def splitPath(path): def createUserVolume(fileService, path, quiet=True): """ Create a user volume. + :param fileService: name of fileService (string), or object (dictionary) that defines the file service that contains the user volume. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param path: path (in the remote file service) to the user volume (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume :param quiet: if set to False, will throw an error if the User Volume already exists. If True, won't throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); Files.createUserVolume(fileServices[0], "volumes","newUserVolume"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList """ token = Authentication.getToken() @@ -252,11 +268,13 @@ def createUserVolume(fileService, path, quiet=True): def deleteUserVolume(fileService, path, quiet=True): """ Delete a user volume. + :param fileService: name of fileService (string), or object (dictionary) that defines the file service that contains the root volume. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param path: path (in the remote file service) to the user volume (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume :param quiet: If set to False, it will throw an error if user volume does not exists. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); Files.deleteUserVolume("volumes","newUserVolume",fileServices[0]); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList """ token = Authentication.getToken() @@ -288,11 +306,13 @@ def deleteUserVolume(fileService, path, quiet=True): def createDir(fileService, path, quiet=True): """ Create a directory. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param path: path (in the remote file service) to the directory (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/directory :param quiet: If set to False, it will throw an error if the directory already exists. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); Files.createDir(fileServices[0], "myRootVolume","myUserVolume", "myNewDir"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList """ token = Authentication.getToken() @@ -331,6 +351,7 @@ def createDir(fileService, path, quiet=True): def upload(fileService, path, data="", localFilePath=None, quiet=True): """ Uploads data or a local file into a path defined in the file system. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param path: path (in the remote file service) to the destination file (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/destinationFile.txt :param data: string containing data to be uploaded, in case localFilePath is not set. @@ -339,6 +360,7 @@ def upload(fileService, path, data="", localFilePath=None, quiet=True): :param quiet: If set to False, it will throw an error if the file already exists. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); Files.upload(fileServices[0], "myRootVolume", "myUserVolume", "/myUploadedFile.txt", None, None, localFilePath="/myFile.txt"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.download, Files.dirList """ token = Authentication.getToken() @@ -378,6 +400,7 @@ def upload(fileService, path, data="", localFilePath=None, quiet=True): def download(fileService, path, localFilePath=None, format="txt", quiet=True): """ Downloads a file from the remote file system into the local file system, or returns the file content as an object in several formats. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param path: String defining the path (in the remote file service) of the file to be downloaded, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/fileToBeDownloaded.txt. :param localFilePath: local destination path of the file to be downloaded. If set to None, then an object of format 'format' will be returned. @@ -387,6 +410,7 @@ def download(fileService, path, localFilePath=None, format="txt", quiet=True): :return: If the 'localFilePath' parameter is defined, then it will return True when the file is downloaded successfully in the local file system. If the 'localFilePath' is not defined, then the type of the returned object depends on the value of the 'format' parameter (either io.StringIO, io.BytesIO, requests.Response or string). :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); isDownloaded = Files.upload("/myUploadedFile.txt","persistent","myUserName", fileServices[0], localFilePath="/myDownloadedFile.txt"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.upload, Files.dirList """ token = Authentication.getToken() @@ -446,6 +470,7 @@ def download(fileService, path, localFilePath=None, format="txt", quiet=True): def dirList(fileService, path, level=1, options=''): """ Lists the contents of a directory. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param path: String defining the path (in the remote file service) of the directory to be listed, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/directoryToBeListed. :param level: amount (int) of listed directory levels that are below or at the same level to that of the relativePath. @@ -453,6 +478,7 @@ def dirList(fileService, path, level=1, options=''): :return: dictionary containing the directory listing. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); dirs = Files.dirList("/","persistent","myUserName", fileServices[0], 2); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.createDir """ token = Authentication.getToken() @@ -484,6 +510,7 @@ def dirList(fileService, path, level=1, options=''): def move(fileService, path, destinationFileService, destinationPath, replaceExisting=True, doCopy=True): """ Moves or copies a file or folder. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param path: String defining the origin path (in the remote fileService) of the file or directory to be copied/moved, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/fileToBeMoved.txt. :param destinationFileService: name of fileService (string), or object (dictionary) that defines a destination file service (where the file is moved/copied into). A list of these kind of objects available to the user is returned by the function Files.getFileServices(). @@ -492,6 +519,7 @@ def move(fileService, path, destinationFileService, destinationPath, replaceExis :param doCopy: if set to True, then it will copy the file or folder. If set to False, then the file or folder will be moved. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); isDownloaded = Files.upload("/myUploadedFile.txt","persistent","myUserName", fileServices[0], localFilePath="/myDownloadedFile.txt"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.upload, Files.dirList """ token = Authentication.getToken() @@ -539,11 +567,13 @@ def move(fileService, path, destinationFileService, destinationPath, replaceExis def delete(fileService, path, quiet=True): """ Deletes a directory or file in the File System. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param path: String defining the path (in the remote fileService) of the file or directory to be deleted, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/fileToBeDeleted.txt. :param quiet: If set to False, it will throw an error if the file does not exist. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList """ token = Authentication.getToken() @@ -573,6 +603,7 @@ def delete(fileService, path, quiet=True): def shareUserVolume(fileService, path, sharedWith, allowedActions, type="USER"): """ Shares a user volume with another user or group + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param path: String defining the path (in the remote fileService) of the user volume to be shared, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume. :param sharedWith: name (string) of user or group that the user volume is shared with. @@ -580,6 +611,7 @@ def shareUserVolume(fileService, path, sharedWith, allowedActions, type="USER"): :param type: type (string) of the entity defined by the "sharedWith" parameter. Can be set to "USER" or "GROUP". :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.shareUserVolume(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]); + .. seealso:: Files.getFileServices(), Files.getFilrmsieServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList """ token = Authentication.getToken() diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py index 41fa533..76f2e70 100644 --- a/py2/SciServer/Jobs.py +++ b/py2/SciServer/Jobs.py @@ -13,9 +13,11 @@ def getDockerComputeDomains(): """ Gets a list of all registered Docker compute domains that the user has access to. + :return: a list of dictionaries, each one containing the definition of a Docker compute domain. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. :example: dockerComputeDomains = Jobs.getDockerComputeDomains(); + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getRDBComputeDomains, Jobs.cancelJob """ token = Authentication.getToken() @@ -42,10 +44,12 @@ def getDockerComputeDomains(): def getDockerComputeDomainsNames(dockerComputeDomains=None): """ Returns the names of the docker compute domains available to the user. + :param dockerComputeDomains: a list of dockerComputeDomain objects (dictionaries), as returned by Jobs.getDockerComputeDomains(). If not set, then an extra internal call to Jobs.getDockerComputeDomains() is made. :return: an array of strings, each being the name of a docker compute domain available to the user. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. :example: dockerComputeDomainsNames = Files.getDockerComputeDomainsNames(); + .. seealso:: Files.getDockerComputeDomains """ if dockerComputeDomains is None: @@ -61,11 +65,13 @@ def getDockerComputeDomainsNames(dockerComputeDomains=None): def getDockerComputeDomainFromName(dockerComputeDomainName, dockerComputeDomains = None): """ Returns a DockerComputeDomain object, given its registered name. + :param dockerComputeDomainName: name of the DockerComputeDomainName, as shown within the results of Jobs.getDockerComputeDomains() :param dockerComputeDomains: a list of dockerComputeDomain objects (dictionaries), as returned by Jobs.getDockerComputeDomains(). If not set, then an internal call to Jobs.getDockerComputeDomains() is made. :return: a DockerComputeDomain object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. :example: dockerComputeDomain = Jobs.getDockerComputeDomainFromName('dockerComputeDomainAtJHU'); + .. seealso:: Jobs.getDockerComputeDomains, Jobs.getRDBComputeDomains, Jobs.getRDBComputeDomainFromName """ if dockerComputeDomainName is None: @@ -87,9 +93,11 @@ def getDockerComputeDomainFromName(dockerComputeDomainName, dockerComputeDomains def getRDBComputeDomains(): """ Gets a list of all registered Relational Database (RDB) compute domains that the user has access to. + :return: a list of dictionaries, each one containing the definition of an RDB compute domain. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. :example: rdbComputeDomains = Jobs.getRDBComputeDomains(); + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob """ token = Authentication.getToken() @@ -114,10 +122,12 @@ def getRDBComputeDomains(): def getRDBComputeDomainsNames(rdbComputeDomains=None): """ Returns the names of the RDB compute domains available to the user. + :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. :return: an array of strings, each being the name of a rdb compute domain available to the user. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. :example: dockerComputeDomainsNames = Files.getDockerComputeDomainsNames(); + .. seealso:: Files.getRDBComputeDomains """ if rdbComputeDomains is None: @@ -133,11 +143,13 @@ def getRDBComputeDomainsNames(rdbComputeDomains=None): def getRDBComputeDomainFromName(rdbComputeDomainName, rdbComputeDomains = None): """ Returns an RDBComputeDomain object, given its registered name. + :param rdbComputeDomainName: name of the RDBComputeDomainName, as shown within the results of Jobs.getRDBComputeDomains() :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. :return: an RDBComputeDomain object (dictionary) that defines an RDB compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. :example: rdbComputeDomain = Jobs.getRDBComputeDomainFromName('rdbComputeDomainAtJHU'); + .. seealso:: Jobs.getDockerComputeDomains, Jobs.getRDBComputeDomains, Jobs.getDockerComputeDomainFromName """ if rdbComputeDomainName is None: @@ -159,6 +171,7 @@ def getRDBComputeDomainFromName(rdbComputeDomainName, rdbComputeDomains = None): def getJobsList(top=10, open=None, start=None, end=None, type='all'): """ Gets the list of Jobs submitted by the user. + :param top: top number of jobs (integer) returned. If top=None, then all jobs are returned. :param open: If set to 'True', then only returns jobs that have not finished executing and wrapped up (status <= FINISHED). If set to 'False' then only returnes jobs that are still running. If set to 'None', then returns both finished and unfinished jobs. :param start: The earliest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no lower bound on date. @@ -167,6 +180,7 @@ def getJobsList(top=10, open=None, start=None, end=None, type='all'): :return: a list of dictionaries, each one containing the definition of a submitted job. :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. :example: jobs = Jobs.getJobsList(top=2); + .. seealso:: Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob """ token = Authentication.getToken() @@ -210,10 +224,12 @@ def getJobsList(top=10, open=None, start=None, end=None, type='all'): def getJobDescription(jobId): """ Gets the definition of the job, + :param jobId: Id of job :return: dictionary containing the description or definition of the job. :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. :example: job1 = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); job2 = Jobs.getJobDescription(job1.get('id')); + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob """ token = Authentication.getToken() @@ -239,10 +255,12 @@ def getJobDescription(jobId): def getJobStatus(jobId): """ Gets a dictionary with the job status as an integer value, together with its semantic meaning. The integer value is a power of 2, that is, 1:PENDING, 2:QUEUED, 4:ACCEPTED, 8:STARTED, 16:FINISHED, 32:SUCCESS, 64:ERROR and 128:CANCELED + :param jobId: Id of job (integer). :return: dictionary with the integer value of the job status, as well as its semantic meaning. :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. :example: jobId = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); status = Jobs.getJobStatus(jobId); + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob """ intStatus = getJobDescription(jobId)["status"] @@ -269,22 +287,20 @@ def getJobStatus(jobId): def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=None, userVolumes=None, dataVolumes=None, resultsFolderPath="", parameters="", jobAlias= ""): """ - Submits a Jupyter Notebook for execution (as an asynchronous job) inside a Docker compute domain, + Submits a Jupyter Notebook for execution (as an asynchronous job) inside a Docker compute domain. + :param notebookPath: path of the notebook within the filesystem mounted in SciServer-Compute (string). Example: notebookPath = '/home/idies/worskpace/persistent/JupyterNotebook.ipynb' :param dockerComputeDomain: object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). :param dockerImageName: name (string) of the Docker image for executing the notebook. E.g., dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object. - :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image. - E.g., userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', , 'needsWriteAccess':True}] - A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=None, then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume. - :param dataVolumes: a list with the names of data volumes that will be mounted to the docker Image. - E.g., dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}]. - A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted. + :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', , 'needsWriteAccess':True}] . A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=None, then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume. + :param dataVolumes: a list with the names of data volumes that will be mounted to the docker Image. E.g.: dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}]. A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted. :param resultsFolderPath: full path to results folder (string) where the original notebook is copied to and executed. E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder. If not set, then a default folder will be set automatically. :param parameters: string containing parameters that the notebook might need during its execution. This string is written in the 'parameters.txt' file in the same directory level where the notebook is being executed. :param jobAlias: alias (string) of job, defined by the user. :return: the job ID (int) :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. - :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitNotebookJob('/home/idies/workspace/persistent/Notebook.ipynb', dockerComputeDomain, 'Python (astro)', [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}], [{'name':'SDSS_DAS'}], 'param1=1\nparam2=2\nparam3=3','myNewJob') + :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitNotebookJob('/home/idies/workspace/persistent/Notebook.ipynb', dockerComputeDomain, 'Python (astro)', [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}], [{'name':'SDSS_DAS'}], 'param1=1\\nparam2=2\\nparam3=3','myNewJob') + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob """ @@ -384,6 +400,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageName = None, userVolumes = None, dataVolumes = None, resultsFolderPath = "", jobAlias = ""): """ Submits a shell command for execution (as an asynchronous job) inside a Docker compute domain. + :param shellCommand: shell command (string) defined by the user. :param dockerComputeDomain: object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). :param dockerImageName: name (string) of the Docker image for executing the notebook. E.g., dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object. @@ -398,6 +415,7 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN :return: the job ID (int) :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitShellCommandJob('pwd', dockerComputeDomain, 'Python (astro)', [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}], [{'name':'SDSS_DAS'}], 'myNewJob') + .. seealso:: Jobs.submitNotebookJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob """ @@ -496,6 +514,7 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = None, resultsName='queryResults', resultsFolderPath="", jobAlias = ""): """ Submits a sql query for execution (as an asynchronous job) inside a relational database (RDB) compute domain. + :param sqlQuery: sql query (string) :param rdbComputeDomain: object (dictionary) that defines a relational database (RDB) compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). :param databaseContextName: database context name (string) on which the sql query is executed. @@ -505,6 +524,7 @@ def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = Non :return: a dictionary containing the definition of the submitted job. :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. :example: job = Jobs.submitRDBQueryJob('select 1';,None, None, 'myQueryResults', 'myNewJob') + .. seealso:: Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob """ @@ -574,9 +594,11 @@ def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = Non def cancelJob(jobId): """ Cancels the execution of a job. + :param jobId: Id of the job (integer) :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error. :example: job = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); isCanceled = Jobs.cancelJob(job.get('id')); + .. seealso:: Jobs.submitNotebookJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains. """ token = Authentication.getToken() @@ -610,6 +632,7 @@ def waitForJob(jobId, verbose=False, pollTime = 5): :return: After the job is finished, returns a dictionary object containing the job definition. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitShellCommandJob(dockerComputeDomain,'pwd', 'Python (astro)');Jobs.waitForJob(job.get('id')) + .. seealso:: Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.submitNotebookJob, Jobs.submitShellCommandJob """ try: diff --git a/py2/SciServer/SkyQuery.py b/py2/SciServer/SkyQuery.py index 2b18fa4..9d24ec6 100644 --- a/py2/SciServer/SkyQuery.py +++ b/py2/SciServer/SkyQuery.py @@ -133,7 +133,7 @@ def getQueueInfo(queue): """ Returns information about a particular job queue (more info in http://www.voservices.net/skyquery). - :param queue: queue name (string) + :param queue: queue name (string) :return: a dictionary containing information associated to the queue. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error. :example: queueInfo = SkyQuery.getQueueInfo('quick') @@ -555,7 +555,7 @@ def uploadTable(uploadData, tableName, datasetName="MyDB", format="csv"): :param format: format of the 'data' parameter. Set to 'csv' for now. :return: returns True if the table was uploaded successfully. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error. - :example: result = SkyQuery.uploadTable("Column1,Column2\n4.5,5.5\n", tableName="myTable", datasetName="MyDB", format="csv") + :example: result = SkyQuery.uploadTable("Column1,Column2\\n4.5,5.5\\n", tableName="myTable", datasetName="MyDB", format="csv") .. seealso:: SkyQuery.listQueues, SkyQuery.listAllDatasets, SkyQuery.getDatasetInfo, SkyQuery.listDatasetTables, SkyQuery.getTableInfo, SkyQuery.getTable, SkyQuery.submitJob """ diff --git a/py2/SciServer/__init__.py b/py2/SciServer/__init__.py index b6dc013..65dc9f6 100644 --- a/py2/SciServer/__init__.py +++ b/py2/SciServer/__init__.py @@ -5,21 +5,31 @@ Some SciServer tools you can access with this package:\n -* `Login Portal `_: Single sign-on portal to all SciServer applications. +* `Login Portal `_: Single sign-on portal to all SciServer applications. \t\tAlthough some tools accept anonymous access, you can use Authentication.login to login and access the tools and your own data and environment (after registering in the Login Portal). If you are running this package in a Jupyter Notebook in the SciServer-Compute environment, the use of Authentication.login is not necessary since it's done automatically. -* `CasJobs `_: Database storage and querying. +* `CasJobs `_: Database storage and querying. \t\tYou can have access big databases, as well as save your data tables in your own database called 'MyDB'. The user can run synchronous or asynchronous SQL queries and get the result back as an R data-frame (using CasJobs.executeQuery or CasJobs.submitJob, respectively). Uploading of CSV files or R data-frames into a database table can be done using CasJobs.uploadCSVToTable and CasJobs.uploadDataFrameToTable, respectively. -* `SciDrive `_: Drag-and-drop file storage and sharing. +* `SciDrive `_: Drag-and-drop file storage and sharing. \t\tYou can create directories in SciDrive using SciDrive.createContainer, upload a file to SciDrive using SciDrive.upload, and share its URL with your collaborators by using SciDrive.publicUrl. -* `SkyServer `_: Access to the SDSS astronomical survey. +* `SkyServer `_: Access to the SDSS astronomical survey. \t\tYou can query the SDSS database using SkyServer.sqlSearch, run cone searches using SkyServer.radialSearch, or get cutout images from the sky using SkyServer.getJpegImgCutout, between other tasks. -* `SkyQuery `_: Cross-match of astronomical source catalogs. +* `SkyQuery `_: Cross-match of astronomical source catalogs. \t\tYou can use this scalable database system for uploading your own catalogs and cross-matching them against huge astronomical source catalogs, or even cross-matching huge catalogs against each other!. Use SkyQuery.submitJob to run the cross-match, and use SkyQuery.listAllDatasets, SkyQuery.listDatasetTables and SkyQuery.listTableColumns to browse the catalogs and database schema. +* `Compute Jobs `_: Submission of Jupyter notebooks or shell commands as jobs +\t\tYou can execute whole Jupyter notebooks and shell commands as asynchronous batch jobs, as well as synchronous jobs. + +* `SciQuery Jobs `_: Submission of SQL queries. +\t\tYou can execute synchronous or s synchronous SQL queries against Postgres and other database backends. + +* `Files `_: Interaction with the SciServer file system. +\t\tYou can upload, and download data into your own file space in SciServer, as well as share data with your collaborators, between other things. + + **References** * SciServer-Python repository: http://www.github.com/sciserver/SciScript-Python diff --git a/py3/SciServer/Authentication.py b/py3/SciServer/Authentication.py index 73e7ca0..abe07cd 100644 --- a/py3/SciServer/Authentication.py +++ b/py3/SciServer/Authentication.py @@ -86,6 +86,7 @@ def login(UserName=None, Password=None): :return: authentication token (string) :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. :example: token1 = Authentication.login(); token2 = Authentication.login('loginName','loginPassword'); + .. seealso:: Authentication.getKeystoneUserWithToken, Authentication.getToken, Authentication.setToken, Authentication.token. """ taskName = "" diff --git a/py3/SciServer/Files.py b/py3/SciServer/Files.py index 5379f8f..2a3cb7d 100644 --- a/py3/SciServer/Files.py +++ b/py3/SciServer/Files.py @@ -12,10 +12,12 @@ def getFileServices(verbose=True): """ Gets the definitions of file services that a user is able to access. A FileService represents a file system that contains root volumes accessible to the user for public/private data storage. Within each rootVolume, users can create sharable userVolumes for storing files. + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). :return: list of dictionaries, where each dictionary represents the description of a FileService that the user is able to access. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. :example: fileServices = Files.getFileServices(); + .. seealso:: Files.getFileServiceFromName """ token = Authentication.getToken() @@ -59,12 +61,14 @@ def getFileServices(verbose=True): def getFileServicesNames(fileServices=None, verbose=True): """ - Returns the names and description of the fileServices available to the user.l + Returns the names and description of the fileServices available to the user. + :param fileServices: a list of FileService objects (dictionaries), as returned by Files.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). :return: an array of dicts, where each dict has the name and description of a file service available to the user. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. :example: fileServiceNames = Files.getFileServicesNames(); + .. seealso:: Files.getFileServices """ @@ -84,12 +88,14 @@ def getFileServicesNames(fileServices=None, verbose=True): def getFileServiceFromName(fileServiceName, fileServices=None, verbose=True): """ Returns a FileService object, given its registered name. + :param fileServiceName: name of the FileService, as shown within the results of Files.getFileServices() :param fileServices: a list of FileService objects (dictionaries), as returned by Files.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). - :return a FileService object (dictionary) that defines a FileService. A list of these kind of objects available to the user is returned by the function Jobs.getFileServices(). If no fileService can be found, then returns None. - :raises Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :return: a FileService object (dictionary) that defines a FileService. A list of these kind of objects available to the user is returned by the function Jobs.getFileServices(). If no fileService can be found, then returns None. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. :example: fileService = Files.getFileServiceFromName('FileServiceAtJHU'); + .. seealso:: Files.getFileServices """ @@ -118,9 +124,11 @@ def getFileServiceFromName(fileServiceName, fileServices=None, verbose=True): def __getFileServiceAPIUrl(fileService): """ Gets the API endpoint URL of a FileService. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :return: API endpoint URL of the FileService (string). :example: fileServiceAPIUrl = Files.__getFileServiceAPIUrl(); + .. seealso:: Files.getFileServiceFromName """ @@ -142,11 +150,13 @@ def __getFileServiceAPIUrl(fileService): def getRootVolumesInfo(fileService, verbose=True): """ Gets the names and descriptions of root volumes available to the user in a particular FileService. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). :return: list of dictionaries, where each dictionary contains the name and description of a root volume. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. :example: fileServices = Files.getFileServices(); rootVolumesInfo = getRootVolumesInfo(fileServices[0]) + .. seealso:: Files.getUserVolumesInfo """ if type(fileService) == str: @@ -163,12 +173,14 @@ def getRootVolumesInfo(fileService, verbose=True): def getUserVolumesInfo(fileService, rootVolumeName = None, verbose=True): """ Gets the names definitions the RootVolumes available in a particular FileService, and of the file services that a user is able to access. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param rootVolumeName: name of root Volume (string) for which the user volumes are fetched. If set to None, then user volumes in all root folders are fetched. :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). :return: list of dictionaries, where each dictionary contains the name and description of a root volumes that a user is able to access. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. :example: fileServices = Files.getFileServices(); rootVolumeNames = Files.getUserVolumesInfo(fileServices[0]) + .. seealso:: Files.getRootVolumes, """ if type(fileService) == str: @@ -193,10 +205,12 @@ def getUserVolumesInfo(fileService, rootVolumeName = None, verbose=True): def splitPath(path): """ Splits a path of the form rootVolume/userVolumeOwner/userVolume/relativePath/... into its 4 components: rootVolume, userVolumeOwner, userVolume, and relativePath. + :param path: file system path (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/relativePath... :return: a tuple containing the four components: (rootVolume, userVolumeOwner, userVolume, relativePath) :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); Files.createUserVolume(fileServices[0], "volumes","newUserVolume"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName """ if path.startswith("/"): @@ -216,11 +230,13 @@ def splitPath(path): def createUserVolume(fileService, path, quiet=True): """ Create a user volume. + :param fileService: name of fileService (string), or object (dictionary) that defines the file service that contains the user volume. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param path: path (in the remote file service) to the user volume (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume :param quiet: if set to False, will throw an error if the User Volume already exists. If True, won't throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); Files.createUserVolume(fileServices[0], "volumes","newUserVolume"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList """ token = Authentication.getToken() @@ -252,11 +268,13 @@ def createUserVolume(fileService, path, quiet=True): def deleteUserVolume(fileService, path, quiet=True): """ Delete a user volume. + :param fileService: name of fileService (string), or object (dictionary) that defines the file service that contains the root volume. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param path: path (in the remote file service) to the user volume (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume :param quiet: If set to False, it will throw an error if user volume does not exists. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); Files.deleteUserVolume("volumes","newUserVolume",fileServices[0]); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList """ token = Authentication.getToken() @@ -288,11 +306,13 @@ def deleteUserVolume(fileService, path, quiet=True): def createDir(fileService, path, quiet=True): """ Create a directory. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param path: path (in the remote file service) to the directory (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/directory :param quiet: If set to False, it will throw an error if the directory already exists. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); Files.createDir(fileServices[0], "myRootVolume","myUserVolume", "myNewDir"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList """ token = Authentication.getToken() @@ -331,6 +351,7 @@ def createDir(fileService, path, quiet=True): def upload(fileService, path, data="", localFilePath=None, quiet=True): """ Uploads data or a local file into a path defined in the file system. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param path: path (in the remote file service) to the destination file (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/destinationFile.txt :param data: string containing data to be uploaded, in case localFilePath is not set. @@ -339,6 +360,7 @@ def upload(fileService, path, data="", localFilePath=None, quiet=True): :param quiet: If set to False, it will throw an error if the file already exists. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); Files.upload(fileServices[0], "myRootVolume", "myUserVolume", "/myUploadedFile.txt", None, None, localFilePath="/myFile.txt"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.download, Files.dirList """ token = Authentication.getToken() @@ -378,6 +400,7 @@ def upload(fileService, path, data="", localFilePath=None, quiet=True): def download(fileService, path, localFilePath=None, format="txt", quiet=True): """ Downloads a file from the remote file system into the local file system, or returns the file content as an object in several formats. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param path: String defining the path (in the remote file service) of the file to be downloaded, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/fileToBeDownloaded.txt. :param localFilePath: local destination path of the file to be downloaded. If set to None, then an object of format 'format' will be returned. @@ -387,6 +410,7 @@ def download(fileService, path, localFilePath=None, format="txt", quiet=True): :return: If the 'localFilePath' parameter is defined, then it will return True when the file is downloaded successfully in the local file system. If the 'localFilePath' is not defined, then the type of the returned object depends on the value of the 'format' parameter (either io.StringIO, io.BytesIO, requests.Response or string). :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); isDownloaded = Files.upload("/myUploadedFile.txt","persistent","myUserName", fileServices[0], localFilePath="/myDownloadedFile.txt"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.upload, Files.dirList """ token = Authentication.getToken() @@ -446,6 +470,7 @@ def download(fileService, path, localFilePath=None, format="txt", quiet=True): def dirList(fileService, path, level=1, options=''): """ Lists the contents of a directory. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param path: String defining the path (in the remote file service) of the directory to be listed, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/directoryToBeListed. :param level: amount (int) of listed directory levels that are below or at the same level to that of the relativePath. @@ -453,6 +478,7 @@ def dirList(fileService, path, level=1, options=''): :return: dictionary containing the directory listing. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); dirs = Files.dirList("/","persistent","myUserName", fileServices[0], 2); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.createDir """ token = Authentication.getToken() @@ -484,6 +510,7 @@ def dirList(fileService, path, level=1, options=''): def move(fileService, path, destinationFileService, destinationPath, replaceExisting=True, doCopy=True): """ Moves or copies a file or folder. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param path: String defining the origin path (in the remote fileService) of the file or directory to be copied/moved, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/fileToBeMoved.txt. :param destinationFileService: name of fileService (string), or object (dictionary) that defines a destination file service (where the file is moved/copied into). A list of these kind of objects available to the user is returned by the function Files.getFileServices(). @@ -492,6 +519,7 @@ def move(fileService, path, destinationFileService, destinationPath, replaceExis :param doCopy: if set to True, then it will copy the file or folder. If set to False, then the file or folder will be moved. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); isDownloaded = Files.upload("/myUploadedFile.txt","persistent","myUserName", fileServices[0], localFilePath="/myDownloadedFile.txt"); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.upload, Files.dirList """ token = Authentication.getToken() @@ -539,11 +567,13 @@ def move(fileService, path, destinationFileService, destinationPath, replaceExis def delete(fileService, path, quiet=True): """ Deletes a directory or file in the File System. + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param path: String defining the path (in the remote fileService) of the file or directory to be deleted, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/fileToBeDeleted.txt. :param quiet: If set to False, it will throw an error if the file does not exist. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.getFileServices(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]); + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList """ token = Authentication.getToken() @@ -573,6 +603,7 @@ def delete(fileService, path, quiet=True): def shareUserVolume(fileService, path, sharedWith, allowedActions, type="USER"): """ Shares a user volume with another user or group + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param path: String defining the path (in the remote fileService) of the user volume to be shared, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume. :param sharedWith: name (string) of user or group that the user volume is shared with. @@ -580,6 +611,7 @@ def shareUserVolume(fileService, path, sharedWith, allowedActions, type="USER"): :param type: type (string) of the entity defined by the "sharedWith" parameter. Can be set to "USER" or "GROUP". :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. :example: fileServices = Files.shareUserVolume(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]); + .. seealso:: Files.getFileServices(), Files.getFilrmsieServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList """ token = Authentication.getToken() diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py index 9548dd6..6a372ef 100644 --- a/py3/SciServer/Jobs.py +++ b/py3/SciServer/Jobs.py @@ -13,9 +13,11 @@ def getDockerComputeDomains(): """ Gets a list of all registered Docker compute domains that the user has access to. + :return: a list of dictionaries, each one containing the definition of a Docker compute domain. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. :example: dockerComputeDomains = Jobs.getDockerComputeDomains(); + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getRDBComputeDomains, Jobs.cancelJob """ token = Authentication.getToken() @@ -42,10 +44,12 @@ def getDockerComputeDomains(): def getDockerComputeDomainsNames(dockerComputeDomains=None): """ Returns the names of the docker compute domains available to the user. + :param dockerComputeDomains: a list of dockerComputeDomain objects (dictionaries), as returned by Jobs.getDockerComputeDomains(). If not set, then an extra internal call to Jobs.getDockerComputeDomains() is made. :return: an array of strings, each being the name of a docker compute domain available to the user. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. :example: dockerComputeDomainsNames = Files.getDockerComputeDomainsNames(); + .. seealso:: Files.getDockerComputeDomains """ if dockerComputeDomains is None: @@ -61,11 +65,13 @@ def getDockerComputeDomainsNames(dockerComputeDomains=None): def getDockerComputeDomainFromName(dockerComputeDomainName, dockerComputeDomains = None): """ Returns a DockerComputeDomain object, given its registered name. + :param dockerComputeDomainName: name of the DockerComputeDomainName, as shown within the results of Jobs.getDockerComputeDomains() :param dockerComputeDomains: a list of dockerComputeDomain objects (dictionaries), as returned by Jobs.getDockerComputeDomains(). If not set, then an internal call to Jobs.getDockerComputeDomains() is made. :return: a DockerComputeDomain object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. :example: dockerComputeDomain = Jobs.getDockerComputeDomainFromName('dockerComputeDomainAtJHU'); + .. seealso:: Jobs.getDockerComputeDomains, Jobs.getRDBComputeDomains, Jobs.getRDBComputeDomainFromName """ if dockerComputeDomainName is None: @@ -87,9 +93,11 @@ def getDockerComputeDomainFromName(dockerComputeDomainName, dockerComputeDomains def getRDBComputeDomains(): """ Gets a list of all registered Relational Database (RDB) compute domains that the user has access to. + :return: a list of dictionaries, each one containing the definition of an RDB compute domain. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. :example: rdbComputeDomains = Jobs.getRDBComputeDomains(); + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob """ token = Authentication.getToken() @@ -114,10 +122,12 @@ def getRDBComputeDomains(): def getRDBComputeDomainsNames(rdbComputeDomains=None): """ Returns the names of the RDB compute domains available to the user. + :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. :return: an array of strings, each being the name of a rdb compute domain available to the user. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. :example: dockerComputeDomainsNames = Files.getDockerComputeDomainsNames(); + .. seealso:: Files.getRDBComputeDomains """ if rdbComputeDomains is None: @@ -133,11 +143,13 @@ def getRDBComputeDomainsNames(rdbComputeDomains=None): def getRDBComputeDomainFromName(rdbComputeDomainName, rdbComputeDomains = None): """ Returns an RDBComputeDomain object, given its registered name. + :param rdbComputeDomainName: name of the RDBComputeDomainName, as shown within the results of Jobs.getRDBComputeDomains() :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. :return: an RDBComputeDomain object (dictionary) that defines an RDB compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. :example: rdbComputeDomain = Jobs.getRDBComputeDomainFromName('rdbComputeDomainAtJHU'); + .. seealso:: Jobs.getDockerComputeDomains, Jobs.getRDBComputeDomains, Jobs.getDockerComputeDomainFromName """ if rdbComputeDomainName is None: @@ -159,6 +171,7 @@ def getRDBComputeDomainFromName(rdbComputeDomainName, rdbComputeDomains = None): def getJobsList(top=10, open=None, start=None, end=None, type='all'): """ Gets the list of Jobs submitted by the user. + :param top: top number of jobs (integer) returned. If top=None, then all jobs are returned. :param open: If set to 'True', then only returns jobs that have not finished executing and wrapped up (status <= FINISHED). If set to 'False' then only returnes jobs that are still running. If set to 'None', then returns both finished and unfinished jobs. :param start: The earliest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no lower bound on date. @@ -167,6 +180,7 @@ def getJobsList(top=10, open=None, start=None, end=None, type='all'): :return: a list of dictionaries, each one containing the definition of a submitted job. :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. :example: jobs = Jobs.getJobsList(top=2); + .. seealso:: Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob """ token = Authentication.getToken() @@ -210,10 +224,12 @@ def getJobsList(top=10, open=None, start=None, end=None, type='all'): def getJobDescription(jobId): """ Gets the definition of the job, + :param jobId: Id of job :return: dictionary containing the description or definition of the job. :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. :example: job1 = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); job2 = Jobs.getJobDescription(job1.get('id')); + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob """ token = Authentication.getToken() @@ -239,10 +255,12 @@ def getJobDescription(jobId): def getJobStatus(jobId): """ Gets a dictionary with the job status as an integer value, together with its semantic meaning. The integer value is a power of 2, that is, 1:PENDING, 2:QUEUED, 4:ACCEPTED, 8:STARTED, 16:FINISHED, 32:SUCCESS, 64:ERROR and 128:CANCELED + :param jobId: Id of job (integer). :return: dictionary with the integer value of the job status, as well as its semantic meaning. :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. :example: jobId = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); status = Jobs.getJobStatus(jobId); + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob """ intStatus = getJobDescription(jobId)["status"] @@ -269,22 +287,20 @@ def getJobStatus(jobId): def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=None, userVolumes=None, dataVolumes=None, resultsFolderPath="", parameters="", jobAlias= ""): """ - Submits a Jupyter Notebook for execution (as an asynchronous job) inside a Docker compute domain, + Submits a Jupyter Notebook for execution (as an asynchronous job) inside a Docker compute domain. + :param notebookPath: path of the notebook within the filesystem mounted in SciServer-Compute (string). Example: notebookPath = '/home/idies/worskpace/persistent/JupyterNotebook.ipynb' :param dockerComputeDomain: object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). :param dockerImageName: name (string) of the Docker image for executing the notebook. E.g., dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object. - :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image. - E.g., userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', , 'needsWriteAccess':True}] - A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=None, then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume. - :param dataVolumes: a list with the names of data volumes that will be mounted to the docker Image. - E.g., dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}]. - A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted. + :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', , 'needsWriteAccess':True}] . A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=None, then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume. + :param dataVolumes: a list with the names of data volumes that will be mounted to the docker Image. E.g.: dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}]. A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted. :param resultsFolderPath: full path to results folder (string) where the original notebook is copied to and executed. E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder. If not set, then a default folder will be set automatically. :param parameters: string containing parameters that the notebook might need during its execution. This string is written in the 'parameters.txt' file in the same directory level where the notebook is being executed. :param jobAlias: alias (string) of job, defined by the user. :return: the job ID (int) :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. - :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitNotebookJob('/home/idies/workspace/persistent/Notebook.ipynb', dockerComputeDomain, 'Python (astro)', [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}], [{'name':'SDSS_DAS'}], 'param1=1\nparam2=2\nparam3=3','myNewJob') + :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitNotebookJob('/home/idies/workspace/persistent/Notebook.ipynb', dockerComputeDomain, 'Python (astro)', [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}], [{'name':'SDSS_DAS'}], 'param1=1\\nparam2=2\\nparam3=3','myNewJob') + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob """ @@ -384,6 +400,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageName = None, userVolumes = None, dataVolumes = None, resultsFolderPath = "", jobAlias = ""): """ Submits a shell command for execution (as an asynchronous job) inside a Docker compute domain. + :param shellCommand: shell command (string) defined by the user. :param dockerComputeDomain: object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). :param dockerImageName: name (string) of the Docker image for executing the notebook. E.g., dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object. @@ -398,6 +415,7 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN :return: the job ID (int) :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitShellCommandJob('pwd', dockerComputeDomain, 'Python (astro)', [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}], [{'name':'SDSS_DAS'}], 'myNewJob') + .. seealso:: Jobs.submitNotebookJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob """ @@ -496,6 +514,7 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = None, resultsName='queryResults', resultsFolderPath="", jobAlias = ""): """ Submits a sql query for execution (as an asynchronous job) inside a relational database (RDB) compute domain. + :param sqlQuery: sql query (string) :param rdbComputeDomain: object (dictionary) that defines a relational database (RDB) compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). :param databaseContextName: database context name (string) on which the sql query is executed. @@ -505,6 +524,7 @@ def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = Non :return: a dictionary containing the definition of the submitted job. :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. :example: job = Jobs.submitRDBQueryJob('select 1';,None, None, 'myQueryResults', 'myNewJob') + .. seealso:: Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob """ @@ -574,9 +594,11 @@ def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = Non def cancelJob(jobId): """ Cancels the execution of a job. + :param jobId: Id of the job (integer) :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error. :example: job = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); isCanceled = Jobs.cancelJob(job.get('id')); + .. seealso:: Jobs.submitNotebookJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains. """ token = Authentication.getToken() @@ -610,6 +632,7 @@ def waitForJob(jobId, verbose=False, pollTime = 5): :return: After the job is finished, returns a dictionary object containing the job definition. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitShellCommandJob(dockerComputeDomain,'pwd', 'Python (astro)');Jobs.waitForJob(job.get('id')) + .. seealso:: Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.submitNotebookJob, Jobs.submitShellCommandJob """ try: diff --git a/py3/SciServer/SkyQuery.py b/py3/SciServer/SkyQuery.py index 63cd765..8da14b5 100644 --- a/py3/SciServer/SkyQuery.py +++ b/py3/SciServer/SkyQuery.py @@ -133,7 +133,7 @@ def getQueueInfo(queue): """ Returns information about a particular job queue (more info in http://www.voservices.net/skyquery). - :param queue: queue name (string) + :param queue: queue name (string) :return: a dictionary containing information associated to the queue. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error. :example: queueInfo = SkyQuery.getQueueInfo('quick') @@ -555,7 +555,7 @@ def uploadTable(uploadData, tableName, datasetName="MyDB", format="csv"): :param format: format of the 'data' parameter. Set to 'csv' for now. :return: returns True if the table was uploaded successfully. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error. - :example: result = SkyQuery.uploadTable("Column1,Column2\n4.5,5.5\n", tableName="myTable", datasetName="MyDB", format="csv") + :example: result = SkyQuery.uploadTable("Column1,Column2\\n4.5,5.5\\n", tableName="myTable", datasetName="MyDB", format="csv") .. seealso:: SkyQuery.listQueues, SkyQuery.listAllDatasets, SkyQuery.getDatasetInfo, SkyQuery.listDatasetTables, SkyQuery.getTableInfo, SkyQuery.getTable, SkyQuery.submitJob """ diff --git a/py3/SciServer/__init__.py b/py3/SciServer/__init__.py index b6dc013..65dc9f6 100644 --- a/py3/SciServer/__init__.py +++ b/py3/SciServer/__init__.py @@ -5,21 +5,31 @@ Some SciServer tools you can access with this package:\n -* `Login Portal `_: Single sign-on portal to all SciServer applications. +* `Login Portal `_: Single sign-on portal to all SciServer applications. \t\tAlthough some tools accept anonymous access, you can use Authentication.login to login and access the tools and your own data and environment (after registering in the Login Portal). If you are running this package in a Jupyter Notebook in the SciServer-Compute environment, the use of Authentication.login is not necessary since it's done automatically. -* `CasJobs `_: Database storage and querying. +* `CasJobs `_: Database storage and querying. \t\tYou can have access big databases, as well as save your data tables in your own database called 'MyDB'. The user can run synchronous or asynchronous SQL queries and get the result back as an R data-frame (using CasJobs.executeQuery or CasJobs.submitJob, respectively). Uploading of CSV files or R data-frames into a database table can be done using CasJobs.uploadCSVToTable and CasJobs.uploadDataFrameToTable, respectively. -* `SciDrive `_: Drag-and-drop file storage and sharing. +* `SciDrive `_: Drag-and-drop file storage and sharing. \t\tYou can create directories in SciDrive using SciDrive.createContainer, upload a file to SciDrive using SciDrive.upload, and share its URL with your collaborators by using SciDrive.publicUrl. -* `SkyServer `_: Access to the SDSS astronomical survey. +* `SkyServer `_: Access to the SDSS astronomical survey. \t\tYou can query the SDSS database using SkyServer.sqlSearch, run cone searches using SkyServer.radialSearch, or get cutout images from the sky using SkyServer.getJpegImgCutout, between other tasks. -* `SkyQuery `_: Cross-match of astronomical source catalogs. +* `SkyQuery `_: Cross-match of astronomical source catalogs. \t\tYou can use this scalable database system for uploading your own catalogs and cross-matching them against huge astronomical source catalogs, or even cross-matching huge catalogs against each other!. Use SkyQuery.submitJob to run the cross-match, and use SkyQuery.listAllDatasets, SkyQuery.listDatasetTables and SkyQuery.listTableColumns to browse the catalogs and database schema. +* `Compute Jobs `_: Submission of Jupyter notebooks or shell commands as jobs +\t\tYou can execute whole Jupyter notebooks and shell commands as asynchronous batch jobs, as well as synchronous jobs. + +* `SciQuery Jobs `_: Submission of SQL queries. +\t\tYou can execute synchronous or s synchronous SQL queries against Postgres and other database backends. + +* `Files `_: Interaction with the SciServer file system. +\t\tYou can upload, and download data into your own file space in SciServer, as well as share data with your collaborators, between other things. + + **References** * SciServer-Python repository: http://www.github.com/sciserver/SciScript-Python From 2aaa63cf0f156f3f919281749337d08f46c1af2f Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Thu, 9 Aug 2018 11:54:45 -0400 Subject: [PATCH 52/79] update example notebook --- Examples/Examples_SciScript-Python.ipynb | 16 ++++---- Examples/TestNotebook.ipynb | 50 +++++++----------------- 2 files changed, 22 insertions(+), 44 deletions(-) diff --git a/Examples/Examples_SciScript-Python.ipynb b/Examples/Examples_SciScript-Python.ipynb index 9783054..7a689c9 100644 --- a/Examples/Examples_SciScript-Python.ipynb +++ b/Examples/Examples_SciScript-Python.ipynb @@ -1016,7 +1016,6 @@ "outputs": [], "source": [ "# get FileService from Name\n", - "Files_FileServiceName = 'NewFileServiceScitest12 INSECURE'\n", "fileService = Files.getFileServiceFromName(Files_FileServiceName);\n", "fileService" ] @@ -1277,19 +1276,19 @@ "metadata": {}, "outputs": [], "source": [ - "Jobs_DockerComputeDomainName = 'Small Jobs Batch Domain'\n", + "Jobs_DockerComputeDomainName = 'Small Jobs Domain'\n", "\n", - "Jobs_FileServiceName = \"NewFileServiceScitest12 INSECURE\"\n", + "Jobs_FileServiceName = \"FileServiceJHU\"\n", "Jobs_RootVolumeName = \"Storage\"\n", "Jobs_UserVolumeName = \"JobsTestVolume\"\n", "Jobs_DirectoryName = \"JobsTestDirectory\"\n", "Jobs_NotebookName = 'TestNotebook.ipynb'\n", "Jobs_ShellCommand = 'ls -laht'\n", - "Jobs_DockerImageName = 'Python (astro)'\n", + "Jobs_DockerImageName = 'Python + R'\n", "\n", "Jobs_UserVolumes = [{'name':Jobs_UserVolumeName, 'rootVolumeName':Jobs_RootVolumeName, 'owner':Authentication_loginName, 'needsWriteAccess':True},\n", " {'name':'scratch', 'rootVolumeName':'Temporary', 'owner':Authentication_loginName, 'needsWriteAccess':True}]\n", - "Jobs_DataVolumes = [{'name':'SDSS_DAS'}]\n", + "Jobs_DataVolumes = [{'name':'SDSS DAS'}]\n", "Jobs_Parameters = 'param1=1\\nparam2=2\\nparam3=3'\n", "Jobs_Alias = 'MyNewJob'\n", "\n", @@ -1359,8 +1358,8 @@ "source": [ "# submit a Jupyter notebook job.\n", "\n", - "jobId = Jobs.submitNotebookJob(notebookPath='',#'/home/idies/workspace/' + \n", - "#\"/\".join([Jobs_RootVolumeName,Authentication_loginName,Jobs_UserVolumeName,Jobs_DirectoryName,Jobs_NotebookName]),\n", + "jobId = Jobs.submitNotebookJob(notebookPath='/home/idies/workspace/' + \n", + "\"/\".join([Jobs_RootVolumeName,Authentication_loginName,Jobs_UserVolumeName,Jobs_DirectoryName,Jobs_NotebookName]),\n", " dockerComputeDomain=dockerComputeDomain,\n", " dockerImageName=Jobs_DockerImageName,\n", " userVolumes=Jobs_UserVolumes,\n", @@ -1403,7 +1402,8 @@ "source": [ "# wait until job is finished and get job status\n", "\n", - "jobId = Jobs.submitNotebookJob(notebookPath='/home/idies/workspace/' + Jobs_UserVolumeName + '/' + Jobs_DirectoryName + '/' + Jobs_NotebookName,\n", + "jobId = Jobs.submitNotebookJob(notebookPath='/home/idies/workspace/' + \n", + "\"/\".join([Jobs_RootVolumeName,Authentication_loginName,Jobs_UserVolumeName,Jobs_DirectoryName,Jobs_NotebookName]),\n", " dockerComputeDomain=dockerComputeDomain,\n", " dockerImageName=Jobs_DockerImageName, \n", " userVolumes=Jobs_UserVolumes, dataVolumes=Jobs_DataVolumes,\n", diff --git a/Examples/TestNotebook.ipynb b/Examples/TestNotebook.ipynb index 51dce8c..b5ef126 100644 --- a/Examples/TestNotebook.ipynb +++ b/Examples/TestNotebook.ipynb @@ -4,9 +4,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -19,11 +17,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "print(str(datetime.now()))" @@ -32,11 +26,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "now = str(datetime.now());\n", @@ -48,9 +38,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -61,11 +49,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "now = str(datetime.now());\n", @@ -74,17 +58,17 @@ "\n", "\n", "jobsPath = None\n", - "with open('/home/idies/job.path', 'r') as f:\n", + "with open('/home/idies/keystone.token', 'r') as f:\n", " jobsPath = f.read().rstrip('\\n')\n", "\n", "if jobsPath == \"\" or jobsPath == None: \n", - " raise Exception(now + ' Could not read /home/idies/job.path') \n", + " raise Exception(now + ' Could not read /home/idies/keystone.token') \n", " \n", - "print(now + \" Was able to read /home/idies/job.path : \" + jobsPath);\n", - "os.system(\"echo '\" + now + \" Was able to read /home/idies/job.path : \" + jobsPath +\"'\");\n", + "print(now + \" Was able to read /home/idies/keystone.token : \" + jobsPath);\n", + "os.system(\"echo '\" + now + \" Was able to read /home/idies/keystone.token : \" + jobsPath +\"'\");\n", "\n", "\n", - "target = open(jobsPath + \"HelloWorld.txt\", 'w')\n", + "target = open(\"HelloWorld.txt\", 'w')\n", "target.truncate()\n", "target.write(jobsPath)\n", "target.write(\"\\n\")\n", @@ -98,9 +82,7 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true, - "deletable": true, - "editable": true + "collapsed": true }, "outputs": [], "source": [ @@ -117,11 +99,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "now = str(datetime.now());\n", @@ -146,9 +124,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.6.5" } }, "nbformat": 4, - "nbformat_minor": 0 + "nbformat_minor": 1 } From d0b92ba2428b97b3af46b069755f8ca458980ffc Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Thu, 9 Aug 2018 14:10:58 -0400 Subject: [PATCH 53/79] use get method in dict --- py2/SciServer/SkyQuery.py | 2 +- py3/SciServer/SkyQuery.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/py2/SciServer/SkyQuery.py b/py2/SciServer/SkyQuery.py index 9d24ec6..65c2ef9 100644 --- a/py2/SciServer/SkyQuery.py +++ b/py2/SciServer/SkyQuery.py @@ -232,7 +232,7 @@ def waitForJob(jobId, verbose=False, pollTime = 5): #print back, print waitingStr, jobDesc = getJobStatus(jobId) - if jobDesc['dateFinished'] is not None: + if jobDesc.get('dateFinished') is not None: complete = True if verbose: #print back, diff --git a/py3/SciServer/SkyQuery.py b/py3/SciServer/SkyQuery.py index 8da14b5..9645262 100644 --- a/py3/SciServer/SkyQuery.py +++ b/py3/SciServer/SkyQuery.py @@ -232,7 +232,7 @@ def waitForJob(jobId, verbose=False, pollTime = 5): #print(back, end="") print(waitingStr, end="") jobDesc = getJobStatus(jobId) - if jobDesc['dateFinished'] is not None: + if jobDesc.get('dateFinished') is not None: complete = True if verbose: #print(back, end="") From fa28e4aeb059a6b444d47506bede132debbeef57 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Thu, 9 Aug 2018 14:13:04 -0400 Subject: [PATCH 54/79] update unittest --- Tests/UnitTests.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/Tests/UnitTests.py b/Tests/UnitTests.py index 8ab71b7..860ba41 100644 --- a/Tests/UnitTests.py +++ b/Tests/UnitTests.py @@ -41,10 +41,10 @@ SciDrive_FileName = "TestFile.csv" SciDrive_FileContent = "Column1,Column2\n4.5,5.5\n" -Files_FileServiceName = "NewFileServiceScitest12_1 INSECURE" -Files_RootVolumeName1 = "persistent" +Files_FileServiceName = "FileServiceJHU" +Files_RootVolumeName1 = "Storage" Files_UserVolumeName1 = "UserVolume555" -Files_RootVolumeName2 = "persistent" +Files_RootVolumeName2 = "Storage" Files_UserVolumeName2 = "UserVolume999" Files_NewDirectoryName1 = "myNewDirectory555" Files_NewDirectoryName2 = "myNewDirectory999" @@ -52,18 +52,19 @@ Files_LocalFileContent = "#ID,Column1,Column2\n1,4.5,5.5" -Jobs_DockerComputeDomainName = 'Small Jobs Batch Domain' +Jobs_DockerComputeDomainName = 'Small Jobs Domain' Jobs_FileServiceName = "FileServiceJHU" -Jobs_RootVolumeName = "volumes" -Jobs_UserVolumeName = Authentication_loginName + "_JobsTestVolume" +Jobs_RootVolumeName = "Storage" +Jobs_UserVolumeName = "JobsTestVolume" Jobs_DirectoryName = "JobsTestDirectory" Jobs_NotebookName = 'TestNotebook.ipynb' Jobs_NoteBookOutPutFile = 'HelloWorld.txt' Jobs_ShellCommand = 'pwd' -Jobs_DockerImageName = 'Python (astro)' +Jobs_DockerImageName = 'Python + R' -Jobs_UserVolumes = [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}] -Jobs_DataVolumes = [{'name':'SDSS_DAS'}] +Jobs_UserVolumes = [{'name':Jobs_UserVolumeName, 'rootVolumeName':Jobs_RootVolumeName, 'owner':Authentication_loginName, 'needsWriteAccess':True}, + {'name':'scratch', 'rootVolumeName':'Temporary', 'owner':Authentication_loginName, 'needsWriteAccess':True}] +Jobs_DataVolumes = [{'name':'SDSS DAS'}] Jobs_Parameters = 'param1=1\nparam2=2\nparam3=3' Jobs_Alias = 'MyNewJob' @@ -727,12 +728,12 @@ def test_Jobs_getJobDirectory(self): unittest.TestLoader.sortTestMethodsUsing = lambda x, y: cmp(x,y); testLoader = unittest.TestLoader() testLoader.sortTestMethodsUsing = lambda x, y: 0; - #suite = testLoader.loadTestsFromTestCase(TestAuthentication); unittest.TextTestRunner(verbosity=2).run(suite) - #suite = testLoader.loadTestsFromTestCase(TestLoginPortal); unittest.TextTestRunner(verbosity=2).run(suite) - #suite = testLoader.loadTestsFromTestCase(TestCasJobs); unittest.TextTestRunner(verbosity=2).run(suite) - #suite = testLoader.loadTestsFromTestCase(TestSkyServer); unittest.TextTestRunner(verbosity=2).run(suite) - #suite = testLoader.loadTestsFromTestCase(TestSciDrive); unittest.TextTestRunner(verbosity=2).run(suite) - #suite = testLoader.loadTestsFromTestCase(TestSkyQuery); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestAuthentication); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestLoginPortal); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestCasJobs); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestSkyServer); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestSciDrive); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestSkyQuery); unittest.TextTestRunner(verbosity=2).run(suite) suite = testLoader.loadTestsFromTestCase(TestFileService); unittest.TextTestRunner(verbosity=2).run(suite) - #suite = testLoader.loadTestsFromTestCase(TestJobs); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestJobs); unittest.TextTestRunner(verbosity=2).run(suite) From 54b6ec605254b620118adac74a214e9217570f20 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Sun, 3 Feb 2019 20:49:28 -0500 Subject: [PATCH 55/79] add data volumes support --- Examples/Examples_SciScript-Python.ipynb | 17 +- py2/SciServer/Files.py | 212 ++++++++++++++-------- py3/SciServer/Files.py | 222 +++++++++++++++-------- 3 files changed, 291 insertions(+), 160 deletions(-) diff --git a/Examples/Examples_SciScript-Python.ipynb b/Examples/Examples_SciScript-Python.ipynb index 7a689c9..c2d90ae 100644 --- a/Examples/Examples_SciScript-Python.ipynb +++ b/Examples/Examples_SciScript-Python.ipynb @@ -967,11 +967,11 @@ "\n", "Files_FileServiceName = \"FileServiceJHU\"\n", "Files_RootVolumeName1 = \"Storage\"\n", - "Files_UserVolumeName1 = \"UserVolume555\"\n", + "Files_UserVolumeName1 = \"UserVolume5\"\n", "Files_RootVolumeName2 = \"Storage\"\n", - "Files_UserVolumeName2 = \"UserVolume999\"\n", - "Files_NewDirectoryName1 = \"myNewDirectory555\"\n", - "Files_NewDirectoryName2 = \"myNewDirectory999\"\n", + "Files_UserVolumeName2 = \"UserVolume9\"\n", + "Files_NewDirectoryName1 = \"myNewDirectory5\"\n", + "Files_NewDirectoryName2 = \"myNewDirectory9\"\n", "Files_LocalFileName = \"MyNewFile.txt\"\n", "Files_LocalFileContent = \"#ID,Column1,Column2\\n1,4.5,5.5\"" ] @@ -1050,7 +1050,7 @@ "metadata": {}, "outputs": [], "source": [ - "# get root volumes\n", + "# get user volumes\n", "\n", "userVolumes = Files.getUserVolumesInfo(fileService)\n", "userVolumes" @@ -1062,7 +1062,10 @@ "metadata": {}, "outputs": [], "source": [ - "str(True)" + "# get data volumes\n", + "\n", + "dataVolumes = Files.getDataVolumesInfo(fileService)\n", + "dataVolumes" ] }, { @@ -1183,7 +1186,7 @@ "\n", "Files.shareUserVolume(fileService, \n", "\"/\".join([Files_RootVolumeName2,Authentication_loginName,Files_UserVolumeName2]), \n", - " sharedWith=Authentication_login_sharedWithName, type=\"USER\",allowedActions=[\"read\"])" + " sharedWith=Authentication_login_sharedWithName, sharee_type=\"USER\",allowedActions=[\"read\"])" ] }, { diff --git a/py2/SciServer/Files.py b/py2/SciServer/Files.py index ee4384c..1c4d39e 100644 --- a/py2/SciServer/Files.py +++ b/py2/SciServer/Files.py @@ -168,6 +168,26 @@ def getRootVolumesInfo(fileService, verbose=True): return rootVolumes +def getDataVolumesInfo(fileService, verbose=True): + """ + Gets the names and descriptions of data volumes available to the user in a particular FileService. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: list of dictionaries, where each dictionary contains the name and description of a data volume. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServices = Files.getFileServices(); dataVolumesInfo = getDataVolumesInfo(fileServices[0]) + + .. seealso:: Files.getRootVolumesInfo + """ + if type(fileService) == str: + fileService = getFileServiceFromName(fileService, verbose=verbose) + + dataVolumes = [] + for dataVolume in fileService.get("dataVolumes"): + dataVolumes.append({"dataVolumeName":dataVolume.get("name"), "dataVolumeDescription":dataVolume.get("description")}) + + return dataVolumes def getUserVolumesInfo(fileService, rootVolumeName = None, verbose=True): @@ -189,42 +209,65 @@ def getUserVolumesInfo(fileService, rootVolumeName = None, verbose=True): userVolumes = []; for rootVolume in fileService.get("rootVolumes"): for userVolume in rootVolume.get('userVolumes'): - path = os.path.join(rootVolume.get('name'), userVolume.get('owner'), userVolume.get('name')) + path=os.path.join(rootVolume.get('name'),userVolume.get('owner'),userVolume.get('name')) if rootVolumeName is not None: if rootVolume.get('name') == rootVolumeName: userVolumes.append({"userVolumeName": userVolume.get('name'), "path":path, "userVolumeDescription": userVolume.get('description'), "rootVolumeName":rootVolume.get("name"), "rootVolumeDescription":rootVolume.get("description")}) else: - userVolumes.append({"userVolumeName": userVolume.get('name'), "path":path, "userVolumeDescription": userVolume.get('description'),"rootVolumeName": rootVolume.get("name"), "rootVolumeDescription": rootVolume.get("description")}) + userVolumes.append({"userVolumeName": userVolume.get('name'), "path":path,"userVolumeDescription": userVolume.get('description'),"rootVolumeName": rootVolume.get("name"), "rootVolumeDescription": rootVolume.get("description")}) return userVolumes -def splitPath(path): +def splitPath(path, fileService): """ - Splits a path of the form rootVolume/userVolumeOwner/userVolume/relativePath/... into its 4 components: rootVolume, userVolumeOwner, userVolume, and relativePath. + If the topVolume in the 'path' is a rootVolume, then this function splits the path of the form rootVolume/userVolumeOwner/userVolume/relativePath/... and returns the tuple (rootVolume, userVolumeOwner, userVolume, relativePath, True). On the other hand, + if the topVolume in the 'path' is a dataVolume, then this function splits the path of the form dataVolume/userVolumeOwner/userVolume/relativePath/... and returns the tuple (dataVolume, None, None, relativePath, False). - :param path: file system path (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/relativePath... - :return: a tuple containing the four components: (rootVolume, userVolumeOwner, userVolume, relativePath) + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: file system path (string), starting from the root volume level or data volume level. Example: rootVolume/userVolumeOwner/userVolume/relativePath... or dataVolume/relativePath... + :return: a tuple containing the components: (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeADataVolume) :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); Files.createUserVolume(fileServices[0], "volumes","newUserVolume"); + :example: fileServices = Files.getFileServices(); Files.splitPath("Storage/myUserName/persistent/my/Relative/Path", fileServices[0]); .. seealso:: Files.getFileServices(), Files.getFileServiceFromName """ if path.startswith("/"): path = path[1:] + path = path.split("/") - if len(path) < 3: - raise Exception("path variable does not conform with the format 'rootVolume/userVolumeOwner/userVolume/relativePath...'") + if len(path) == 0: + raise Exception("path variable does not conform with the format 'rootVolume/userVolumeOwner/userVolume/relativePath...' or format 'dataVolume/relativePath...'") + + topVolume = path[0] + isTopVolumeARootVolume = False + + for rootVolume in fileService.get("rootVolumes"): + if rootVolume.get("name") == topVolume: + isTopVolumeARootVolume = True + + if isTopVolumeARootVolume: + if len(path) < 3: + raise Exception("path variable does not conform with the format 'rootVolume/userVolumeOwner/userVolume/relativePath...'") + + rootVolume = path[0] + userVolumeOwner = path[1] + userVolume = path[2] + relativePath = "/".join(path[3:]) + return (rootVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) + else: + if len(path) < 2: + raise Exception("path variable does not conform with the format 'dataVolume/relativePath...'") + + dataVolume = path[0] + relativePath = "/".join(path[1:]) + return (dataVolume, None, None, relativePath, isTopVolumeARootVolume) + - rootVolume = path[0] - userVolumeOwner = path[1] - userVolume = path[2] - relativePath = "/".join(path[3:]) - return (rootVolume, userVolumeOwner, userVolume, relativePath) def createUserVolume(fileService, path, quiet=True): @@ -250,7 +293,7 @@ def createUserVolume(fileService, path, quiet=True): if type(fileService) == str: fileService = getFileServiceFromName(fileService) - (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); + (rootVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path); url = __getFileServiceAPIUrl(fileService) + "api/volume/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?quiet="+str(quiet) + "&TaskName="+taskName; @@ -260,7 +303,7 @@ def createUserVolume(fileService, path, quiet=True): if res.status_code >= 200 and res.status_code < 300: pass; else: - raise Exception("Error when creating user volume '" + str(path) + "' in file service '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when creating user volume '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") @@ -285,11 +328,11 @@ def deleteUserVolume(fileService, path, quiet=True): else: taskName = "SciScript-Python.Files.createUserVolume" - (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); - if type(fileService) == str: fileService = getFileServiceFromName(fileService) + (rootVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + url = __getFileServiceAPIUrl(fileService) + "api/volume/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?quiet="+str(quiet)+"&TaskName="+taskName; headers = {'X-Auth-Token': token} @@ -298,7 +341,7 @@ def deleteUserVolume(fileService, path, quiet=True): if res.status_code >= 200 and res.status_code < 300: pass else: - raise Exception("Error when deleting user volume '" + str(path) + "' in file service '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when deleting user volume '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") @@ -308,10 +351,10 @@ def createDir(fileService, path, quiet=True): Create a directory. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param path: path (in the remote file service) to the directory (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/directory + :param path: path (in the remote file service) to the directory (string), starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/directory or dataVolume/directory :param quiet: If set to False, it will throw an error if the directory already exists. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); Files.createDir(fileServices[0], "myRootVolume","myUserVolume", "myNewDir"); + :example: fileServices = Files.getFileServices(); Files.createDir(fileServices[0], "Storage/myUserName/persistent/myNewDir"); .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList """ @@ -323,19 +366,15 @@ def createDir(fileService, path, quiet=True): else: taskName = "SciScript-Python.Files.createDir" - - (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); - - if not relativePath.startswith("/"): - relativePath = "/" + relativePath; - - if userVolumeOwner is None: - userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - if type(fileService) == str: fileService = getFileServiceFromName(fileService) - url = __getFileServiceAPIUrl(fileService) + "api/folder/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName; + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/folder/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName; + else: + url = __getFileServiceAPIUrl(fileService) + "api/folder/" + topVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName; headers = {'X-Auth-Token': token} res = requests.put(url, headers=headers) @@ -343,7 +382,7 @@ def createDir(fileService, path, quiet=True): if res.status_code >= 200 and res.status_code < 300: pass; else: - raise Exception("Error when creating directory '" + str(path) + "' in file service '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when creating directory '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") @@ -353,13 +392,13 @@ def upload(fileService, path, data="", localFilePath=None, quiet=True): Uploads data or a local file into a path defined in the file system. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param path: path (in the remote file service) to the destination file (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/destinationFile.txt + :param path: path (in the remote file service) to the destination file (string), starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/destinationFile.txt or dataVolume/destinationFile.txt :param data: string containing data to be uploaded, in case localFilePath is not set. :param localFilePath: path to a local file to be uploaded (string), :param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume. :param quiet: If set to False, it will throw an error if the file already exists. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); Files.upload(fileServices[0], "myRootVolume", "myUserVolume", "/myUploadedFile.txt", None, None, localFilePath="/myFile.txt"); + :example: fileServices = Files.getFileServices(); Files.upload(fileServices[0], "myRootVolume/myUserName/myUserVolume/myUploadedFile.txt", None, localFilePath="/myFile.txt"); .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.download, Files.dirList """ @@ -371,12 +410,15 @@ def upload(fileService, path, data="", localFilePath=None, quiet=True): else: taskName = "SciScript-Python.Files.UploadFile" - (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); - if type(fileService) == str: fileService = getFileServiceFromName(fileService) - url = __getFileServiceAPIUrl(fileService) + "api/file/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/file/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName + else: + url = __getFileServiceAPIUrl(fileService) + "api/file/" + topVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName headers = {'X-Auth-Token': token} @@ -392,7 +434,7 @@ def upload(fileService, path, data="", localFilePath=None, quiet=True): if res.status_code >= 200 and res.status_code < 300: pass; else: - raise Exception("Error when uploading file to '" + str(path) + "' in file service '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when uploading file to '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") @@ -402,41 +444,46 @@ def download(fileService, path, localFilePath=None, format="txt", quiet=True): Downloads a file from the remote file system into the local file system, or returns the file content as an object in several formats. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param path: String defining the path (in the remote file service) of the file to be downloaded, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/fileToBeDownloaded.txt. + :param path: String defining the path (in the remote file service) of the file to be downloaded, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/fileToBeDownloaded.txt or dataVolume/fileToBeDownloaded.txt :param localFilePath: local destination path of the file to be downloaded. If set to None, then an object of format 'format' will be returned. :param format: name (string) of the returned object's type (if localFilePath is not defined). This parameter can be "StringIO" (io.StringIO object containing readable text), "BytesIO" (io.BytesIO object containing readable binary data), "response" ( the HTTP response as an object of class requests.Response) or "txt" (a text string). If the parameter 'localFilePath' is defined, then the 'format' parameter is not used and the file is downloaded to the local file system instead. :param userVolumeOwner: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. :param quiet: If set to False, it will throw an error if the file already exists. If set to True. it will not throw an error. :return: If the 'localFilePath' parameter is defined, then it will return True when the file is downloaded successfully in the local file system. If the 'localFilePath' is not defined, then the type of the returned object depends on the value of the 'format' parameter (either io.StringIO, io.BytesIO, requests.Response or string). :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); isDownloaded = Files.upload("/myUploadedFile.txt","persistent","myUserName", fileServices[0], localFilePath="/myDownloadedFile.txt"); + :example: fileServices = Files.getFileServices(); Files.upload(fileServices[0], "Storage/myUserName/persistent/fileToBeDownloaded.txt", localFilePath="/fileToBeDownloaded.txt"); .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.upload, Files.dirList """ token = Authentication.getToken() if token is not None and token != "": - (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); if type(fileService) == str: fileService = getFileServiceFromName(fileService) + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + if localFilePath is not None: if os.path.isfile(localFilePath) and not quiet: - raise Exception("Error when downloading '" + str(path) + "' from file service '" + fileService.get("name") + "'. Local file '" + localFilePath + "' already exists."); + raise Exception("Error when downloading '" + str(path) + "' from file service '" + str(fileService.get("name")) + "'. Local file '" + localFilePath + "' already exists."); if Config.isSciServerComputeEnvironment(): taskName = "Compute.SciScript-Python.Files.DownloadFile" else: taskName = "SciScript-Python.Files.DownloadFile" - url = __getFileServiceAPIUrl(fileService) + "api/file/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?TaskName=" + taskName; + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/file/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?TaskName=" + taskName; + else: + url = __getFileServiceAPIUrl(fileService) + "api/file/" + topVolume + "/" + relativePath + "?TaskName=" + taskName; + headers = {'X-Auth-Token': token} res = requests.get(url, stream=True, headers=headers) if res.status_code < 200 or res.status_code >= 300: - raise Exception("Error when downloading '" + str(path) + "' from file service '" + fileService.get("name") + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when downloading '" + str(path) + "' from file service '" + str(fileService.get("name")) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); if localFilePath is not None and localFilePath != "": @@ -472,12 +519,12 @@ def dirList(fileService, path, level=1, options=''): Lists the contents of a directory. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param path: String defining the path (in the remote file service) of the directory to be listed, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/directoryToBeListed. + :param path: String defining the path (in the remote file service) of the directory to be listed, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/directoryToBeListed or dataVolume/directoryToBeListed :param level: amount (int) of listed directory levels that are below or at the same level to that of the relativePath. :param options: string of file filtering options. :return: dictionary containing the directory listing. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); dirs = Files.dirList("/","persistent","myUserName", fileServices[0], 2); + :example: fileServices = Files.getFileServices(); dirs = Files.dirList(fileServices[0], "Storage/myUserName/persistent/", level=2); .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.createDir """ @@ -489,12 +536,15 @@ def dirList(fileService, path, level=1, options=''): else: taskName = "SciScript-Python.Files.dirList" - (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); - if type(fileService) == str: fileService = getFileServiceFromName(fileService) - url = __getFileServiceAPIUrl(fileService) + "api/jsontree/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?options=" + options + "&level=" + str(level) + "&TaskName=" + taskName; + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/jsontree/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?options=" + options + "&level=" + str(level) + "&TaskName=" + taskName; + else: + url = __getFileServiceAPIUrl(fileService) + "api/jsontree/" + topVolume + "/" + relativePath + "?options=" + options + "&level=" + str(level) + "&TaskName=" + taskName; headers = {'X-Auth-Token': token} res = requests.get(url, headers=headers) @@ -512,13 +562,13 @@ def move(fileService, path, destinationFileService, destinationPath, replaceExis Moves or copies a file or folder. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param path: String defining the origin path (in the remote fileService) of the file or directory to be copied/moved, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/fileToBeMoved.txt. + :param path: String defining the origin path (in the remote fileService) of the file or directory to be copied/moved, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/fileToBeMoved.txt or dataVolume/fileToBeMoved.txt :param destinationFileService: name of fileService (string), or object (dictionary) that defines a destination file service (where the file is moved/copied into). A list of these kind of objects available to the user is returned by the function Files.getFileServices(). :param destinationRelativePath: String defining the destination path (in the remote destinationFileService) of the file or directory to be copied/moved, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/recentlyMovedFile.txt. :param replaceExisting: If set to False, it will throw an error if the file already exists, If set to True, it will not throw and eeror in that case. :param doCopy: if set to True, then it will copy the file or folder. If set to False, then the file or folder will be moved. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); isDownloaded = Files.upload("/myUploadedFile.txt","persistent","myUserName", fileServices[0], localFilePath="/myDownloadedFile.txt"); + :example: fileServices = Files.getFileServices(); Files.move(fileServices[0], "Storage/myUserName/persistent/myFile.txt", fileServices[0], "Storage/myUserName/persistent/myFile2.txt") .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.upload, Files.dirList """ @@ -530,34 +580,37 @@ def move(fileService, path, destinationFileService, destinationPath, replaceExis else: taskName = "SciScript-Python.Files.Move" - (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); - (destinationRootVolume, destinationUserVolumeOwner, destinationUserVolume, destinationRelativePath) = splitPath(destinationPath); - - if userVolumeOwner is None: - userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - - if destinationUserVolumeOwner is None: - destinationUserVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - if type(fileService) == str: + destinationFileServiceName = destinationFileService fileService = getFileServiceFromName(fileService) + else: + destinationFileServiceName = destinationFileService['name'] + if type(destinationFileService) == str: destinationFileService = getFileServiceFromName(destinationFileService) - url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName; - headers = {'X-Auth-Token': token, "Content-Type": "application/json"} - - if type(destinationFileService) == dict: - destinationFileService = destinationFileService['name'] + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + (destinationTopVolume, destinationUserVolumeOwner, destinationUserVolume, destinationRelativePath, isDestinationTopVolumeARootVolume) = splitPath(destinationPath, destinationFileService); + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/data/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName; + jsonDict = {'destinationPath': destinationRelativePath, + 'destinationRootVolume': destinationTopVolume, + 'destinationUserVolume': destinationUserVolume, + 'destinationOwnerName': destinationUserVolumeOwner, + 'destinationFileService': destinationFileServiceName}; + else: + url = __getFileServiceAPIUrl(fileService) + "api/data/" + topVolume + "/" + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName; + jsonDict = {'destinationPath': destinationRelativePath, + 'destinationDataVolume': destinationTopVolume, + 'destinationFileService': destinationFileServiceName}; - jsonDict = {'destinationPath': destinationRelativePath, 'destinationRootVolume': destinationRootVolume, 'destinationUserVolume':destinationUserVolume, 'destinationOwnerName': destinationUserVolumeOwner, 'destinationFileService': destinationFileService}; - data = json.dumps(jsonDict).encode() + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} res = requests.put(url, stream=True, headers=headers, json=jsonDict) if res.status_code < 200 or res.status_code >= 300: - raise Exception("Error when moving '" + str(path) + "' in file service '" + fileService.get("name") + "' to '" + str(path) + "' in file service '" + destinationFileService.get("name)") + "'. \nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when moving '" + str(path) + "' in file service '" + str(fileService.get("name")) + "' to '" + str(destinationPath) + "' in file service '" + str(destinationFileService.get("name")) + "'. \nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") @@ -569,10 +622,10 @@ def delete(fileService, path, quiet=True): Deletes a directory or file in the File System. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param path: String defining the path (in the remote fileService) of the file or directory to be deleted, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/fileToBeDeleted.txt. + :param path: String defining the path (in the remote fileService) of the file or directory to be deleted, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/fileToBeDeleted.txt or dataVolume/fileToBeDeleted.txt :param quiet: If set to False, it will throw an error if the file does not exist. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]); + :example: fileServices = Files.getFileServices(); Files.delete(fileServices[0], "Storage/myUserName/persistent/myUselessFile.txt"); .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList """ @@ -584,9 +637,15 @@ def delete(fileService, path, quiet=True): else: taskName = "SciScript-Python.Files.delete" - (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); - url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/data/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName + else: + url = __getFileServiceAPIUrl(fileService) + "api/data/" + topVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName headers = {'X-Auth-Token': token} res = requests.delete(url, headers=headers) @@ -610,7 +669,7 @@ def shareUserVolume(fileService, path, sharedWith, allowedActions, type="USER"): :param allowedActions: array of strings defining actions the user or group is allowed to do with respect to the shared user volume. E.g.: ["read","write","grant","delete"]. The "grant" action means that the user or group can also share the user volume with another user or group. The "delete" action meand ability to delete the user volume (use with care). :param type: type (string) of the entity defined by the "sharedWith" parameter. Can be set to "USER" or "GROUP". :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.shareUserVolume(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]); + :example: fileServices = Files.getFileServices(); Files.shareUserVolume(getFileServices[0], "Storage/myUserName/myUSerVolume", "userName", ["read","write"], type="USER"); .. seealso:: Files.getFileServices(), Files.getFilrmsieServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList """ @@ -622,7 +681,10 @@ def shareUserVolume(fileService, path, sharedWith, allowedActions, type="USER"): else: taskName = "SciScript-Python.Files.ShareUserVolume" - (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (rootVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); data = [{'name': sharedWith, 'type':type, 'allowedActions': allowedActions }] @@ -636,6 +698,6 @@ def shareUserVolume(fileService, path, sharedWith, allowedActions, type="USER"): if res.status_code >= 200 and res.status_code < 300: pass; else: - raise Exception("Error when sharing userVolume '" + str(path) + "' in file service '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()) + raise Exception("Error when sharing userVolume '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()) else: raise Exception("User token is not defined. First log into SciServer.") diff --git a/py3/SciServer/Files.py b/py3/SciServer/Files.py index 2a3cb7d..0e306fd 100644 --- a/py3/SciServer/Files.py +++ b/py3/SciServer/Files.py @@ -168,6 +168,26 @@ def getRootVolumesInfo(fileService, verbose=True): return rootVolumes +def getDataVolumesInfo(fileService, verbose=True): + """ + Gets the names and descriptions of data volumes available to the user in a particular FileService. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: list of dictionaries, where each dictionary contains the name and description of a data volume. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServices = Files.getFileServices(); dataVolumesInfo = getDataVolumesInfo(fileServices[0]) + + .. seealso:: Files.getRootVolumesInfo + """ + if type(fileService) == str: + fileService = getFileServiceFromName(fileService, verbose=verbose) + + dataVolumes = [] + for dataVolume in fileService.get("dataVolumes"): + dataVolumes.append({"dataVolumeName":dataVolume.get("name"), "dataVolumeDescription":dataVolume.get("description")}) + + return dataVolumes def getUserVolumesInfo(fileService, rootVolumeName = None, verbose=True): @@ -202,29 +222,56 @@ def getUserVolumesInfo(fileService, rootVolumeName = None, verbose=True): -def splitPath(path): +def splitPath(path, fileService): """ - Splits a path of the form rootVolume/userVolumeOwner/userVolume/relativePath/... into its 4 components: rootVolume, userVolumeOwner, userVolume, and relativePath. + If the topVolume in the 'path' is a rootVolume, then this function splits the path of the form rootVolume/userVolumeOwner/userVolume/relativePath and returns the tuple (rootVolume, userVolumeOwner, userVolume, relativePath, True). On the other hand, + if the topVolume in the 'path' is a dataVolume, then this function splits the path of the form dataVolume/relativePath and returns the tuple (dataVolume, None, None, relativePath, False). - :param path: file system path (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/relativePath... - :return: a tuple containing the four components: (rootVolume, userVolumeOwner, userVolume, relativePath) + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: file system path (string), starting from the root volume level or data volume level. Example: rootVolume/userVolumeOwner/userVolume/relativePath... or dataVolume/relativePath... + :return: a tuple containing the components: (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeADataVolume) :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); Files.createUserVolume(fileServices[0], "volumes","newUserVolume"); + :example: fileServices = Files.getFileServices(); Files.splitPath("Storage/myUserName/persistent/my/Relative/Path", fileServices[0]); .. seealso:: Files.getFileServices(), Files.getFileServiceFromName """ + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + if path.startswith("/"): path = path[1:] + path = path.split("/") - if len(path) < 3: - raise Exception("path variable does not conform with the format 'rootVolume/userVolumeOwner/userVolume/relativePath...'") + if len(path) == 0: + raise Exception("path variable does not conform with the format 'rootVolume/userVolumeOwner/userVolume/relativePath...' or format 'dataVolume/relativePath...'") + + topVolume = path[0] + isTopVolumeARootVolume = False + + for rootVolume in fileService.get("rootVolumes"): + if rootVolume.get("name") == topVolume: + isTopVolumeARootVolume = True + + if isTopVolumeARootVolume: + if len(path) < 3: + raise Exception("path variable does not conform with the format 'rootVolume/userVolumeOwner/userVolume/relativePath...'") + + rootVolume = path[0] + userVolumeOwner = path[1] + userVolume = path[2] + relativePath = "/".join(path[3:]) + return (rootVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) + else: + if len(path) < 1: + raise Exception("path variable does not conform with the format 'dataVolume/relativePath...'") + + dataVolume = path[0] + relativePath = "/".join(path[1:]) + return (dataVolume, None, None, relativePath, isTopVolumeARootVolume) + - rootVolume = path[0] - userVolumeOwner = path[1] - userVolume = path[2] - relativePath = "/".join(path[3:]) - return (rootVolume, userVolumeOwner, userVolume, relativePath) def createUserVolume(fileService, path, quiet=True): @@ -250,7 +297,7 @@ def createUserVolume(fileService, path, quiet=True): if type(fileService) == str: fileService = getFileServiceFromName(fileService) - (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); + (rootVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); url = __getFileServiceAPIUrl(fileService) + "api/volume/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?quiet="+str(quiet) + "&TaskName="+taskName; @@ -260,7 +307,7 @@ def createUserVolume(fileService, path, quiet=True): if res.status_code >= 200 and res.status_code < 300: pass; else: - raise Exception("Error when creating user volume '" + str(path) + "' in file service '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when creating user volume '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") @@ -285,11 +332,11 @@ def deleteUserVolume(fileService, path, quiet=True): else: taskName = "SciScript-Python.Files.createUserVolume" - (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); - if type(fileService) == str: fileService = getFileServiceFromName(fileService) + (rootVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + url = __getFileServiceAPIUrl(fileService) + "api/volume/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?quiet="+str(quiet)+"&TaskName="+taskName; headers = {'X-Auth-Token': token} @@ -298,7 +345,7 @@ def deleteUserVolume(fileService, path, quiet=True): if res.status_code >= 200 and res.status_code < 300: pass else: - raise Exception("Error when deleting user volume '" + str(path) + "' in file service '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when deleting user volume '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") @@ -308,10 +355,10 @@ def createDir(fileService, path, quiet=True): Create a directory. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param path: path (in the remote file service) to the directory (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/directory + :param path: path (in the remote file service) to the directory (string), starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/directory or dataVolume/directory :param quiet: If set to False, it will throw an error if the directory already exists. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); Files.createDir(fileServices[0], "myRootVolume","myUserVolume", "myNewDir"); + :example: fileServices = Files.getFileServices(); Files.createDir(fileServices[0], "Storage/myUserName/persistent/myNewDir"); .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList """ @@ -323,19 +370,15 @@ def createDir(fileService, path, quiet=True): else: taskName = "SciScript-Python.Files.createDir" - - (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); - - if not relativePath.startswith("/"): - relativePath = "/" + relativePath; - - if userVolumeOwner is None: - userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - if type(fileService) == str: fileService = getFileServiceFromName(fileService) - url = __getFileServiceAPIUrl(fileService) + "api/folder/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName; + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/folder/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName; + else: + url = __getFileServiceAPIUrl(fileService) + "api/folder/" + topVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName; headers = {'X-Auth-Token': token} res = requests.put(url, headers=headers) @@ -343,7 +386,7 @@ def createDir(fileService, path, quiet=True): if res.status_code >= 200 and res.status_code < 300: pass; else: - raise Exception("Error when creating directory '" + str(path) + "' in file service '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when creating directory '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") @@ -353,13 +396,13 @@ def upload(fileService, path, data="", localFilePath=None, quiet=True): Uploads data or a local file into a path defined in the file system. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param path: path (in the remote file service) to the destination file (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/destinationFile.txt + :param path: path (in the remote file service) to the destination file (string), starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/destinationFile.txt or dataVolume/destinationFile.txt :param data: string containing data to be uploaded, in case localFilePath is not set. :param localFilePath: path to a local file to be uploaded (string), :param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume. :param quiet: If set to False, it will throw an error if the file already exists. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); Files.upload(fileServices[0], "myRootVolume", "myUserVolume", "/myUploadedFile.txt", None, None, localFilePath="/myFile.txt"); + :example: fileServices = Files.getFileServices(); Files.upload(fileServices[0], "myRootVolume/myUserName/myUserVolume/myUploadedFile.txt", None, localFilePath="/myFile.txt"); .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.download, Files.dirList """ @@ -371,12 +414,15 @@ def upload(fileService, path, data="", localFilePath=None, quiet=True): else: taskName = "SciScript-Python.Files.UploadFile" - (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); - if type(fileService) == str: fileService = getFileServiceFromName(fileService) - url = __getFileServiceAPIUrl(fileService) + "api/file/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/file/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName + else: + url = __getFileServiceAPIUrl(fileService) + "api/file/" + topVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName headers = {'X-Auth-Token': token} @@ -392,7 +438,7 @@ def upload(fileService, path, data="", localFilePath=None, quiet=True): if res.status_code >= 200 and res.status_code < 300: pass; else: - raise Exception("Error when uploading file to '" + str(path) + "' in file service '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when uploading file to '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") @@ -402,41 +448,46 @@ def download(fileService, path, localFilePath=None, format="txt", quiet=True): Downloads a file from the remote file system into the local file system, or returns the file content as an object in several formats. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param path: String defining the path (in the remote file service) of the file to be downloaded, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/fileToBeDownloaded.txt. + :param path: String defining the path (in the remote file service) of the file to be downloaded, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/fileToBeDownloaded.txt or dataVolume/fileToBeDownloaded.txt :param localFilePath: local destination path of the file to be downloaded. If set to None, then an object of format 'format' will be returned. :param format: name (string) of the returned object's type (if localFilePath is not defined). This parameter can be "StringIO" (io.StringIO object containing readable text), "BytesIO" (io.BytesIO object containing readable binary data), "response" ( the HTTP response as an object of class requests.Response) or "txt" (a text string). If the parameter 'localFilePath' is defined, then the 'format' parameter is not used and the file is downloaded to the local file system instead. :param userVolumeOwner: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. :param quiet: If set to False, it will throw an error if the file already exists. If set to True. it will not throw an error. :return: If the 'localFilePath' parameter is defined, then it will return True when the file is downloaded successfully in the local file system. If the 'localFilePath' is not defined, then the type of the returned object depends on the value of the 'format' parameter (either io.StringIO, io.BytesIO, requests.Response or string). :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); isDownloaded = Files.upload("/myUploadedFile.txt","persistent","myUserName", fileServices[0], localFilePath="/myDownloadedFile.txt"); + :example: fileServices = Files.getFileServices(); Files.upload(fileServices[0], "Storage/myUserName/persistent/fileToBeDownloaded.txt", localFilePath="/fileToBeDownloaded.txt"); .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.upload, Files.dirList """ token = Authentication.getToken() if token is not None and token != "": - (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); if type(fileService) == str: fileService = getFileServiceFromName(fileService) + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + if localFilePath is not None: if os.path.isfile(localFilePath) and not quiet: - raise Exception("Error when downloading '" + str(path) + "' from file service '" + fileService.get("name") + "'. Local file '" + localFilePath + "' already exists."); + raise Exception("Error when downloading '" + str(path) + "' from file service '" + str(fileService.get("name")) + "'. Local file '" + localFilePath + "' already exists."); if Config.isSciServerComputeEnvironment(): taskName = "Compute.SciScript-Python.Files.DownloadFile" else: taskName = "SciScript-Python.Files.DownloadFile" - url = __getFileServiceAPIUrl(fileService) + "api/file/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?TaskName=" + taskName; + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/file/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?TaskName=" + taskName; + else: + url = __getFileServiceAPIUrl(fileService) + "api/file/" + topVolume + "/" + relativePath + "?TaskName=" + taskName; + headers = {'X-Auth-Token': token} res = requests.get(url, stream=True, headers=headers) if res.status_code < 200 or res.status_code >= 300: - raise Exception("Error when downloading '" + str(path) + "' from file service '" + fileService.get("name") + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when downloading '" + str(path) + "' from file service '" + str(fileService.get("name")) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); if localFilePath is not None and localFilePath != "": @@ -472,12 +523,12 @@ def dirList(fileService, path, level=1, options=''): Lists the contents of a directory. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param path: String defining the path (in the remote file service) of the directory to be listed, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/directoryToBeListed. + :param path: String defining the path (in the remote file service) of the directory to be listed, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/directoryToBeListed or dataVolume/directoryToBeListed :param level: amount (int) of listed directory levels that are below or at the same level to that of the relativePath. :param options: string of file filtering options. :return: dictionary containing the directory listing. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); dirs = Files.dirList("/","persistent","myUserName", fileServices[0], 2); + :example: fileServices = Files.getFileServices(); dirs = Files.dirList(fileServices[0], "Storage/myUserName/persistent/", level=2); .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.createDir """ @@ -489,12 +540,15 @@ def dirList(fileService, path, level=1, options=''): else: taskName = "SciScript-Python.Files.dirList" - (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); - if type(fileService) == str: fileService = getFileServiceFromName(fileService) - url = __getFileServiceAPIUrl(fileService) + "api/jsontree/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?options=" + options + "&level=" + str(level) + "&TaskName=" + taskName; + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/jsontree/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?options=" + options + "&level=" + str(level) + "&TaskName=" + taskName; + else: + url = __getFileServiceAPIUrl(fileService) + "api/jsontree/" + topVolume + "/" + relativePath + "?options=" + options + "&level=" + str(level) + "&TaskName=" + taskName; headers = {'X-Auth-Token': token} res = requests.get(url, headers=headers) @@ -512,13 +566,13 @@ def move(fileService, path, destinationFileService, destinationPath, replaceExis Moves or copies a file or folder. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param path: String defining the origin path (in the remote fileService) of the file or directory to be copied/moved, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/fileToBeMoved.txt. + :param path: String defining the origin path (in the remote fileService) of the file or directory to be copied/moved, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/fileToBeMoved.txt or dataVolume/fileToBeMoved.txt :param destinationFileService: name of fileService (string), or object (dictionary) that defines a destination file service (where the file is moved/copied into). A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param destinationRelativePath: String defining the destination path (in the remote destinationFileService) of the file or directory to be copied/moved, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/recentlyMovedFile.txt. + :param destinationRelativePath: String defining the destination path (in the remote destinationFileService) of the file or directory to be copied/moved, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/recentlyMovedFile.txt or dataVolume/recentlyMovedFile.txt} :param replaceExisting: If set to False, it will throw an error if the file already exists, If set to True, it will not throw and eeror in that case. :param doCopy: if set to True, then it will copy the file or folder. If set to False, then the file or folder will be moved. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); isDownloaded = Files.upload("/myUploadedFile.txt","persistent","myUserName", fileServices[0], localFilePath="/myDownloadedFile.txt"); + :example: fileServices = Files.getFileServices(); Files.move(fileServices[0], "Storage/myUserName/persistent/myFile.txt", fileServices[0], "Storage/myUserName/persistent/myFile2.txt") .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.upload, Files.dirList """ @@ -530,34 +584,37 @@ def move(fileService, path, destinationFileService, destinationPath, replaceExis else: taskName = "SciScript-Python.Files.Move" - (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); - (destinationRootVolume, destinationUserVolumeOwner, destinationUserVolume, destinationRelativePath) = splitPath(destinationPath); - - if userVolumeOwner is None: - userVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - - if destinationUserVolumeOwner is None: - destinationUserVolumeOwner = Authentication.getKeystoneUserWithToken(token).userName; - if type(fileService) == str: + destinationFileServiceName = destinationFileService fileService = getFileServiceFromName(fileService) + else: + destinationFileServiceName = destinationFileService['name'] + if type(destinationFileService) == str: destinationFileService = getFileServiceFromName(destinationFileService) - url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName; - headers = {'X-Auth-Token': token, "Content-Type": "application/json"} - - if type(destinationFileService) == dict: - destinationFileService = destinationFileService['name'] + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + (destinationTopVolume, destinationUserVolumeOwner, destinationUserVolume, destinationRelativePath, isDestinationTopVolumeARootVolume) = splitPath(destinationPath, destinationFileService); + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/data/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName; + jsonDict = {'destinationPath': destinationRelativePath, + 'destinationRootVolume': destinationTopVolume, + 'destinationUserVolume': destinationUserVolume, + 'destinationOwnerName': destinationUserVolumeOwner, + 'destinationFileService': destinationFileServiceName}; + else: + url = __getFileServiceAPIUrl(fileService) + "api/data/" + topVolume + "/" + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName; + jsonDict = {'destinationPath': destinationRelativePath, + 'destinationDataVolume': destinationTopVolume, + 'destinationFileService': destinationFileServiceName}; - jsonDict = {'destinationPath': destinationRelativePath, 'destinationRootVolume': destinationRootVolume, 'destinationUserVolume':destinationUserVolume, 'destinationOwnerName': destinationUserVolumeOwner, 'destinationFileService': destinationFileService}; - data = json.dumps(jsonDict).encode() + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} res = requests.put(url, stream=True, headers=headers, json=jsonDict) if res.status_code < 200 or res.status_code >= 300: - raise Exception("Error when moving '" + str(path) + "' in file service '" + fileService.get("name") + "' to '" + str(path) + "' in file service '" + destinationFileService.get("name)") + "'. \nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when moving '" + str(path) + "' in file service '" + str(fileService.get("name")) + "' to '" + str(destinationPath) + "' in file service '" + str(destinationFileService.get("name")) + "'. \nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); else: raise Exception("User token is not defined. First log into SciServer.") @@ -569,10 +626,10 @@ def delete(fileService, path, quiet=True): Deletes a directory or file in the File System. :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). - :param path: String defining the path (in the remote fileService) of the file or directory to be deleted, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/fileToBeDeleted.txt. + :param path: String defining the path (in the remote fileService) of the file or directory to be deleted, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/fileToBeDeleted.txt or dataVolume/fileToBeDeleted.txt :param quiet: If set to False, it will throw an error if the file does not exist. If set to True. it will not throw an error. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.getFileServices(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]); + :example: fileServices = Files.getFileServices(); Files.delete(fileServices[0], "Storage/myUserName/persistent/myUselessFile.txt"); .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList """ @@ -584,9 +641,15 @@ def delete(fileService, path, quiet=True): else: taskName = "SciScript-Python.Files.delete" - (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); - url = __getFileServiceAPIUrl(fileService) + "api/data/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/data/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName + else: + url = __getFileServiceAPIUrl(fileService) + "api/data/" + topVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName headers = {'X-Auth-Token': token} res = requests.delete(url, headers=headers) @@ -600,7 +663,7 @@ def delete(fileService, path, quiet=True): -def shareUserVolume(fileService, path, sharedWith, allowedActions, type="USER"): +def shareUserVolume(fileService, path, sharedWith, allowedActions, sharee_type="USER"): """ Shares a user volume with another user or group @@ -608,11 +671,11 @@ def shareUserVolume(fileService, path, sharedWith, allowedActions, type="USER"): :param path: String defining the path (in the remote fileService) of the user volume to be shared, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume. :param sharedWith: name (string) of user or group that the user volume is shared with. :param allowedActions: array of strings defining actions the user or group is allowed to do with respect to the shared user volume. E.g.: ["read","write","grant","delete"]. The "grant" action means that the user or group can also share the user volume with another user or group. The "delete" action meand ability to delete the user volume (use with care). - :param type: type (string) of the entity defined by the "sharedWith" parameter. Can be set to "USER" or "GROUP". + :param sharee_type: type (string) of the entity defined by the "sharedWith" parameter. Can be set to "USER" or "GROUP". :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. - :example: fileServices = Files.shareUserVolume(); isDeleted = Files.delete("/myUselessFile.txt","persistent","myUserName", fileServices[0]); + :example: fileServices = Files.getFileServices(); Files.shareUserVolume(getFileServices[0], "Storage/myUserName/myUSerVolume", "userName", ["read","write"], type="USER"); - .. seealso:: Files.getFileServices(), Files.getFilrmsieServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList """ token = Authentication.getToken() if token is not None and token != "": @@ -622,10 +685,13 @@ def shareUserVolume(fileService, path, sharedWith, allowedActions, type="USER"): else: taskName = "SciScript-Python.Files.ShareUserVolume" - (rootVolume, userVolumeOwner, userVolume, relativePath) = splitPath(path); + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (rootVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); - data = [{'name': sharedWith, 'type':type, 'allowedActions': allowedActions }] + data = [{'name': sharedWith, 'type':sharee_type, 'allowedActions': allowedActions }] body = json.dumps(data).encode() url = __getFileServiceAPIUrl(fileService) + "api/share/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?TaskName="+taskName @@ -636,6 +702,6 @@ def shareUserVolume(fileService, path, sharedWith, allowedActions, type="USER"): if res.status_code >= 200 and res.status_code < 300: pass; else: - raise Exception("Error when sharing userVolume '" + str(path) + "' in file service '" + fileService.get('name') + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()) + raise Exception("Error when sharing userVolume '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()) else: raise Exception("User token is not defined. First log into SciServer.") From 3b409447aea601d3642b830803f4fed355d19eba Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Thu, 28 Feb 2019 12:53:17 -0500 Subject: [PATCH 56/79] update version to 2.0.13 --- docs_sphinx/conf.py | 4 ++-- py2/SciServer/Config.py | 2 +- py2/SciServer/__init__.py | 2 +- py2/setup.py | 2 +- py3/SciServer/Config.py | 4 ++-- py3/SciServer/__init__.py | 2 +- py3/setup.py | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs_sphinx/conf.py b/docs_sphinx/conf.py index ba2c023..592d54c 100644 --- a/docs_sphinx/conf.py +++ b/docs_sphinx/conf.py @@ -55,9 +55,9 @@ # built documents. # # The short X.Y version. -version = '2.0.0' +version = '2.0.13' # The full version, including alpha/beta/rc tags. -release = '2.0.0' +release = '2.0.13' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/py2/SciServer/Config.py b/py2/SciServer/Config.py index 0ca212b..b8f1c91 100644 --- a/py2/SciServer/Config.py +++ b/py2/SciServer/Config.py @@ -31,7 +31,7 @@ RacmApiURL = "https://apps.sciserver.org/racm" DataRelease = "DR14" KeystoneTokenPath = "/home/idies/keystone.token" #the path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment -version = "sciserver-v2.0.0" #sciserver release version +version = "sciserver-v2.0.13" #sciserver release version ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. def _load_config(filename): diff --git a/py2/SciServer/__init__.py b/py2/SciServer/__init__.py index 65dc9f6..f6da570 100644 --- a/py2/SciServer/__init__.py +++ b/py2/SciServer/__init__.py @@ -41,7 +41,7 @@ **Authors**: Gerard Lemson , Manuchehr Taghizadeh-Popp -**Version**: sciserver-v2.0.0 +**Version**: sciserver-v2.0.13 """ diff --git a/py2/setup.py b/py2/setup.py index a968c68..9021fe4 100644 --- a/py2/setup.py +++ b/py2/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages setup( name = "SciServer", - version = "2.0.0", + version = "2.0.13", packages = find_packages(), ) \ No newline at end of file diff --git a/py3/SciServer/Config.py b/py3/SciServer/Config.py index 0ca212b..08f9aac 100644 --- a/py3/SciServer/Config.py +++ b/py3/SciServer/Config.py @@ -29,9 +29,9 @@ SkyQueryUrl = "http://voservices.net/skyquery/Api/V1" SkyServerWSurl = "https://skyserver.sdss.org" RacmApiURL = "https://apps.sciserver.org/racm" -DataRelease = "DR14" +DataRelease = "DR15" KeystoneTokenPath = "/home/idies/keystone.token" #the path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment -version = "sciserver-v2.0.0" #sciserver release version +version = "sciserver-v2.0.13" #sciserver release version ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. def _load_config(filename): diff --git a/py3/SciServer/__init__.py b/py3/SciServer/__init__.py index 65dc9f6..f6da570 100644 --- a/py3/SciServer/__init__.py +++ b/py3/SciServer/__init__.py @@ -41,7 +41,7 @@ **Authors**: Gerard Lemson , Manuchehr Taghizadeh-Popp -**Version**: sciserver-v2.0.0 +**Version**: sciserver-v2.0.13 """ diff --git a/py3/setup.py b/py3/setup.py index a968c68..9021fe4 100644 --- a/py3/setup.py +++ b/py3/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages setup( name = "SciServer", - version = "2.0.0", + version = "2.0.13", packages = find_packages(), ) \ No newline at end of file From bf45d93732469d06b0ab74421b5f4c3fca78d85a Mon Sep 17 00:00:00 2001 From: "Arik W. Mitschang" Date: Fri, 29 Nov 2019 19:14:12 -0500 Subject: [PATCH 57/79] Fixing incorrect doc strings Also providing information on coordinate type and image truncation. --- py2/SciServer/SkyServer.py | 11 +++++------ py3/SciServer/SkyServer.py | 11 +++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/py2/SciServer/SkyServer.py b/py2/SciServer/SkyServer.py index a664723..ca20934 100644 --- a/py2/SciServer/SkyServer.py +++ b/py2/SciServer/SkyServer.py @@ -60,12 +60,11 @@ def getJpegImgCutout(ra, dec, scale=0.7, width=512, height=512, opt="", query="" """ Gets a rectangular image cutout from a region of the sky in SDSS, centered at (ra,dec). Return type is numpy.ndarray.\n - :param ra: Right Ascension of the image's center. - :param dec: Declination of the image's center. - :param scale: scale of the image, measured in [arcsec/pix] - :param width: Right Ascension of the image's center. - :param ra: Right Ascension of the image's center. - :param height: Height of the image, measured in [pix]. + :param ra: Right Ascension of the image's center in floating point ICRS J2000 coordinates. + :param dec: Declination of the image's center in floating point ICRS J2000 coordinates. + :param scale: Scale of the image, measured in [arcsec/pix] + :param width: Width of image, measured in pixels (width in arcsec = width * scale). If larger than 2048, silently truncated to 2048. + :param height: Height of image, measured in pixels (width in arcsec = width * scale). If larger than 2048, silently truncated to 2048. :param opt: Optional drawing options, expressed as concatenation of letters (string). The letters options are \n \t"G": Grid. Draw a N-S E-W grid through the center\n \t"L": Label. Draw the name, scale, ra, and dec on image.\n diff --git a/py3/SciServer/SkyServer.py b/py3/SciServer/SkyServer.py index a664723..ca20934 100644 --- a/py3/SciServer/SkyServer.py +++ b/py3/SciServer/SkyServer.py @@ -60,12 +60,11 @@ def getJpegImgCutout(ra, dec, scale=0.7, width=512, height=512, opt="", query="" """ Gets a rectangular image cutout from a region of the sky in SDSS, centered at (ra,dec). Return type is numpy.ndarray.\n - :param ra: Right Ascension of the image's center. - :param dec: Declination of the image's center. - :param scale: scale of the image, measured in [arcsec/pix] - :param width: Right Ascension of the image's center. - :param ra: Right Ascension of the image's center. - :param height: Height of the image, measured in [pix]. + :param ra: Right Ascension of the image's center in floating point ICRS J2000 coordinates. + :param dec: Declination of the image's center in floating point ICRS J2000 coordinates. + :param scale: Scale of the image, measured in [arcsec/pix] + :param width: Width of image, measured in pixels (width in arcsec = width * scale). If larger than 2048, silently truncated to 2048. + :param height: Height of image, measured in pixels (width in arcsec = width * scale). If larger than 2048, silently truncated to 2048. :param opt: Optional drawing options, expressed as concatenation of letters (string). The letters options are \n \t"G": Grid. Draw a N-S E-W grid through the center\n \t"L": Label. Draw the name, scale, ra, and dec on image.\n From c8aa133c032e794bb49ae1d97730e0fab754d793 Mon Sep 17 00:00:00 2001 From: Arik Mitschang Date: Mon, 8 Feb 2021 19:10:57 +0000 Subject: [PATCH 58/79] Fixing typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fa7a80f..6b7b5af 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## Python libraries for Jupyter Notebooks -This python package provides functions for quick access of [SciServer](http://www.sciserver.org) APIs (web services) and tools. +This python package provides functions for quick access to [SciServer](http://www.sciserver.org) APIs (web services) and tools. [SciServer](http://www.sciserver.org) provides a new online framework for data-intensive scientifc computing in the cloud, where the motto is to bring the computation close where the data is stored, and allow seamless access and sharing of big data sets within the scientific community. From 2f9137813c811276980b820d64824879ea5938b4 Mon Sep 17 00:00:00 2001 From: Dmitry Medvedev Date: Wed, 10 Feb 2021 21:55:46 -0500 Subject: [PATCH 59/79] Add Dask module --- py3/SciServer/Dask.py | 61 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 py3/SciServer/Dask.py diff --git a/py3/SciServer/Dask.py b/py3/SciServer/Dask.py new file mode 100644 index 0000000..faf0c76 --- /dev/null +++ b/py3/SciServer/Dask.py @@ -0,0 +1,61 @@ +import requests +from requests.exceptions import HTTPError +from base64 import b64decode +from pathlib import Path +from dask.distributed import Client +from distributed.security import Security +from os.path import expanduser +import SciServer.Authentication +import json + +def getClient(compute_url=None, ref_id=None): + """ + Creates a new client for the specified Dask cluster. + If `compute_url` and `ref_id` are not set, cluster connection properties will be read from the ``~/dask-cluster.json`` file injected into new SciServer Compute containers automatically when they are created with an attached Dask cluster. + + :param compute_url: the URL of a SciServer Compute instance used to manage Dask clusters, defaults to None + :param ref_id: Dask cluster ref. ID in SciServer Compute, defaults to None + + :return: Dask client + :rtype: :class:`dask.distributed.Client` + """ + token = SciServer.Authentication.getToken() + data = None + if (compute_url is None): + with open(expanduser('~/dask-cluster.json')) as f: + data = json.load(f) + else: + try: + response = requests.get(''.join([compute_url.rstrip('/'), '/api/dask/clusters/', ref_id]), + params = {'connectionInfo': 'true'}, + headers = {'X-Auth-Token': token}) + response.raise_for_status() + data = response.json()['connection'] + except HTTPError as err: + try: + msg = str(err) + '\n' + response.json()['error']['stackTrace'] + except: + raise err + raise HTTPError(msg) + + base_dir = expanduser('~/.sciserver/dask') + Path(base_dir).mkdir(parents=True, exist_ok=True) + + with open(base_dir + '/ca.pem', 'w+b') as f: + f.write(b64decode(data['ca'])) + + with open(base_dir + '/client-cert.pem', 'w+b') as f: + f.write(b64decode(data['clientCert'])) + + with open(base_dir + '/client-key.pem', 'w+b') as f: + f.write(b64decode(data['clientKey'])) + + scheduler_url = data['schedulerUrl'] + client = Client( + scheduler_url, + security = Security(tls_ca_file = base_dir + '/ca.pem', + tls_client_cert = base_dir + '/client-cert.pem', + tls_client_key = base_dir + '/client-key.pem', + require_encryption=True)) + + return client From b2445bb68c4e1ad6d882d9bd395a2f35666adfe1 Mon Sep 17 00:00:00 2001 From: Dmitry Medvedev Date: Wed, 10 Feb 2021 21:57:56 -0500 Subject: [PATCH 60/79] Update Sphinx doc --- docs_sphinx/SciServer.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs_sphinx/SciServer.rst b/docs_sphinx/SciServer.rst index bb3bc03..fa71e05 100644 --- a/docs_sphinx/SciServer.rst +++ b/docs_sphinx/SciServer.rst @@ -78,4 +78,10 @@ SciServer.Jobs module :undoc-members: :show-inheritance: +SciServer.Dask module +--------------------- +.. automodule:: SciServer.Dask + :members: + :undoc-members: + :show-inheritance: From e91a1c1f4307d8203ae11face9894c586342397f Mon Sep 17 00:00:00 2001 From: Dmitry Medvedev Date: Wed, 10 Feb 2021 22:24:47 -0500 Subject: [PATCH 61/79] Update doc --- py3/SciServer/Dask.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/py3/SciServer/Dask.py b/py3/SciServer/Dask.py index faf0c76..7921ca9 100644 --- a/py3/SciServer/Dask.py +++ b/py3/SciServer/Dask.py @@ -11,12 +11,13 @@ def getClient(compute_url=None, ref_id=None): """ Creates a new client for the specified Dask cluster. + If `compute_url` and `ref_id` are not set, cluster connection properties will be read from the ``~/dask-cluster.json`` file injected into new SciServer Compute containers automatically when they are created with an attached Dask cluster. - :param compute_url: the URL of a SciServer Compute instance used to manage Dask clusters, defaults to None - :param ref_id: Dask cluster ref. ID in SciServer Compute, defaults to None + :param compute_url: the URL of a SciServer Compute instance used to manage Dask clusters, defaults to `None` + :param ref_id: Dask cluster reference ID in SciServer Compute, defaults to `None` - :return: Dask client + :return: Dask client object :rtype: :class:`dask.distributed.Client` """ token = SciServer.Authentication.getToken() From 468207bfab4fd4c33f8a16e00b148dc4257e972e Mon Sep 17 00:00:00 2001 From: Dmitry Medvedev Date: Thu, 11 Feb 2021 11:03:06 -0500 Subject: [PATCH 62/79] Docstring formatting --- py3/SciServer/Dask.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/py3/SciServer/Dask.py b/py3/SciServer/Dask.py index 7921ca9..b5a8343 100644 --- a/py3/SciServer/Dask.py +++ b/py3/SciServer/Dask.py @@ -12,10 +12,15 @@ def getClient(compute_url=None, ref_id=None): """ Creates a new client for the specified Dask cluster. - If `compute_url` and `ref_id` are not set, cluster connection properties will be read from the ``~/dask-cluster.json`` file injected into new SciServer Compute containers automatically when they are created with an attached Dask cluster. + If `compute_url` and `ref_id` are not set, cluster connection + properties will be read from the ``~/dask-cluster.json`` file + injected into new SciServer Compute containers automatically + when they are created with an attached Dask cluster. - :param compute_url: the URL of a SciServer Compute instance used to manage Dask clusters, defaults to `None` - :param ref_id: Dask cluster reference ID in SciServer Compute, defaults to `None` + :param compute_url: the URL of a SciServer Compute instance + used to manage Dask clusters, defaults to `None` + :param ref_id: Dask cluster reference ID in SciServer Compute, + defaults to `None` :return: Dask client object :rtype: :class:`dask.distributed.Client` From 7306a2c48d4d30b7b80a7b59b84f9588843e8a46 Mon Sep 17 00:00:00 2001 From: Dmitry Medvedev Date: Fri, 12 Feb 2021 10:29:47 -0500 Subject: [PATCH 63/79] Get Compute URL from Config --- py3/SciServer/Config.py | 5 +++++ py3/SciServer/Dask.py | 11 +++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/py3/SciServer/Config.py b/py3/SciServer/Config.py index 08f9aac..6238c1b 100644 --- a/py3/SciServer/Config.py +++ b/py3/SciServer/Config.py @@ -20,6 +20,8 @@ - **Config.KeystoneTokenPath**: defines the local path (string) to the file containing the user's authentication token in the SciServer-Compute environment. E.g., "/home/idies/keystone.token". Unlikely to change since it is hardcoded in SciServer-Compute. +- **Config.ComputeUrl**: defines the base URL of the Compute webapp (string). E.g., "https://apps.sciserver.org/compute". + - **Config.version**: defines the SciServer release version tag (string), to which this package belongs. E.g., "sciserver-v1.9.3" """ # URLs for accessing SciServer web services (API endpoints) @@ -33,6 +35,7 @@ KeystoneTokenPath = "/home/idies/keystone.token" #the path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment version = "sciserver-v2.0.13" #sciserver release version ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. +ComputeUrl = "https://apps.sciserver.org/compute" def _load_config(filename): if os.path.exists(filename): @@ -40,6 +43,7 @@ def _load_config(filename): _config_data = json.load(f) global CasJobsRESTUri, AuthenticationURL, SciDriveHost, SkyQueryUrl, SkyServerWSurl global RacmApiURL, DataRelease, KeystoneTokenPath, version, ComputeJobDirectoryFile + global ComputeUrl CasJobsRESTUri = _config_data.get('CasJobsRESTUri', CasJobsRESTUri) AuthenticationURL = _config_data.get('AuthenticationURL', AuthenticationURL) SciDriveHost = _config_data.get('SciDriveHost', SciDriveHost) @@ -50,6 +54,7 @@ def _load_config(filename): KeystoneTokenPath = _config_data.get('KeystoneTokenPath', KeystoneTokenPath) version = _config_data.get('version', version) ComputeJobDirectoryFile = _config_data.get('ComputeJobDirectoryFile', ComputeJobDirectoryFile) + ComputeUrl = _config_data.get('ComputeUrl', ComputeUrl) _CONFIG_DIR = os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')) _SCISERVER_SYSTEM_CONFIG_DIR = '/etc/' # will not likely exist on non *nix systems diff --git a/py3/SciServer/Dask.py b/py3/SciServer/Dask.py index b5a8343..e7251c3 100644 --- a/py3/SciServer/Dask.py +++ b/py3/SciServer/Dask.py @@ -6,19 +6,18 @@ from distributed.security import Security from os.path import expanduser import SciServer.Authentication +import SciServer.Config import json -def getClient(compute_url=None, ref_id=None): +def getClient(ref_id=None): """ Creates a new client for the specified Dask cluster. - If `compute_url` and `ref_id` are not set, cluster connection + If `ref_id` is not set, cluster connection properties will be read from the ``~/dask-cluster.json`` file injected into new SciServer Compute containers automatically when they are created with an attached Dask cluster. - :param compute_url: the URL of a SciServer Compute instance - used to manage Dask clusters, defaults to `None` :param ref_id: Dask cluster reference ID in SciServer Compute, defaults to `None` @@ -27,12 +26,12 @@ def getClient(compute_url=None, ref_id=None): """ token = SciServer.Authentication.getToken() data = None - if (compute_url is None): + if (ref_id is None): with open(expanduser('~/dask-cluster.json')) as f: data = json.load(f) else: try: - response = requests.get(''.join([compute_url.rstrip('/'), '/api/dask/clusters/', ref_id]), + response = requests.get(''.join([SciServer.Config.ComputeUrl.rstrip('/'), '/api/dask/clusters/', ref_id]), params = {'connectionInfo': 'true'}, headers = {'X-Auth-Token': token}) response.raise_for_status() From dd88e88d84fa4d57720e02dcaf466fdd968ce187 Mon Sep 17 00:00:00 2001 From: Dmitry Medvedev Date: Wed, 24 Feb 2021 09:43:51 -0500 Subject: [PATCH 64/79] Update version to 2.1.0 --- docs_sphinx/conf.py | 4 ++-- py2/SciServer/Config.py | 2 +- py2/SciServer/__init__.py | 2 +- py2/setup.py | 2 +- py3/SciServer/Config.py | 2 +- py3/SciServer/__init__.py | 2 +- py3/setup.py | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs_sphinx/conf.py b/docs_sphinx/conf.py index 592d54c..63600e8 100644 --- a/docs_sphinx/conf.py +++ b/docs_sphinx/conf.py @@ -55,9 +55,9 @@ # built documents. # # The short X.Y version. -version = '2.0.13' +version = '2.1.0' # The full version, including alpha/beta/rc tags. -release = '2.0.13' +release = '2.1.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/py2/SciServer/Config.py b/py2/SciServer/Config.py index b8f1c91..3c78c03 100644 --- a/py2/SciServer/Config.py +++ b/py2/SciServer/Config.py @@ -31,7 +31,7 @@ RacmApiURL = "https://apps.sciserver.org/racm" DataRelease = "DR14" KeystoneTokenPath = "/home/idies/keystone.token" #the path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment -version = "sciserver-v2.0.13" #sciserver release version +version = "sciserver-v2.1.0" #sciserver release version ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. def _load_config(filename): diff --git a/py2/SciServer/__init__.py b/py2/SciServer/__init__.py index f6da570..bf3877c 100644 --- a/py2/SciServer/__init__.py +++ b/py2/SciServer/__init__.py @@ -41,7 +41,7 @@ **Authors**: Gerard Lemson , Manuchehr Taghizadeh-Popp -**Version**: sciserver-v2.0.13 +**Version**: sciserver-v2.1.0 """ diff --git a/py2/setup.py b/py2/setup.py index 9021fe4..b61abb2 100644 --- a/py2/setup.py +++ b/py2/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages setup( name = "SciServer", - version = "2.0.13", + version = "2.1.0", packages = find_packages(), ) \ No newline at end of file diff --git a/py3/SciServer/Config.py b/py3/SciServer/Config.py index 6238c1b..0a7818c 100644 --- a/py3/SciServer/Config.py +++ b/py3/SciServer/Config.py @@ -33,7 +33,7 @@ RacmApiURL = "https://apps.sciserver.org/racm" DataRelease = "DR15" KeystoneTokenPath = "/home/idies/keystone.token" #the path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment -version = "sciserver-v2.0.13" #sciserver release version +version = "sciserver-v2.1.0" #sciserver release version ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. ComputeUrl = "https://apps.sciserver.org/compute" diff --git a/py3/SciServer/__init__.py b/py3/SciServer/__init__.py index f6da570..bf3877c 100644 --- a/py3/SciServer/__init__.py +++ b/py3/SciServer/__init__.py @@ -41,7 +41,7 @@ **Authors**: Gerard Lemson , Manuchehr Taghizadeh-Popp -**Version**: sciserver-v2.0.13 +**Version**: sciserver-v2.1.0 """ diff --git a/py3/setup.py b/py3/setup.py index 9021fe4..b61abb2 100644 --- a/py3/setup.py +++ b/py3/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages setup( name = "SciServer", - version = "2.0.13", + version = "2.1.0", packages = find_packages(), ) \ No newline at end of file From e335961865366c8d8c3e77194621fbe194abea40 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Wed, 8 Dec 2021 14:10:14 -0500 Subject: [PATCH 65/79] Added base functionality for SciQuery module. --- py3/SciServer/Config.py | 2 + py3/SciServer/SciQuery.py | 352 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 354 insertions(+) create mode 100644 py3/SciServer/SciQuery.py diff --git a/py3/SciServer/Config.py b/py3/SciServer/Config.py index 0a7818c..afb993e 100644 --- a/py3/SciServer/Config.py +++ b/py3/SciServer/Config.py @@ -36,6 +36,8 @@ version = "sciserver-v2.1.0" #sciserver release version ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. ComputeUrl = "https://apps.sciserver.org/compute" +SciqueryURL = "https://apps.sciserver.org/sciquery" +ComputeWorkDir = "/home/idies/workspace/" def _load_config(filename): if os.path.exists(filename): diff --git a/py3/SciServer/SciQuery.py b/py3/SciServer/SciQuery.py new file mode 100644 index 0000000..ef59e6d --- /dev/null +++ b/py3/SciServer/SciQuery.py @@ -0,0 +1,352 @@ +from SciServer import Authentication, Config, Files, Jobs +import pandas as pd +import numpy as np +import json +import requests +from functools import lru_cache +from datetime import datetime + +def _get_default_rdb_domain(): + rdb_domains = Jobs.getRDBComputeDomainsNames() + if len(rdb_domains) > 0: + return rdb_domains[0] + else: + raise Exception("There are no rdbComputeDomains available for the user."); + + +class OutputTargetType: + """ + Contains a set of allowed database output types. + """ + FILE_JSON = "FILE_JSON" + FILE_CSV = "FILE_CSV" + DATABASE_TABLE = "TABLE" + + +class FileOutput: + """ + Defines the output of a database query to a file. + """ + def __init__(self, target_name: str = "result.json", target_type: str = OutputTargetType.FILE_JSON, + statement_indexes: list = [1]): + """ + :param target_name: name of the file (string), such as "result.json" + :param target_type: type (string) of the file containing the query result(s) (e.g., "FILE_JSON"). As set of possible values is given by the static members of class 'SciQuery.OutputTargetType' + :param statement_indexes: list of integers or integer. Each integer value denotes the index or position (>=1) of the sql statements whithin the input query, whose resultset is going to be written into this OutputTarget + """ + if type(target_name) != str or type(target_type) != str: + raise ValueError("Invalid type(s) for input parameter(s) 'target_name' or 'target_type'") + self.target_name = target_name + self.target_type = target_type + self.set_statement_indexes(statement_indexes) + + def set_statement_indexes(self, statement_indexes: list = [1]): + """ + Sets the index(es) of the sql statement(s) whithin the input query, whose resultset(s) is(are) going to be written into this OutputTarget. + :param statement_indexes: list of integers, which are the indices (starting with 1) of the sql statements whithin the input query, whose resultsets are going to be written into this OutputTarget. + """ + if type(statement_indexes) != list: + statement_indexes = [statement_indexes] + for index in statement_indexes: + if type(index) != int or index <= 0: + raise ValueError("Invalid type for input parameter 'statement_indexes'") + self.statement_indexes = [i for i in sorted(set(statement_indexes))] + return self + + @classmethod + def get_default(cls): + """ + Gets a OutputTarget object filled with default values: JSON output file where only the 1st SQL statement of the query is written in it. + """ + cls.target_name = "result.json" + cls.target_type = OutputTargetType.FILE_JSON + cls.statement_indexes = [1] + return cls + + def __str__(self): + return "File Output of target_name = {}, target_type= {}, statement_indexes = {}".format(self.target_name, + self.target_type, + self.statement_indexes) + + def __repr__(self): + return "FileOutput(target_name = {}, target_type= {}, statement_indexes = {})".format(self.target_name, + self.target_type, + self.statement_indexes) + + +class DatabaseTableOutput: + """ + Defines the output of a database query to a database table + """ + def __init__(self, table: str = "resultTable", database: str = "", rdb_domain: str = "", schema: str = "", + statement_indexes: list = [1]): + """ + :param table: name of the database table (string), such as "resultTable" + :param database: name of the database (string) where the output table in created. If it is owned explicitly by a user, then it should follow the pattern "mydb:username" + :param rdb_domain: name (string) of the relational database (RDB) compute domain that contains the database. Name of such domains available to the user is returned by the function Jobs.getRDBComputeDomainNames(). + :param schema: database schema (string) + :param statement_indexes: list of integers or integer. Each integer value denotes the index or position (>=1) of the sql statements whithin the input query, whose resultset is going to be written into this OutputTarget + """ + if type(table) != str or type(rdb_domain) != str or type(schema) != str: + raise ValueError("Invalid type(s) for input parameter(s) 'target_name' or 'target_type'") + + self.table = table + self.database = database if database != "" else "mydb:" + Authentication.keystoneUser.userName + self.rdb_domain = rdb_domain if rdb_domain != "" else _get_default_rdb_domain() + self.schema = schema + self.set_statement_indexes(statement_indexes) + self.target_name = ".".join([rdb_domain, database, schema, table]) + self.target_type = OutputTargetType.DATABASE_TABLE + + def set_statement_indexes(self, statement_indexes: list = [1]): + """ + Sets the index(es) of the sql statement(s) whithin the input query, whose resultset(s) is(are) going to be written into this OutputTarget. + :param statement_indexes: list of integers, which are the indices (starting with 1) of the sql statements whithin the input query, whose resultsets are going to be written into this OutputTarget. + """ + if type(statement_indexes) != list: + statement_indexes = [statement_indexes] + for index in statement_indexes: + if type(index) != int or index <= 0: + raise ValueError("Invalid type for input parameter 'statement_indexes'") + self.statement_indexes = [i for i in sorted(set(statement_indexes))] + return self + + @classmethod + def get_default(cls): + """ + Gets a OutputTarget object filled with default values: JSON output file where only the 1st SQL statement of the query is written in it. + """ + cls.target_name = "resultTable" + cls.database = "mydb:" + Authentication.keystoneUser.userName + cls.rdb_domain = _get_default_rdb_domain() + cls.schema = "" + cls.target_type = OutputTargetType.DATABASE_TABLE + cls.statement_indexes = [1] + return cls + + def __str__(self): + return "Database Table Output of table = {}, database= {}, rdb_domain = {}, schema = {}, statement_indexes = {}".format( + self.table, self.database, self.rdb_domain, self.schema, self.statement_indexes) + + def __repr__(self): + return "DatabaseTableOutput(table = {}, database= {}, rdb_domain = {}, schema = {}, statement_indexes = {})".format( + self.table, self.database, self.rdb_domain, self.schema, self.statement_indexes) + + +class RDBJob: + """ + Contains the definition of an RDB job + """ + def __init__(self, job): + """ + :param job: can be the job ID (string), or the + """ + if type(job) != dict: + job = Jobs.getJobDescription(job) + for k, v in job.items(): + setattr(self, k, v) + + def get_output_targets(self): + output_targets = {} + for t in self.targets: + i = (t['location'], t['type']) + if i not in output_targets: + output_targets[i] = [t['resultNumber']] + else: + output_targets[i] = output_targets[i].append(t['resultNumber']) + + targets = [] + for k in output_targets: + if k[1] == OutputTargetType.DATABASE_TABLE: + p = k[0].split(".") + targets.append(DatabaseTableOutput(table=p[3], database=p[1], rdb_domain=p[0], schema=p[2], + statement_indexes=output_targets[k])) + else: + targets.append(FileOutput(target_name=k[0], target_type=k[1], statement_indexes=output_targets[k])) + + # output_targets = [ OutputTarget(l,t,output_targets[(l,t)]) for l,t in output_targets] + # return output_targets + return targets + + def get_results_folder_path(self): + path = ":".join(self.resultsFolderURI.split(":")[1:]) + if not path.startswith(Config.ComputeWorkDir): + path = Config.ComputeWorkDir + path[1:] + return path + + def get_output_target_path(self, output_target): + if output_target.target_type == OutputTargetType.DATABASE_TABLE: + raise ValueError("Output target is not a file but a database") + return self.get_results_folder_path() + output_target.target_name + + def get_fileservice_folder_path(self): + path = ":".join(self.resultsFolderURI.split(":")[1:]) + if path.startswith(Config.ComputeWorkDir): + path = "/" + path.replace(Config.ComputeWorkDir, "", 1) + return path + + def get_start_time(self): + return datetime.fromtimestamp(self.startTime / 1000.0) + + def get_end_time(self): + return datetime.fromtimestamp(self.endTime / 1000.0) + + def get_duration(self): + return self.duration + + def __str__(self): + return "RDB Job of id = {}".format(self.id) + + def __repr__(self): + return "RDBJob(id = {})".format(self.id) + + + +@lru_cache(128) +def _get_file_service(file_service_id=""): + print(file_service_id) + file_services = Files.getFileServices(verbose=False) + for file_service in file_services: + if file_service["name"] == file_service_id or file_service["identifier"] == file_service_id: + return file_service + + if len(file_services) > 0: + return file_services[0] + else: + raise Exception("No fileservices available for the user") + + +def submitQueryJob(sqlQuery, + rdbComputeDomain=None, + databaseContextName=None, + output_targets=FileOutput.get_default(), + resultsFolderPath="", + jobAlias="", + file_service_name=""): + """ + Submits a sql query for execution (as an asynchronous job) inside a relational database (RDB) compute domain. + + :param sqlQuery: sql query (string) + :param rdbComputeDomain: object (dictionary) that defines a relational database (RDB) compute domain. A list of + these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). + :param databaseContextName: database context name (string) on which the sql query is executed. + :param output_targets: object of type SciQuery.OutputTarget defining the output of one or multiple statements + within the input query. Could also be a list of OutputTarget objects. + :param resultsFolderPath: full path to results folder (string) where query output tables are written into. + E.g.: /home/idies/workspace/rootVOlume/username/userVolume/jobsFolder . If not set, + then a default folder will be set automatically. + :param jobAlias: alias (string) of job, defined by the user. + :param file_service_name: name or uuid (string) of FileService where the results folder (resultsFolderPath) is + going to be created. If not defined, then the first available FileService is chosen by default. + :return: the ID (integer) that labels the job. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if + the HTTP request to the SciQuery API returns an error, or if the volumes defined by the user are not available in + the Docker compute domain. + :example: job_id = SciQuery.submitQueryJob('select 1';,None, None, 'myQueryResults', 'myNewJob') + + .. seealso:: Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, + Jobs.cancelJob + """ + + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Sciquery.submitQueryJob" + else: + taskName = "SciScript-Python.Sciquery.submitQueryJob" + + if rdbComputeDomain is None: + rdbComputeDomains = Jobs.getRDBComputeDomains(); + if len(rdbComputeDomains) > 0: + rdbComputeDomain = rdbComputeDomains[0]; + else: + raise Exception("There are no rdbComputeDomains available for the user."); + + if databaseContextName is None: + databaseContexts = rdbComputeDomain.get('databaseContexts'); + if len(databaseContexts) > 0: + databaseContextName = databaseContexts[0].get('name') + else: + raise Exception("rbdComputeDomain has no database contexts available for the user."); + + if type(output_targets) != list: + output_targets = [output_targets] + + targets = [] + for target in output_targets: + for index in target.statement_indexes: + targets.append({'location': target.target_name, 'type': target.target_type, 'resultNumber': index}) + + rdbDomainId = rdbComputeDomain.get('id'); + + file_service = _get_file_service(file_service_name) + resultsFolderPath = file_service['identifier'] + ":" + resultsFolderPath + + dockerJobModel = { + "inputSql": sqlQuery, + "submitterDID": jobAlias, + "databaseContextName": databaseContextName, + "rdbDomainId": rdbDomainId, + "targets": targets, + "resultsFolderURI": resultsFolderPath + } + + print(dockerJobModel) + + data = json.dumps(dockerJobModel).encode() + url = Config.SciqueryURL + "/api/jobs/" + str(rdbDomainId) + "?TaskName=" + taskName; + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + print(url) + res = requests.post(url, data=data, headers=headers, stream=True) + + if res.status_code < 200 or res.status_code >= 300: + raise Exception("Error when submitting a job to the SciQuery API.\nHttp Response from SciQuery API " + + "returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return res.content.decode() + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def execute_query(query, + rdb_compute_domain=None, + database_context=None, + results_folder_path="", + job_alias="", + poll_time=0.2, + file_service_name=""): + output_target = FileOutput("result1.json", OutputTargetType.FILE_JSON).set_statement_indexes([1]) + + jobId = submitQueryJob(sqlQuery=query, rdbComputeDomain=rdb_compute_domain, databaseContextName=database_context, + output_targets=output_target, + resultsFolderPath=results_folder_path, + jobAlias=job_alias, file_service_name=file_service_name) + + job_status = Jobs.waitForJob(jobId, verbose=False, pollTime=poll_time) + job = RDBJob(jobId) + if job.status > 32: + messages = ". ".join(job.messages) if len(job.messages) > 0 else "" + if (job.status == 64): + raise Exception("Query ended with an error. " + messages) + if (job.status == 128): + raise Exception("Query was cancelled. " + messages) + + if Config.isSciServerComputeEnvironment(): + file_path = job.get_output_target_path(output_target) + with open(file_path, ) as f: + j = json.load(f) + else: + file_service_id = job.resultsFolderURI.split(":")[0] + file_service = _get_file_service(file_service_id) + path = job.get_fileservice_folder_path(output_target) + s = Files.download(file_service, path, format="txt", quiet=True) + j = json.loads(s) + + data = np.asarray(j['Result'][0]['Data']) + column_names = j['Result'][0]['ColumnNames'] + name = j['Result'][0]['TableName'] + + df = pd.DataFrame(data=data, columns=column_names) + df.name = name + return df From 81d299ed8abee09a21b96527c79ba19801774394 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Fri, 17 Dec 2021 06:40:11 -0500 Subject: [PATCH 66/79] Added more functions to SciQuery module and improved docs. Removed RDB references in Jobs module. --- py3/SciServer/Jobs.py | 161 +------------------------- py3/SciServer/SciQuery.py | 231 ++++++++++++++++++++++++++++++++------ 2 files changed, 199 insertions(+), 193 deletions(-) diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py index 6a372ef..525598e 100644 --- a/py3/SciServer/Jobs.py +++ b/py3/SciServer/Jobs.py @@ -18,7 +18,7 @@ def getDockerComputeDomains(): :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. :example: dockerComputeDomains = Jobs.getDockerComputeDomains(); - .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getRDBComputeDomains, Jobs.cancelJob + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.cancelJob """ token = Authentication.getToken() if token is not None and token != "": @@ -72,7 +72,7 @@ def getDockerComputeDomainFromName(dockerComputeDomainName, dockerComputeDomains :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. :example: dockerComputeDomain = Jobs.getDockerComputeDomainFromName('dockerComputeDomainAtJHU'); - .. seealso:: Jobs.getDockerComputeDomains, Jobs.getRDBComputeDomains, Jobs.getRDBComputeDomainFromName + .. seealso:: Jobs.getDockerComputeDomains """ if dockerComputeDomainName is None: raise Exception("dockerComputeDomainName is not defined.") @@ -90,83 +90,6 @@ def getDockerComputeDomainFromName(dockerComputeDomainName, dockerComputeDomains raise Exception("DockerComputeDomain of name '" + dockerComputeDomainName + "' is not available or does not exist."); -def getRDBComputeDomains(): - """ - Gets a list of all registered Relational Database (RDB) compute domains that the user has access to. - - :return: a list of dictionaries, each one containing the definition of an RDB compute domain. - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. - :example: rdbComputeDomains = Jobs.getRDBComputeDomains(); - - .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob - """ - token = Authentication.getToken() - if token is not None and token != "": - - if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.Jobs.getRDBComputeDomains" - else: - taskName = "SciScript-Python.Jobs.getRDBComputeDomains" - - url = Config.RacmApiURL + "/jobm/rest/computedomains/rdb?TaskName=" + taskName - headers = {'X-Auth-Token': token, "Content-Type": "application/json"} - res = requests.get(url, headers=headers, stream=True) - if res.status_code != 200: - raise Exception("Error when getting RDB Compute Domains from JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); - else: - return json.loads(res.content.decode()) - else: - raise Exception("User token is not defined. First log into SciServer.") - - -def getRDBComputeDomainsNames(rdbComputeDomains=None): - """ - Returns the names of the RDB compute domains available to the user. - - :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. - :return: an array of strings, each being the name of a rdb compute domain available to the user. - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. - :example: dockerComputeDomainsNames = Files.getDockerComputeDomainsNames(); - - .. seealso:: Files.getRDBComputeDomains - """ - if rdbComputeDomains is None: - rdbComputeDomains = getRDBComputeDomains(); - - rdbComputeDomainsNames = []; - for rdbComputeDomain in rdbComputeDomains: - rdbComputeDomainsNames.append(rdbComputeDomain.get('name')) - - return rdbComputeDomainsNames; - - -def getRDBComputeDomainFromName(rdbComputeDomainName, rdbComputeDomains = None): - """ - Returns an RDBComputeDomain object, given its registered name. - - :param rdbComputeDomainName: name of the RDBComputeDomainName, as shown within the results of Jobs.getRDBComputeDomains() - :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. - :return: an RDBComputeDomain object (dictionary) that defines an RDB compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. - :example: rdbComputeDomain = Jobs.getRDBComputeDomainFromName('rdbComputeDomainAtJHU'); - - .. seealso:: Jobs.getDockerComputeDomains, Jobs.getRDBComputeDomains, Jobs.getDockerComputeDomainFromName - """ - if rdbComputeDomainName is None: - raise Exception("rdbComputeDomainName is not defined.") - else: - if rdbComputeDomains is None: - rdbComputeDomains = getRDBComputeDomains(); - - if rdbComputeDomains.__len__() > 0: - for rdbComputeDomain in rdbComputeDomains: - if rdbComputeDomainName == rdbComputeDomain.get('name'): - return rdbComputeDomain; - else: - raise Exception("There are no RDBComputeDomains available for the user."); - - raise Exception("RDBComputeDomain of name '" + rdbComputeDomainName + "' is not available or does not exist."); - def getJobsList(top=10, open=None, start=None, end=None, type='all'): """ @@ -511,86 +434,6 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN else: raise Exception("User token is not defined. First log into SciServer.") -def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = None, resultsName='queryResults', resultsFolderPath="", jobAlias = ""): - """ - Submits a sql query for execution (as an asynchronous job) inside a relational database (RDB) compute domain. - - :param sqlQuery: sql query (string) - :param rdbComputeDomain: object (dictionary) that defines a relational database (RDB) compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). - :param databaseContextName: database context name (string) on which the sql query is executed. - :param resultsName: name (string) of the table or file (without file type ending) that contains the query result. In case the sql query has multiple statements, should be set to a list of names (e.g., ['result1','result2']). - :param resultsFolderPath: full path to results folder (string) where query output tables are written into. E.g.: /home/idies/workspace/rootVOlume/username/userVolume/jobsFolder . If not set, then a default folder will be set automatically. - :param jobAlias: alias (string) of job, defined by the user. - :return: a dictionary containing the definition of the submitted job. - :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. - :example: job = Jobs.submitRDBQueryJob('select 1';,None, None, 'myQueryResults', 'myNewJob') - - .. seealso:: Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob - """ - - token = Authentication.getToken() - if token is not None and token != "": - - if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.Jobs.submitRDBQueryJob" - else: - taskName = "SciScript-Python.Jobs.submitRDBQueryJob" - - if rdbComputeDomain is None: - rdbComputeDomains = getRDBComputeDomains(); - if rdbComputeDomains .__len__() > 0: - rdbComputeDomain = rdbComputeDomains[0]; - else: - raise Exception("There are no rdbComputeDomains available for the user."); - - if databaseContextName is None: - databaseContexts = rdbComputeDomain.get('databaseContexts'); - if databaseContexts.__len__() > 0: - databaseContextName = databaseContexts[0].get('name') - else: - raise Exception("rbdComputeDomain has no database contexts available for the user."); - - targets = []; - if type(resultsName) == str: - targets.append({'location': resultsName, 'type': 'FILE_CSV', 'resultNumber': 1}); - elif type(resultsName) == list: - if len(set(resultsName)) != len(resultsName): - raise Exception("Elements of parameter 'resultsName' must be unique"); - - for i in range(len(resultsName)): - if type(resultsName[i]) == str: - targets.append({'location': resultsName[i], 'type': 'FILE_CSV', 'resultNumber': i+1}); - else: - raise Exception("Elements of array 'resultsName' are not strings"); - - else: - raise Exception("Type of parameter 'resultsName' is not supported"); - - - rdbDomainId = rdbComputeDomain.get('id'); - - dockerJobModel = { - "inputSql": sqlQuery, - "submitterDID": jobAlias, - "databaseContextName": databaseContextName, - "rdbDomainId": rdbDomainId, - "targets": targets, - "resultsFolderURI":resultsFolderPath - } - - data = json.dumps(dockerJobModel).encode() - url = Config.RacmApiURL + "/jobm/rest/jobs/rdb?TaskName="+taskName; - headers = {'X-Auth-Token': token, "Content-Type": "application/json"} - res = requests.post(url, data=data, headers=headers, stream=True) - - if res.status_code != 200: - raise Exception("Error when submitting a job to the JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); - else: - return (json.loads(res.content.decode())).get('id') - else: - raise Exception("User token is not defined. First log into SciServer.") - - def cancelJob(jobId): """ Cancels the execution of a job. diff --git a/py3/SciServer/SciQuery.py b/py3/SciServer/SciQuery.py index ef59e6d..1dc8d28 100644 --- a/py3/SciServer/SciQuery.py +++ b/py3/SciServer/SciQuery.py @@ -7,7 +7,7 @@ from datetime import datetime def _get_default_rdb_domain(): - rdb_domains = Jobs.getRDBComputeDomainsNames() + rdb_domains = getRDBComputeDomainNames() if len(rdb_domains) > 0: return rdb_domains[0] else: @@ -204,7 +204,6 @@ def __repr__(self): @lru_cache(128) def _get_file_service(file_service_id=""): - print(file_service_id) file_services = Files.getFileServices(verbose=False) for file_service in file_services: if file_service["name"] == file_service_id or file_service["identifier"] == file_service_id: @@ -219,33 +218,32 @@ def _get_file_service(file_service_id=""): def submitQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName=None, - output_targets=FileOutput.get_default(), + outputTargets=FileOutput.get_default(), resultsFolderPath="", jobAlias="", - file_service_name=""): + fileServiceName=""): """ Submits a sql query for execution (as an asynchronous job) inside a relational database (RDB) compute domain. :param sqlQuery: sql query (string) :param rdbComputeDomain: object (dictionary) that defines a relational database (RDB) compute domain. A list of - these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). + these kind of objects available to the user is returned by the function 'getRDBComputeDomains'. :param databaseContextName: database context name (string) on which the sql query is executed. - :param output_targets: object of type SciQuery.OutputTarget defining the output of one or multiple statements + :param outputTargets: object of type SciQuery.OutputTarget defining the output of one or multiple statements within the input query. Could also be a list of OutputTarget objects. :param resultsFolderPath: full path to results folder (string) where query output tables are written into. E.g.: /home/idies/workspace/rootVOlume/username/userVolume/jobsFolder . If not set, then a default folder will be set automatically. :param jobAlias: alias (string) of job, defined by the user. - :param file_service_name: name or uuid (string) of FileService where the results folder (resultsFolderPath) is + :param fileServiceName: name or uuid (string) of FileService where the results folder (resultsFolderPath) is going to be created. If not defined, then the first available FileService is chosen by default. :return: the ID (integer) that labels the job. :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the SciQuery API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. - :example: job_id = SciQuery.submitQueryJob('select 1';,None, None, 'myQueryResults', 'myNewJob') + :example: job_id = SciQuery.submitQueryJob('select 1;') - .. seealso:: Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, - Jobs.cancelJob + .. seealso:: SciQuery.submitQueryJob, SciQuery.getJobStatus, SciQuery.getJob """ token = Authentication.getToken() @@ -257,7 +255,7 @@ def submitQueryJob(sqlQuery, taskName = "SciScript-Python.Sciquery.submitQueryJob" if rdbComputeDomain is None: - rdbComputeDomains = Jobs.getRDBComputeDomains(); + rdbComputeDomains = getRDBComputeDomains(); if len(rdbComputeDomains) > 0: rdbComputeDomain = rdbComputeDomains[0]; else: @@ -270,17 +268,17 @@ def submitQueryJob(sqlQuery, else: raise Exception("rbdComputeDomain has no database contexts available for the user."); - if type(output_targets) != list: - output_targets = [output_targets] + if type(outputTargets) != list: + outputTargets = [outputTargets] targets = [] - for target in output_targets: + for target in outputTargets: for index in target.statement_indexes: targets.append({'location': target.target_name, 'type': target.target_type, 'resultNumber': index}) rdbDomainId = rdbComputeDomain.get('id'); - file_service = _get_file_service(file_service_name) + file_service = _get_file_service(fileServiceName) resultsFolderPath = file_service['identifier'] + ":" + resultsFolderPath dockerJobModel = { @@ -292,12 +290,9 @@ def submitQueryJob(sqlQuery, "resultsFolderURI": resultsFolderPath } - print(dockerJobModel) - data = json.dumps(dockerJobModel).encode() url = Config.SciqueryURL + "/api/jobs/" + str(rdbDomainId) + "?TaskName=" + taskName; headers = {'X-Auth-Token': token, "Content-Type": "application/json"} - print(url) res = requests.post(url, data=data, headers=headers, stream=True) if res.status_code < 200 or res.status_code >= 300: @@ -309,19 +304,40 @@ def submitQueryJob(sqlQuery, raise Exception("User token is not defined. First log into SciServer.") -def execute_query(query, - rdb_compute_domain=None, - database_context=None, - results_folder_path="", - job_alias="", +def executeQuery(sqlQuery, + rdbComputeDomain=None, + databaseContextName=None, + resultsFolderPath="", + jobAlias="", poll_time=0.2, - file_service_name=""): + fileServiceName=""): + """ + Returns the query result (as a Pandas data frame) of a sql query submitted as a job to a relational database (RDB) compute domain. + + :param sqlQuery: sql query (string) + :param rdbComputeDomain: object (dictionary) that defines a relational database (RDB) compute domain. A list of + these kind of objects available to the user is returned by the function 'getRDBComputeDomains'. + :param databaseContextName: database context name (string) on which the sql query is executed. + :param resultsFolderPath: full path to results folder (string) where query output tables are written into. + E.g.: /home/idies/workspace/rootVOlume/username/userVolume/jobsFolder . If not set, + then a default folder will be set automatically. + :param jobAlias: alias (string) of job, defined by the user. + :param fileServiceName: name or uuid (string) of FileService where the results folder (resultsFolderPath) is + going to be created. If not defined, then the first available FileService is chosen by default. + :return: Pandas data frame containing the result of the query. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if + the HTTP request to the SciQuery API returns an error. + :example: df = SciQuery.executeQuery('select 1;') + + .. seealso:: SciQuery.submitQueryJob, SciQuery.getJobStatus, SciQuery.getJob + Jobs.cancelJob + """ output_target = FileOutput("result1.json", OutputTargetType.FILE_JSON).set_statement_indexes([1]) - jobId = submitQueryJob(sqlQuery=query, rdbComputeDomain=rdb_compute_domain, databaseContextName=database_context, - output_targets=output_target, - resultsFolderPath=results_folder_path, - jobAlias=job_alias, file_service_name=file_service_name) + jobId = submitQueryJob(sqlQuery=sqlQuery, rdbComputeDomain=rdbComputeDomain, databaseContextName=databaseContextName, + outputTargets=output_target, + resultsFolderPath=resultsFolderPath, + jobAlias=jobAlias, fileServiceName=fileServiceName) job_status = Jobs.waitForJob(jobId, verbose=False, pollTime=poll_time) job = RDBJob(jobId) @@ -343,10 +359,157 @@ def execute_query(query, s = Files.download(file_service, path, format="txt", quiet=True) j = json.loads(s) - data = np.asarray(j['Result'][0]['Data']) - column_names = j['Result'][0]['ColumnNames'] - name = j['Result'][0]['TableName'] - - df = pd.DataFrame(data=data, columns=column_names) - df.name = name + result=j['Result'][0] + df=pd.DataFrame(result['Data'],columns=result['ColumnNames']) + df.name = result['TableName'] return df + + +def getRDBComputeDomains(): + """ + Gets a list of all registered Relational Database (RDB) compute domains that the user has access to. + + :return: a list of dictionaries, each one containing the definition of an RDB compute domain. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: rdb_compute_domains = SciQuery.getRDBComputeDomains(); + + .. seealso:: SciQuery.executeQuery, SciQuery.submitQueryJob + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciQuery.get_compute_domains" + else: + taskName = "SciScript-Python.SciQuery.get_compute_domains" + + url = Config.RacmApiURL + "/jobm/rest/computedomains/rdb?TaskName=" + taskName + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + if res.status_code != 200: + raise Exception("Error when getting RDB Compute Domains from JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def getRDBComputeDomainNames(rdbComputeDomains=None): + """ + Returns the names of the RDB compute domains available to the user. + + :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. + :return: an array of strings, each being the name of a rdb compute domain available to the user. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: dockerComputeDomainsNames = Files.getDockerComputeDomainsNames(); + + .. seealso:: Files.getRDBComputeDomains + """ + if rdbComputeDomains is None: + rdbComputeDomains = getRDBComputeDomains(); + + rdbComputeDomainsNames = []; + for rdbComputeDomain in rdbComputeDomains: + rdbComputeDomainsNames.append(rdbComputeDomain.get('name')) + + return rdbComputeDomainsNames; + +def getRDBComputeDomainFromName(rdbComputeDomainName, rdbComputeDomains = None): + """ + Returns an RDBComputeDomain object, given its registered name. + + :param rdbComputeDomainName: name of the RDBComputeDomainName, as shown within the results of Jobs.getRDBComputeDomains() + :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. + :return: an RDBComputeDomain object (dictionary) that defines an RDB compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: rdbComputeDomain = SciQuery.getRDBComputeDomainFromName(rdbComputeDomainName); + + .. seealso:: SciQuery.getRDBComputeDomains + """ + if rdbComputeDomainName is None: + raise Exception("rdbComputeDomainName is not defined.") + else: + if rdbComputeDomains is None: + rdbComputeDomains = getRDBComputeDomains(); + + if rdbComputeDomains.__len__() > 0: + for rdbComputeDomain in rdbComputeDomains: + if rdbComputeDomainName == rdbComputeDomain.get('name'): + return rdbComputeDomain; + else: + raise Exception("There are no RDBComputeDomains available for the user."); + + raise Exception("RDBComputeDomain of name '" + rdbComputeDomainName + "' is not available or does not exist."); + +def getJobsList(top=10, open=None, start=None, end=None): + """ + Gets the list of SciQuery Jobs submitted by the user. + + :param top: top number of jobs (integer) returned. If top=None, then all jobs are returned. + :param open: If set to 'True', then only returns jobs that have not finished executing and wrapped up (status <= FINISHED). If set to 'False' then only returnes jobs that are still running. If set to 'None', then returns both finished and unfinished jobs. + :param start: The earliest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no lower bound on date. + :param end: The latest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no upper bound on date. + :return: a list of dictionaries, each one containing the definition of a submitted job. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. + :example: jobs = SciQuery.getJobsList(top=2); + + .. seealso:: SciQuery,getJob, SciQuery.getJobStatus, SciQuery.cancelJob + """ + job_dict_list = Jobs.getJobsList(top=top, open=open, start=start, end=end, type='rdb') + rdb_job_list = [] + for job_dict in job_dict_list: + rdb_job_list.append(RDBJob(job_dict)) + return rdb_job_list + +def getJob(jobId): + """ + Gets the definition of the job as a RDBJob object. + + :param jobId: Id of job + :return: RDBJob object containing the description or definition of the job. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. + :example: job = SciQuery.getJob(jobId) + + .. seealso:: SciQuery.getJob, SciQuery.cancelJob, SciQuery.submitQueryJob + """ + return RDBJob(Jobs.getJobDescription(jobId)) + +def getJobStatus(jobId): + """ + Gets a dictionary with the job status as an integer value, together with its semantic meaning. The integer value is a power of 2, that is, 1:PENDING, 2:QUEUED, 4:ACCEPTED, 8:STARTED, 16:FINISHED, 32:SUCCESS, 64:ERROR and 128:CANCELED + + :param jobId: Id of job (integer). + :return: dictionary with the integer value of the job status, as well as its semantic meaning. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. + :example: status = SciQuery.getJobStatus(jobId) + + .. seealso:: SciQuery.cancelJob, SciQuery.waitForJob, SciQuery.getJob, SciQuery.cancelJob + """ + return Jobs.getJobStatus(jobId) + +def cancelJob(jobId): + """ + Cancels the execution of a job. + + :param jobId: Id of the job (integer) + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error. + :example: SciQuery.cancelJob(jobId); + + .. seealso:: SciQuery.getJobStatus, SciQuery.getJobDescription + """ + Jobs.cancelJob(jobId) + +def waitForJob(jobId, verbose=False, pollTime = 5): + """ + Queries the job status regularly and waits until the job is completed. + + :param jobId: id of job (integer) + :param verbose: if True, will print "wait" messages on the screen while the job is still running. If False, will suppress the printing of messages on the screen. + :param pollTime: idle time interval (integer, in seconds) before querying again for the job status. Minimum value allowed is 5 seconds. + :return: After the job is finished, returns a dictionary object containing the job status. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: jobStatus = SciQuery.waitForJob(jobId) + + .. seealso:: SciQuery.getJobStatus, SciQuery.getJobDescription + """ + return Jobs.waitForJob(jobId=jobId, verbose=verbose, pollTime = pollTime) \ No newline at end of file From 05aa804f5c16660a9f7c5b41a32a95b50e82c023 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Thu, 24 Feb 2022 09:09:08 -0500 Subject: [PATCH 67/79] Added metadata functions and improved docs. --- py3/SciServer/SciQuery.py | 517 +++++++++++++++++++++++++++++++++++--- 1 file changed, 480 insertions(+), 37 deletions(-) diff --git a/py3/SciServer/SciQuery.py b/py3/SciServer/SciQuery.py index 1dc8d28..c1eddae 100644 --- a/py3/SciServer/SciQuery.py +++ b/py3/SciServer/SciQuery.py @@ -31,8 +31,10 @@ def __init__(self, target_name: str = "result.json", target_type: str = OutputTa statement_indexes: list = [1]): """ :param target_name: name of the file (string), such as "result.json" - :param target_type: type (string) of the file containing the query result(s) (e.g., "FILE_JSON"). As set of possible values is given by the static members of class 'SciQuery.OutputTargetType' - :param statement_indexes: list of integers or integer. Each integer value denotes the index or position (>=1) of the sql statements whithin the input query, whose resultset is going to be written into this OutputTarget + :param target_type: type (string) of the file containing the query result(s) (e.g., "FILE_JSON"). + As set of possible values is given by the static members of class 'SciQuery.OutputTargetType' + :param statement_indexes: list of integers or integer. Each integer value denotes the index or position (>=1) + of the sql statements whithin the input query, whose resultset is going to be written into this OutputTarget """ if type(target_name) != str or type(target_type) != str: raise ValueError("Invalid type(s) for input parameter(s) 'target_name' or 'target_type'") @@ -42,8 +44,10 @@ def __init__(self, target_name: str = "result.json", target_type: str = OutputTa def set_statement_indexes(self, statement_indexes: list = [1]): """ - Sets the index(es) of the sql statement(s) whithin the input query, whose resultset(s) is(are) going to be written into this OutputTarget. - :param statement_indexes: list of integers, which are the indices (starting with 1) of the sql statements whithin the input query, whose resultsets are going to be written into this OutputTarget. + Sets the index(es) of the sql statement(s) whithin the input query, whose resultset(s) is(are) going to + be written into this OutputTarget. + :param statement_indexes: list of integers, which are the indices (starting with 1) of the sql statements + within the input query, whose resultsets are going to be written into this OutputTarget. """ if type(statement_indexes) != list: statement_indexes = [statement_indexes] @@ -56,7 +60,8 @@ def set_statement_indexes(self, statement_indexes: list = [1]): @classmethod def get_default(cls): """ - Gets a OutputTarget object filled with default values: JSON output file where only the 1st SQL statement of the query is written in it. + Gets a OutputTarget object filled with default values: JSON output file where only the 1st SQL statement of + the query is written in it. """ cls.target_name = "result.json" cls.target_type = OutputTargetType.FILE_JSON @@ -82,10 +87,13 @@ def __init__(self, table: str = "resultTable", database: str = "", rdb_domain: s statement_indexes: list = [1]): """ :param table: name of the database table (string), such as "resultTable" - :param database: name of the database (string) where the output table in created. If it is owned explicitly by a user, then it should follow the pattern "mydb:username" - :param rdb_domain: name (string) of the relational database (RDB) compute domain that contains the database. Name of such domains available to the user is returned by the function Jobs.getRDBComputeDomainNames(). + :param database: name of the database (string) where the output table in created. If it is owned explicitly by + a user, then it should follow the pattern "mydb:username" + :param rdb_domain: name (string) of the relational database (RDB) compute domain that contains the database. + Name of such domains available to the user is returned by the function Jobs.getRDBComputeDomainNames(). :param schema: database schema (string) - :param statement_indexes: list of integers or integer. Each integer value denotes the index or position (>=1) of the sql statements whithin the input query, whose resultset is going to be written into this OutputTarget + :param statement_indexes: list of integers or integer. Each integer value denotes the index or position (>=1) + of the sql statements whithin the input query, whose resultset is going to be written into this OutputTarget """ if type(table) != str or type(rdb_domain) != str or type(schema) != str: raise ValueError("Invalid type(s) for input parameter(s) 'target_name' or 'target_type'") @@ -100,8 +108,10 @@ def __init__(self, table: str = "resultTable", database: str = "", rdb_domain: s def set_statement_indexes(self, statement_indexes: list = [1]): """ - Sets the index(es) of the sql statement(s) whithin the input query, whose resultset(s) is(are) going to be written into this OutputTarget. - :param statement_indexes: list of integers, which are the indices (starting with 1) of the sql statements whithin the input query, whose resultsets are going to be written into this OutputTarget. + Sets the index(es) of the sql statement(s) whithin the input query, whose resultset(s) is(are) going to be + written into this OutputTarget. + :param statement_indexes: list of integers, which are the indices (starting with 1) of the sql statements + within the input query, whose resultsets are going to be written into this OutputTarget. """ if type(statement_indexes) != list: statement_indexes = [statement_indexes] @@ -114,7 +124,8 @@ def set_statement_indexes(self, statement_indexes: list = [1]): @classmethod def get_default(cls): """ - Gets a OutputTarget object filled with default values: JSON output file where only the 1st SQL statement of the query is written in it. + Gets a OutputTarget object filled with default values: JSON output file where only the 1st SQL statement of + the query is written in it. """ cls.target_name = "resultTable" cls.database = "mydb:" + Authentication.keystoneUser.userName @@ -125,12 +136,12 @@ def get_default(cls): return cls def __str__(self): - return "Database Table Output of table = {}, database= {}, rdb_domain = {}, schema = {}, statement_indexes = {}".format( - self.table, self.database, self.rdb_domain, self.schema, self.statement_indexes) + return "Database Table Output of table= {}, database= {}, rdb_domain= {}, schema= {}, statement_indexes= {}"\ + .format(self.table, self.database, self.rdb_domain, self.schema, self.statement_indexes) def __repr__(self): - return "DatabaseTableOutput(table = {}, database= {}, rdb_domain = {}, schema = {}, statement_indexes = {})".format( - self.table, self.database, self.rdb_domain, self.schema, self.statement_indexes) + return "DatabaseTableOutput(table= {}, database= {}, rdb_domain= {}, schema= {}, statement_indexes= {})"\ + .format(self.table, self.database, self.rdb_domain, self.schema, self.statement_indexes) class RDBJob: @@ -294,7 +305,6 @@ def submitQueryJob(sqlQuery, url = Config.SciqueryURL + "/api/jobs/" + str(rdbDomainId) + "?TaskName=" + taskName; headers = {'X-Auth-Token': token, "Content-Type": "application/json"} res = requests.post(url, data=data, headers=headers, stream=True) - if res.status_code < 200 or res.status_code >= 300: raise Exception("Error when submitting a job to the SciQuery API.\nHttp Response from SciQuery API " + "returned status code " + str(res.status_code) + ":\n" + res.content.decode()); @@ -312,7 +322,8 @@ def executeQuery(sqlQuery, poll_time=0.2, fileServiceName=""): """ - Returns the query result (as a Pandas data frame) of a sql query submitted as a job to a relational database (RDB) compute domain. + Returns the query result (as a Pandas data frame) of a sql query submitted as a job to a + relational database (RDB) compute domain. :param sqlQuery: sql query (string) :param rdbComputeDomain: object (dictionary) that defines a relational database (RDB) compute domain. A list of @@ -370,7 +381,8 @@ def getRDBComputeDomains(): Gets a list of all registered Relational Database (RDB) compute domains that the user has access to. :return: a list of dictionaries, each one containing the definition of an RDB compute domain. - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). + Throws an exception if the HTTP request to the JOBM API returns an error. :example: rdb_compute_domains = SciQuery.getRDBComputeDomains(); .. seealso:: SciQuery.executeQuery, SciQuery.submitQueryJob @@ -387,7 +399,8 @@ def getRDBComputeDomains(): headers = {'X-Auth-Token': token, "Content-Type": "application/json"} res = requests.get(url, headers=headers, stream=True) if res.status_code != 200: - raise Exception("Error when getting RDB Compute Domains from JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception("Error when getting RDB Compute Domains from JOBM API.\nHttp Response from JOBM API returned" + " status code " + str(res.status_code) + ":\n" + res.content.decode()); else: return json.loads(res.content.decode()) else: @@ -398,9 +411,11 @@ def getRDBComputeDomainNames(rdbComputeDomains=None): """ Returns the names of the RDB compute domains available to the user. - :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. + :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by + Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. :return: an array of strings, each being the name of a rdb compute domain available to the user. - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). + Throws an exception if the HTTP request to the RACM API returns an error. :example: dockerComputeDomainsNames = Files.getDockerComputeDomainsNames(); .. seealso:: Files.getRDBComputeDomains @@ -418,10 +433,14 @@ def getRDBComputeDomainFromName(rdbComputeDomainName, rdbComputeDomains = None): """ Returns an RDBComputeDomain object, given its registered name. - :param rdbComputeDomainName: name of the RDBComputeDomainName, as shown within the results of Jobs.getRDBComputeDomains() - :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. - :return: an RDBComputeDomain object (dictionary) that defines an RDB compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :param rdbComputeDomainName: name of the RDBComputeDomainName, as shown within the results of + Jobs.getRDBComputeDomains() + :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by + Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. + :return: an RDBComputeDomain object (dictionary) that defines an RDB compute domain. A list of these kind of objects + available to the user is returned by the function Jobs.getRDBComputeDomains(). + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). + Throws an exception if the HTTP request to the JOBM API returns an error. :example: rdbComputeDomain = SciQuery.getRDBComputeDomainFromName(rdbComputeDomainName); .. seealso:: SciQuery.getRDBComputeDomains @@ -446,11 +465,16 @@ def getJobsList(top=10, open=None, start=None, end=None): Gets the list of SciQuery Jobs submitted by the user. :param top: top number of jobs (integer) returned. If top=None, then all jobs are returned. - :param open: If set to 'True', then only returns jobs that have not finished executing and wrapped up (status <= FINISHED). If set to 'False' then only returnes jobs that are still running. If set to 'None', then returns both finished and unfinished jobs. - :param start: The earliest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no lower bound on date. - :param end: The latest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no upper bound on date. + :param open: If set to 'True', then only returns jobs that have not finished executing and wrapped up + (status <= FINISHED). If set to 'False' then only returnes jobs that are still running. If set to 'None', + then returns both finished and unfinished jobs. + :param start: The earliest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. + If set to 'None', then there is no lower bound on date. + :param end: The latest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. + If set to 'None', then there is no upper bound on date. :return: a list of dictionaries, each one containing the definition of a submitted job. - :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request + to the JOBM API returns an error. :example: jobs = SciQuery.getJobsList(top=2); .. seealso:: SciQuery,getJob, SciQuery.getJobStatus, SciQuery.cancelJob @@ -467,7 +491,8 @@ def getJob(jobId): :param jobId: Id of job :return: RDBJob object containing the description or definition of the job. - :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request + to the JOBM API returns an error. :example: job = SciQuery.getJob(jobId) .. seealso:: SciQuery.getJob, SciQuery.cancelJob, SciQuery.submitQueryJob @@ -476,11 +501,13 @@ def getJob(jobId): def getJobStatus(jobId): """ - Gets a dictionary with the job status as an integer value, together with its semantic meaning. The integer value is a power of 2, that is, 1:PENDING, 2:QUEUED, 4:ACCEPTED, 8:STARTED, 16:FINISHED, 32:SUCCESS, 64:ERROR and 128:CANCELED + Gets a dictionary with the job status as an integer value, together with its semantic meaning. The integer value is + a power of 2, that is, 1:PENDING, 2:QUEUED, 4:ACCEPTED, 8:STARTED, 16:FINISHED, 32:SUCCESS, 64:ERROR, 128:CANCELED :param jobId: Id of job (integer). :return: dictionary with the integer value of the job status, as well as its semantic meaning. - :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request + to the JOBM API returns an error. :example: status = SciQuery.getJobStatus(jobId) .. seealso:: SciQuery.cancelJob, SciQuery.waitForJob, SciQuery.getJob, SciQuery.cancelJob @@ -492,7 +519,8 @@ def cancelJob(jobId): Cancels the execution of a job. :param jobId: Id of the job (integer) - :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if + the HTTP request to the JOBM API returns an error. :example: SciQuery.cancelJob(jobId); .. seealso:: SciQuery.getJobStatus, SciQuery.getJobDescription @@ -504,12 +532,427 @@ def waitForJob(jobId, verbose=False, pollTime = 5): Queries the job status regularly and waits until the job is completed. :param jobId: id of job (integer) - :param verbose: if True, will print "wait" messages on the screen while the job is still running. If False, will suppress the printing of messages on the screen. - :param pollTime: idle time interval (integer, in seconds) before querying again for the job status. Minimum value allowed is 5 seconds. + :param verbose: if True, will print "wait" messages on the screen while the job is still running. If False, it will + suppress the printing of messages on the screen. + :param pollTime: idle time interval (integer, in seconds) before querying again for the job status. Minimum value + allowed is 5 seconds. :return: After the job is finished, returns a dictionary object containing the job status. - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). + Throws an exception if the HTTP request to the JOBM API returns an error. :example: jobStatus = SciQuery.waitForJob(jobId) .. seealso:: SciQuery.getJobStatus, SciQuery.getJobDescription """ - return Jobs.waitForJob(jobId=jobId, verbose=verbose, pollTime = pollTime) \ No newline at end of file + return Jobs.waitForJob(jobId=jobId, verbose=verbose, pollTime = pollTime) + + +def getDatabasesMetadata(rdbComputeDomain, format="pandas"): + """ + Gets metadata (name and description) of databases in an RDBComputeDomain. + + :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) that + defines a RDBComputeDomain. A list of these kind of objects available to the user is returned by + the function 'getRDBComputeDomains'. + :param format: String that defines the returned format. Either "pandas" for a pandas dataframe, or "dict" for + a dictionary object. + :return: pandas dataframe or dict (depending of the value of 'param') with associated metadata. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). + Throws an exception if the HTTP request to the JOBM API returns an error. + :example: metadata = SciQuery.getDatabasesMetadata(rdbComputeDomainName); + + .. seealso:: SciQuery.getDatabaseNames + """ + + if type(rdbComputeDomain) == str: + rdbComputeDomain = getRDBComputeDomainFromName(rdbComputeDomain) + databaseContexts = rdbComputeDomain.get("databaseContexts") + if format == "dict": + return databaseContexts + columnNames = ['database_name', 'database_description'] + data = [] + for i in range(len(databaseContexts)): + data.append([databaseContexts[i]['name'], databaseContexts[i]['description']]) + df = pd.DataFrame(data=data, columns=columnNames) + return df + + +def getDatabaseNames(rdbComputeDomain): + """ + Gets a list of the names of databases in an RDBComputeDomain + + :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) that + defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by + the function 'getRDBComputeDomains'. + :return: array of database names (strings) + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). + Throws an exception if the HTTP request to the JOBM API returns an error. + :example: dbnames = SciQuery.getDatabaseNames(rdbComputeDomainName); + + .. seealso:: SciQuery.getDatabasesMetadata + """ + databases = getDatabasesMetadata(rdbComputeDomain, format="pandas") + return [name for name in databases['database_name']] + +def getRDBComputeDomainsMetadata(format="pandas", includeDatabases=False): + """ + Gets metadata related to all relational database (RDB) compute domains (RDBComputeDomains) available to the user. + + :param format: String that defines the returned format. Either "pandas" for a pandas dataframe, or "dict" for + a dictionary object. + :param includeDatabases: Boolean parameter. If True, it will return metadata related to all available databases in + each RDBComputeDomain as well. + :return: pandas dataframe or dict (depending of the value of 'param') containing associated metadata. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). + Throws an exception if the HTTP request to the JOBM API returns an error. + :example: metadata = SciQuery.getRDBComputeDomainsMetadata(); + + .. seealso:: SciQuery.getDatabasesMetadata, Sciquery.getDatabaseNames + """ + rdbComputeDomains = getRDBComputeDomains() + if format == "dict": + return rdbComputeDomains + columnNames = ['domain_name', 'domain_description'] + if includeDatabases: + columnNames.append("database_name") + columnNames.append("database_description") + data = [] + for i in range(len(rdbComputeDomains)): + domName = rdbComputeDomains[i].get('name') + domDescr = rdbComputeDomains[i].get('description') + dbs = rdbComputeDomains[i].get('databaseContexts') + if includeDatabases: + for j in range(len(dbs)): + data.append([domName, domDescr, dbs[j].get('name'), dbs[j].get('description')]) + else: + data.append([domName, domDescr]) + + df = pd.DataFrame(data=data, columns=columnNames) + return df + + + + +class MetadataType: + """ + Contains a set of metadata types. + """ + TABLES = "TABLES" + VIEWS = "VIEWS" + COLUMNS = "COLUMNS" + ROUTINES = "ROUTINES" + CONSTRAINTS = "CONSTRAINTS" + PARAMETERS = "PARAMETERS" + + +def _getMetadata(rdbComputeDomain, databaseContextName, resourceName="", metadataType=None, format="pandas"): + """ + Utility function for the use of other metadata functions. + + :param rdbComputeDomain: object (dictionary) that defines a relational database (RDB) compute domain. A list of + these kind of objects available to the user is returned by the function 'getRDBComputeDomains'. + :param databaseContextName: database context name (string) on which the sql query is executed. + :return: the ID (integer) that labels the job. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if + the HTTP request to the SciQuery API returns an error, or if the volumes defined by the user are not available in + the Docker compute domain. + :example: job_id = SciQuery.submitQueryJob('select 1;') + + .. seealso:: SciQuery.submitQueryJob, SciQuery.getJobStatus, SciQuery.getJob + """ + + token = Authentication.getToken() + if token is not None and token != "": + + if metadataType not in [a for a in dir(MetadataType) if not a.startswith("__")]: + raise ValueError("Invalid value of metadataType paramter") + + if format not in ["pandas", "dict"]: + raise ValueError("Invalid value of format paramter") + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Sciquery.getMetadata_" + metadataType + else: + taskName = "SciScript-Python.Sciquery.getMetadata_" + metadataType + + if type(rdbComputeDomain) == str: + rdbComputeDomain = getRDBComputeDomainFromName(rdbComputeDomain) + + rdbComputeDomainId = rdbComputeDomain.get("id") + + url = Config.SciqueryURL + "/api/metadata/{0}/{1}/".format(rdbComputeDomainId, databaseContextName); + if metadataType == MetadataType.TABLES: + url += "tables" + elif metadataType == MetadataType.VIEWS: + url += "views" + elif metadataType == MetadataType.ROUTINES: + url += "routines" + elif metadataType == MetadataType.COLUMNS: + url += "{0}/{1}".format(resourceName, "columns") + elif metadataType == MetadataType.PARAMETERS: + url += "{0}/{1}".format(resourceName, "parameters") + elif metadataType == MetadataType.CONSTRAINTS: + url += "{0}/{1}".format(resourceName, "constraints") + else: + raise ValueError("Wrong metadataType parameter value of " + metadataType) + + url += "?taskName=" + taskName + + headers = {'X-Auth-Token': token} + res = requests.get(url, headers=headers, stream=True) + + if res.status_code < 200 or res.status_code >= 300: + raise Exception("Error when getting metadata from SciQuery API.\nHttp Response from SciQuery API " + + "returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + res = json.loads(res.content.decode()) + result = res['Result'][0] + if format == "pandas": + df = pd.DataFrame(result['Data'], columns=[c.upper() for c in result['ColumnNames']]) + df.name = result['TableName'] + return df + else: + return result + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def getTablesMetadata(rdbComputeDomain, databaseContextName, format="pandas"): + """ + Gets metadata related to tables in a particular database belonging to an RDBComputeDomain. + + :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) + that defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by + the function 'getRDBComputeDomains'. + :param databaseContextName: name (string) of the database. + :param format: String that defines the returned format. Either "pandas" for a pandas dataframe, + or "dict" for a dictionary object. + :return: pandas dataframe or dict (depending of the value of 'param') containing associated metadata. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). + Throws an exception if the HTTP request to the JOBM API returns an error. + :example: metadata = SciQuery.getTablesMetadata(rdbComputeDomain, databaseContextName) + + .. seealso:: SciQuery.getTableNames, SciQuery.getDatabasesMetadata, Sciquery.getDatabaseNames + """ + return _getMetadata(rdbComputeDomain, databaseContextName, metadataType=MetadataType.TABLES, format=format) + +def getTableNames(rdbComputeDomain, databaseContextName): + """ + Gets a list of the names of tables in a particular database belonging to an RDBComputeDomain. + + :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) + that defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by + the function 'getRDBComputeDomains'. + :param databaseContextName: name (string) of the database. + :return: array of table names (strings) + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). + Throws an exception if the HTTP request to the JOBM API returns an error. + :example: tablenames = getTableNames(rdbComputeDomain, databaseContextName) + + .. seealso:: SciQuery.getTablesMetadata, SciQuery.getDatabasesMetadata, Sciquery.getDatabaseNames + """ + tables = getTablesMetadata(rdbComputeDomain, databaseContextName, format="pandas") + return [name for name in tables['TABLE_NAME']] + + +def getViewsMetadata(rdbComputeDomain, databaseContextName, format="pandas"): + """ + Gets metadata related to views in a particular database belonging to an RDBComputeDomain. + + :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) that + defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by + the function 'getRDBComputeDomains'. + :param databaseContextName: name (string) of the database. + :param format: String that defines the returned format. Either "pandas" for a pandas dataframe, or "dict" for + a dictionary object. + :return: pandas dataframe or dict (depending of the value of 'param') containing associated metadata. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). + Throws an exception if the HTTP request to the JOBM API returns an error. + :example: metadata = SciQuery.getViewsMetadata(rdbComputeDomain, databaseContextName) + + .. seealso:: SciQuery.getTablesMetadata, SciQuery.getViewNames, SciQuery.getDatabasesMetadata + """ + return _getMetadata(rdbComputeDomain, databaseContextName, metadataType=MetadataType.VIEWS, format=format) + + +def getViewNames(rdbComputeDomain, databaseContextName): + """ + Gets a list of the names of views in a particular database belonging to an RDBComputeDomain. + + :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) + that defines an RDBComputeDomain. A list of + these kind of objects available to the user is returned by the function 'getRDBComputeDomains'. + :param databaseContextName: name (string) of the database. + :return: array of view names (strings) + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). + Throws an exception if the HTTP request to the JOBM API returns an error. + :example: viewnames = SciQuery.getViewNames(rdbComputeDomain, databaseContextName) + + .. seealso:: SciQuery.getViewsMetadata, SciQuery.getTablesMetadata, SciQuery.getDatabasesMetadata + """ + tables = getViewsMetadata(rdbComputeDomain, databaseContextName, format="pandas") + return [name for name in tables['TABLE_NAME']] + + +def getRoutinesMetadata(rdbComputeDomain, databaseContextName, format="pandas"): + """ + Gets metadata related to routines or functions in a particular database belonging to an RDBComputeDomain. + + :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) that + defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by + the function 'getRDBComputeDomains'. + :param databaseContextName: name (string) of the database. + :param format: String that defines the returned format. Either "pandas" for a pandas dataframe, or "dict" for + a dictionary object. + :return: pandas dataframe or dict (depending of the value of 'param') containing associated metadata. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). + Throws an exception if the HTTP request to the JOBM API returns an error. + :example: metadata = SciQuery.getRoutinesMetadata(rdbComputeDomain, databaseContextName) + + .. seealso:: SciQuery.getTablesMetadata, SciQuery.getViewsMetadata, SciQuery.getDatabasesMetadata + """ + return _getMetadata(rdbComputeDomain, databaseContextName, metadataType=MetadataType.ROUTINES, format=format) + + +def getRoutineNames(rdbComputeDomain, databaseContextName): + """ + Gets a list of the names of routines or functions in a particular database belonging to an RDBComputeDomain. + + :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) + that defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by + the function 'getRDBComputeDomains'. + :param databaseContextName: name (string) of the database. + :return: array of routine names (strings) + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). + Throws an exception if the HTTP request to the JOBM API returns an error. + :example: routinenames = getRoutineNames(rdbComputeDomain, databaseContextName) + + .. seealso:: SciQuery.getTableNames, SciQuery.getDatabasesMetadata, Sciquery.getDatabaseNames + """ + routines = getRoutinesMetadata(rdbComputeDomain, databaseContextName, format="pandas") + return [routineName for routineName in routines['ROUTINE_NAME']] + + +def getColumnsMetadata(rdbComputeDomain, databaseContextName, tableName, format="pandas"): + """ + Gets metadata related to columns in a particular database table belonging to an RDBComputeDomain. + + :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) + that defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by + the function 'getRDBComputeDomains'. + :param databaseContextName: name (string) of the database. + :param tableName: name (string) of the database table. + :param format: String that defines the returned format. Either "pandas" for a pandas dataframe, or "dict" for + a dictionary object. + :return: pandas dataframe or dict (depending of the value of 'param') containing associated metadata. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). + Throws an exception if the HTTP request to the JOBM API returns an error. + :example: metadata = SciQuery.getColumnsMetadata(rdbComputeDomain, databaseContextName, tableName) + + .. seealso:: SciQuery.getTablesMetadata, SciQuery.getViewsMetadata, SciQuery.getDatabasesMetadata + """ + return _getMetadata(rdbComputeDomain, databaseContextName, tableName, metadataType=MetadataType.COLUMNS, + format=format) + + +def getColumnNames(rdbComputeDomain, databaseContextName, tableName): + """ + Gets a list of the names of table columns in a particular database belonging to an RDBComputeDomain. + + :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) + that defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by + the function 'getRDBComputeDomains'. + :param databaseContextName: name (string) of the database. + :param tableName: name (string) of the database table + :return: array of column names (strings) + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). + Throws an exception if the HTTP request to the JOBM API returns an error. + :example: columnnames = SciQuery.getColumnNames(rdbComputeDomain, databaseContextName, tableName) + + .. seealso:: SciQuery.getTableNames, SciQuery.getDatabasesMetadata, Sciquery.getDatabaseNames + """ + columns = getColumnsMetadata(rdbComputeDomain, databaseContextName, tableName, format="pandas") + return [columnName for columnName in columns['COLUMN_NAME']] + + +def getConstraintsMetadata(rdbComputeDomain, databaseContextName, tableName, format="pandas"): + """ + Gets metadata related to table constraints in a particular database table belonging to an RDBComputeDomain. + + :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) + that defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by + the function 'getRDBComputeDomains'. + :param databaseContextName: name (string) of the database. + :param tableName: name (string) of the database table. + :param format: String that defines the returned format. Either "pandas" for a pandas dataframe, or "dict" for + a dictionary object. + :return: pandas dataframe or dict (depending of the value of 'param') containing associated metadata. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). + Throws an exception if the HTTP request to the JOBM API returns an error. + :example: metadata = SciQuery.getConstraintsMetadata(rdbComputeDomain, databaseContextName, tableName) + + .. seealso:: SciQuery.getTablesMetadata, SciQuery.getColumnsMetadata, SciQuery.getDatabasesMetadata + """ + return _getMetadata(rdbComputeDomain, databaseContextName, tableName, metadataType=MetadataType.CONSTRAINTS, + format=format) + + +def getConstraintNames(rdbComputeDomain, databaseContextName, tableName): + """ + Gets a list of the names of table constraints in a particular database belonging to an RDBComputeDomain. + + :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) + that defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by + the function 'getRDBComputeDomains'. + :param databaseContextName: name (string) of the database. + :param tableName: name (string) of the database table. + :return: array of constraint names (strings) + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). + Throws an exception if the HTTP request to the JOBM API returns an error. + :example: names = SciQuery.getConstraintNames(rdbComputeDomain, databaseContextName, tableName) + + .. seealso:: SciQuery.getTableNames, SciQuery.getColumnNames, SciQuery.getDatabasesMetadata + """ + constraints = getConstraintsMetadata(rdbComputeDomain, databaseContextName, tableName, format="pandas") + return [constraintName for constraintName in constraints['CONSTRAINT_NAME']] + + +def getRoutineParametersMetadata(rdbComputeDomain, databaseContextName, routineName, format="pandas"): + """ + Gets metadata related to routine parameters in a particular database belonging to an RDBComputeDomain. + + :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) + that defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by + the function 'getRDBComputeDomains'. + :param databaseContextName: name (string) of the database. + :param routineName: name (string) of the routine or function. + :param format: String that defines the returned format. Either "pandas" for a pandas dataframe, or "dict" + for a dictionary object. + :return: pandas dataframe or dict (depending of the value of 'param') containing associated metadata. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). + Throws an exception if the HTTP request to the JOBM API returns an error. + :example: metadata = SciQuery.getRoutineParametersMetadata(rdbComputeDomain, databaseContextName, routineName) + + .. seealso:: SciQuery.getTablesMetadata, SciQuery.getColumnsMetadata, SciQuery.getDatabasesMetadata + """ + return _getMetadata(rdbComputeDomain, databaseContextName, routineName, metadataType=MetadataType.PARAMETERS, + format=format) + + +def getRoutineParameterNames(rdbComputeDomain, databaseContextName, routineName): + """ + Gets a list of the names of routine parameters in a particular database belonging to an RDBComputeDomain. + + :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) + that defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by + the function 'getRDBComputeDomains'. + :param databaseContextName: name (string) of the database. + :param routineName: name (string) of the routine or function. + :return: array of parameter names (strings) + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). + Throws an exception if the HTTP request to the JOBM API returns an error. + :example: names = SciQuery.getRoutineParameterNames(rdbComputeDomain, databaseContextName, routineName) + + .. seealso:: SciQuery.getTableNames, SciQuery.getDatabasesMetadata, Sciquery.getDatabaseNames + """ + parameters = getRoutineParametersMetadata(rdbComputeDomain, databaseContextName, routineName, format="pandas") + return [parametersName for parametersName in parameters['SPECIFIC_NAME']] From 2decb39e9c38919734bcc0b629416f4f2909c71a Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Tue, 8 Mar 2022 10:52:16 -0500 Subject: [PATCH 68/79] Fixed job messages in executeQuery function. --- py3/SciServer/SciQuery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py3/SciServer/SciQuery.py b/py3/SciServer/SciQuery.py index c1eddae..80bcb37 100644 --- a/py3/SciServer/SciQuery.py +++ b/py3/SciServer/SciQuery.py @@ -353,7 +353,7 @@ def executeQuery(sqlQuery, job_status = Jobs.waitForJob(jobId, verbose=False, pollTime=poll_time) job = RDBJob(jobId) if job.status > 32: - messages = ". ".join(job.messages) if len(job.messages) > 0 else "" + messages = ". ".join([j.get('content') for j in job.messages if j.get('content') is not None]) if (job.status == 64): raise Exception("Query ended with an error. " + messages) if (job.status == 128): From 905faf5fd8a77dc3791a2c20e62bc1e6374181e4 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Thu, 2 Jun 2022 09:38:22 -0400 Subject: [PATCH 69/79] Pythonifying and adding functionality with more classes. --- py3/SciServer/Config.py | 9 +- py3/SciServer/SciQuery.py | 2007 +++++++++++++++++++++++-------------- 2 files changed, 1256 insertions(+), 760 deletions(-) diff --git a/py3/SciServer/Config.py b/py3/SciServer/Config.py index afb993e..b241298 100644 --- a/py3/SciServer/Config.py +++ b/py3/SciServer/Config.py @@ -22,6 +22,8 @@ - **Config.ComputeUrl**: defines the base URL of the Compute webapp (string). E.g., "https://apps.sciserver.org/compute". +- **Config.SciqueryURL**: defines the base URL of the SciQuery web API (string). E.g., "https://apps.sciserver.org/sciquery-api". + - **Config.version**: defines the SciServer release version tag (string), to which this package belongs. E.g., "sciserver-v1.9.3" """ # URLs for accessing SciServer web services (API endpoints) @@ -36,16 +38,17 @@ version = "sciserver-v2.1.0" #sciserver release version ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. ComputeUrl = "https://apps.sciserver.org/compute" -SciqueryURL = "https://apps.sciserver.org/sciquery" +SciqueryURL = "https://apps.sciserver.org/sciquery-api" ComputeWorkDir = "/home/idies/workspace/" + def _load_config(filename): if os.path.exists(filename): with open(filename) as f: _config_data = json.load(f) global CasJobsRESTUri, AuthenticationURL, SciDriveHost, SkyQueryUrl, SkyServerWSurl global RacmApiURL, DataRelease, KeystoneTokenPath, version, ComputeJobDirectoryFile - global ComputeUrl + global ComputeUrl, SciqueryURL, ComputeWorkDir CasJobsRESTUri = _config_data.get('CasJobsRESTUri', CasJobsRESTUri) AuthenticationURL = _config_data.get('AuthenticationURL', AuthenticationURL) SciDriveHost = _config_data.get('SciDriveHost', SciDriveHost) @@ -57,6 +60,8 @@ def _load_config(filename): version = _config_data.get('version', version) ComputeJobDirectoryFile = _config_data.get('ComputeJobDirectoryFile', ComputeJobDirectoryFile) ComputeUrl = _config_data.get('ComputeUrl', ComputeUrl) + SciqueryURL = _config_data.get('SciqueryURL', SciqueryURL) + ComputeWorkDir = _config_data.get('ComputeWorkDir', ComputeWorkDir) _CONFIG_DIR = os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')) _SCISERVER_SYSTEM_CONFIG_DIR = '/etc/' # will not likely exist on non *nix systems diff --git a/py3/SciServer/SciQuery.py b/py3/SciServer/SciQuery.py index 80bcb37..cec1a25 100644 --- a/py3/SciServer/SciQuery.py +++ b/py3/SciServer/SciQuery.py @@ -1,20 +1,17 @@ from SciServer import Authentication, Config, Files, Jobs import pandas as pd -import numpy as np import json import requests -from functools import lru_cache +import cachetools.func +from collections.abc import Iterable from datetime import datetime - -def _get_default_rdb_domain(): - rdb_domains = getRDBComputeDomainNames() - if len(rdb_domains) > 0: - return rdb_domains[0] - else: - raise Exception("There are no rdbComputeDomains available for the user."); +import warnings +from pathlib import Path +from typing import Union, List +import time -class OutputTargetType: +class OutputType: """ Contains a set of allowed database output types. """ @@ -23,103 +20,203 @@ class OutputTargetType: DATABASE_TABLE = "TABLE" -class FileOutput: +@cachetools.func.ttl_cache(maxsize=128, ttl=120) +def _get_file_service(file_service: str = None): + file_services = Files.getFileServices(verbose=False) + if file_service is None: + if len(file_services) > 0: + return file_services[0] + raise Exception("No file services available for the user.") + else: + for fs in file_services: + if file_service == fs.get("name") or file_service == fs.get("identifier"): + return fs + raise Exception("Unable to find fileService") + + +class Output: """ - Defines the output of a database query to a file. + Base class for output objects """ - def __init__(self, target_name: str = "result.json", target_type: str = OutputTargetType.FILE_JSON, - statement_indexes: list = [1]): - """ - :param target_name: name of the file (string), such as "result.json" - :param target_type: type (string) of the file containing the query result(s) (e.g., "FILE_JSON"). - As set of possible values is given by the static members of class 'SciQuery.OutputTargetType' - :param statement_indexes: list of integers or integer. Each integer value denotes the index or position (>=1) - of the sql statements whithin the input query, whose resultset is going to be written into this OutputTarget - """ - if type(target_name) != str or type(target_type) != str: - raise ValueError("Invalid type(s) for input parameter(s) 'target_name' or 'target_type'") - self.target_name = target_name - self.target_type = target_type + def __init__(self, + name: str = "output.json", + output_type: str = OutputType.FILE_JSON, + statement_indexes: Union[int, List[int]] = 1): + + if type(name) != str: + raise TypeError("Invalid type for input parameter 'name'.") + if type(output_type) != str: + raise TypeError("Invalid type for input parameter 'output_type'.") + if type(statement_indexes) not in [list, int]: + raise TypeError("Invalid type for input parameter 'statement_indexes'.") + self.name = name + self.output_type = output_type + self.statement_indexes = None self.set_statement_indexes(statement_indexes) - def set_statement_indexes(self, statement_indexes: list = [1]): + def set_statement_indexes(self, statement_indexes: Union[int, List[int]] = 1): """ - Sets the index(es) of the sql statement(s) whithin the input query, whose resultset(s) is(are) going to - be written into this OutputTarget. - :param statement_indexes: list of integers, which are the indices (starting with 1) of the sql statements - within the input query, whose resultsets are going to be written into this OutputTarget. + Sets the index(es) of the sql statement(s) within the input query, whose result-set(s) is(are) going to be + written into this Output. + :param statement_indexes: integer or list of integers, which are the indices (starting with 1) of the sql + statements within the input query, whose resultsets are going to be written into this Output. """ - if type(statement_indexes) != list: + if not isinstance(statement_indexes, Iterable): statement_indexes = [statement_indexes] for index in statement_indexes: if type(index) != int or index <= 0: - raise ValueError("Invalid type for input parameter 'statement_indexes'") + raise TypeError("Invalid type for input parameter 'statement_indexes'") self.statement_indexes = [i for i in sorted(set(statement_indexes))] - return self + + def __str__(self): + return "Output of name = {}, type= {}, statement_indexes = {}".format(self.name, self.output_type, + self.statement_indexes) + def __repr__(self): + return "Output(name = {}, type= {}, statement_indexes = {})".format(self.name, self.output_type, + self.statement_indexes) + + +class FileOutput(Output): + """ + Defines the output of a database query into a file. + """ + + def __init__(self, + name: str = "output.json", + output_type: str = OutputType.FILE_JSON, + statement_indexes: Union[int, List[int]] = 1, + file_service: str = None): + """ + :param name: name of the file (string), such as "result.json" + :param output_type: type (string) of the file containing the query result(s) (e.g., "FILE_JSON"). + As set of possible values is given by the static members of class 'SciQuery.OutputTargetType' + :param statement_indexes: list of integers or integer. Each integer value denotes the index or position (>=1) + of the sql statements within the input query, whose resultset is going to be written into this OutputTarget + :param file_service: string denoting name or identifier of file service where the output file is written into. + """ + if file_service: + file_service = FileOutput.find_file_service(file_service) + self.file_service_name = file_service['name'] + self.file_service_identifier = file_service['identifier'] + else: + self.file_service_name = None + self.file_service_identifier = None + + if not name: + raise NameError("Input parameter name cannot be empty or None") + name = name.rstrip("/") + file_path = Path(name) + if name == file_path.name: # means no path included in 'name' input parameter + self.file_base_path = None + self.file = name + self.path = None + self.file_service_path = None + else: + if not name.startswith(Config.ComputeWorkDir): + file_path = Path(Config.ComputeWorkDir + name) # in case it is relative path + self.file_base_path = str(file_path.parent) + self.file_base_path = self.file_base_path if self.file_base_path.endswith("/") \ + else self.file_base_path + "/" + self.file = file_path.name + self.path = self.file_base_path + self.file + self.file_service_path = self.path.replace(Config.ComputeWorkDir, "", 1) + name = file_path.name + + super().__init__(name, output_type, statement_indexes) + + def get_path(self) -> str: + if self.path: + return self.path + else: + raise Exception("Attribute 'file_base_path' is not set.") + @classmethod def get_default(cls): """ - Gets a OutputTarget object filled with default values: JSON output file where only the 1st SQL statement of + Gets an Output object filled with default values: JSON output file where only the 1st SQL statement of the query is written in it. """ - cls.target_name = "result.json" - cls.target_type = OutputTargetType.FILE_JSON - cls.statement_indexes = [1] - return cls + return cls("result.json", OutputType.FILE_JSON, 1) + + @staticmethod + def build_file_base_path(top_volume: str = "Temporary", + user_volume: str = "scratch", + user_volume_owner_name: str = "", + relative_path: str = "sciqueryjobs", + add_date_ending: bool = False) -> str: + if not top_volume: + raise NameError("Input parameter top_volume cannot be empty or None") + + if add_date_ending: + now = datetime.now() + date1 = now.strftime("%Y-%m-%d") + date2 = now.strftime("%Hh%Mm%S.%fs") + relative_path = "{0}/{1}/{2}".format(relative_path.rstrip('/'), date1, date2) + + if user_volume: + if not user_volume_owner_name: + user_volume_owner_name = SciQuery.get_user().userName + path = str(Path(Config.ComputeWorkDir, top_volume, user_volume_owner_name, user_volume, relative_path)) + else: + path = str(Path(Config.ComputeWorkDir, top_volume, relative_path)) + return path if path.endswith("/") else path + "/" + + @staticmethod + def find_file_service(file_service: Union[str, dict] = None) -> dict: + if isinstance(file_service, dict): + file_service = file_service.get("identifier") + return _get_file_service(file_service) def __str__(self): - return "File Output of target_name = {}, target_type= {}, statement_indexes = {}".format(self.target_name, - self.target_type, - self.statement_indexes) + return f"File Output of name = {self.name}, type= {self.output_type}, statement_indexes = " \ + f"{self.statement_indexes}" def __repr__(self): - return "FileOutput(target_name = {}, target_type= {}, statement_indexes = {})".format(self.target_name, - self.target_type, - self.statement_indexes) + return f"FileOutput(name= {self.name}, type= {self.output_type}, statement_indexes = {self.statement_indexes})" -class DatabaseTableOutput: +class DatabaseTableOutput(Output): """ - Defines the output of a database query to a database table + Defines the output of a database query into a database table """ - def __init__(self, table: str = "resultTable", database: str = "", rdb_domain: str = "", schema: str = "", - statement_indexes: list = [1]): + + def __init__(self, + table: str = "resultTable", + database: str = None, + statement_indexes: Union[int, List[int]] = 1, + rdb_compute_domain: str = None, + schema: str = ""): + """ - :param table: name of the database table (string), such as "resultTable" + :param table: name of the output database table (string), such as "resultTable" :param database: name of the database (string) where the output table in created. If it is owned explicitly by - a user, then it should follow the pattern "mydb:username" - :param rdb_domain: name (string) of the relational database (RDB) compute domain that contains the database. + a user, then it should follow the pattern "mydb:" + :param statement_indexes: list of integers or integer. Each integer value denotes the index or position (>=1) + :param rdb_compute_domain: name (string) of the relational database (RDB) compute domain that contains the + database, or object of class RDBComputeDomain corresponding to it. Name of such domains available to the user is returned by the function Jobs.getRDBComputeDomainNames(). :param schema: database schema (string) - :param statement_indexes: list of integers or integer. Each integer value denotes the index or position (>=1) - of the sql statements whithin the input query, whose resultset is going to be written into this OutputTarget + of the sql statements within the input query, whose resultset is going to be written into this OutputTarget """ - if type(table) != str or type(rdb_domain) != str or type(schema) != str: - raise ValueError("Invalid type(s) for input parameter(s) 'target_name' or 'target_type'") + if type(table) != str or type(schema) != str: + raise TypeError("Input parameter(s) 'table' or 'schema' should be of type string.") + + domain = RDBComputeDomains.get_default_rdb_compute_domain() if not rdb_compute_domain else rdb_compute_domain + if not database: + if type(domain) == str: + database = SciQuery.get_rdb_compute_domains().get_rdb_compute_domain(domain).get_default_database().name + else: + database = domain.get_default_database().name + domain = domain.name + self.table = table - self.database = database if database != "" else "mydb:" + Authentication.keystoneUser.userName - self.rdb_domain = rdb_domain if rdb_domain != "" else _get_default_rdb_domain() + self.database = database + self.rdb_compute_domain_name = domain self.schema = schema - self.set_statement_indexes(statement_indexes) - self.target_name = ".".join([rdb_domain, database, schema, table]) - self.target_type = OutputTargetType.DATABASE_TABLE - - def set_statement_indexes(self, statement_indexes: list = [1]): - """ - Sets the index(es) of the sql statement(s) whithin the input query, whose resultset(s) is(are) going to be - written into this OutputTarget. - :param statement_indexes: list of integers, which are the indices (starting with 1) of the sql statements - within the input query, whose resultsets are going to be written into this OutputTarget. - """ - if type(statement_indexes) != list: - statement_indexes = [statement_indexes] - for index in statement_indexes: - if type(index) != int or index <= 0: - raise ValueError("Invalid type for input parameter 'statement_indexes'") - self.statement_indexes = [i for i in sorted(set(statement_indexes))] - return self + name = ".".join([self.rdb_compute_domain_name, self.database, self.schema, self.table]) + super().__init__(name, OutputType.DATABASE_TABLE, statement_indexes) @classmethod def get_default(cls): @@ -127,832 +224,1226 @@ def get_default(cls): Gets a OutputTarget object filled with default values: JSON output file where only the 1st SQL statement of the query is written in it. """ - cls.target_name = "resultTable" - cls.database = "mydb:" + Authentication.keystoneUser.userName - cls.rdb_domain = _get_default_rdb_domain() - cls.schema = "" - cls.target_type = OutputTargetType.DATABASE_TABLE - cls.statement_indexes = [1] - return cls + return cls(table = "resultTable", + database = RDBComputeDomains.get_default_rdb_compute_domain().get_default_database().name, + rdb_compute_domain = RDBComputeDomains.get_default_rdb_compute_domain().name, + schema = "", + statement_indexes = [1]) def __str__(self): - return "Database Table Output of table= {}, database= {}, rdb_domain= {}, schema= {}, statement_indexes= {}"\ - .format(self.table, self.database, self.rdb_domain, self.schema, self.statement_indexes) + return "Database Table Output of table= {}, database= {}, rdb_compute_domain_name= {}, schema= {}, " \ + "statement_indexes= {}".format(self.table, self.database, self.rdb_compute_domain_name, self.schema, + self.statement_indexes) def __repr__(self): - return "DatabaseTableOutput(table= {}, database= {}, rdb_domain= {}, schema= {}, statement_indexes= {})"\ - .format(self.table, self.database, self.rdb_domain, self.schema, self.statement_indexes) + return "DatabaseTableOutput(table= {}, database= {}, rdb_compute_domain_name= {}, schema= {}, " \ + "statement_indexes= {})".format(self.table, self.database, self.rdb_compute_domain_name, self.schema, + self.statement_indexes) + + +class Outputs(list): + """ + Contains a list of output objects, defining database query resultset outputs. + """ + + def __init__(self, *outputs): + super().__init__() + for output in outputs: + outs = output if isinstance(output, Iterable) else [output] + for out in outs: + self.append(out) + + def append(self, obj): + if isinstance(obj, Output): + super().append(obj) + else: + raise NameError("Input object is not a subclass of the 'Output' class.") + + def get_target_list(self, file_base_path: str = None, file_service: str = None): + targets = [] + fs = FileOutput.find_file_service(file_service) + for output in self: + for index in output.statement_indexes: + location = output.name + if output.output_type != OutputType.DATABASE_TABLE: # for files + if not output.file_service_identifier: + file_service_identifier = fs.get('identifier') + else: + file_service_identifier = output.file_service_identifier + + if not output.file_base_path: + if file_base_path: + location = file_base_path.rstrip("/") + "/" + output.file + else: + location = FileOutput.build_file_base_path().rstrip("/") + "/" + output.file + else: + location = output.file_base_path.rstrip("/") + "/" + output.file + + location = file_service_identifier + ":" + location; + targets.append({'location': location, 'type': output.output_type, 'resultNumber': index}) + return targets + + @staticmethod + def get_default(): + return Outputs(FileOutput(name="result.json", output_type=OutputType.FILE_JSON, statement_indexes = [1])) class RDBJob: """ - Contains the definition of an RDB job + Contains the definition of a job consisting on a query run in a Relational Database (RDB) """ + _JOB_STATUS_MAP = {1: "PENDING", 2: "QUEUED", 4: "ACCEPTED", 8: "STARTED", 16: "FINISHED", 32: "SUCCESS", + 64: "ERROR", 128: "CANCELED"} + def __init__(self, job): """ - :param job: can be the job ID (string), or the + :param job: can be the job ID (string), or a dictionary containing all the attributes of an RDBJob object. """ if type(job) != dict: job = Jobs.getJobDescription(job) - for k, v in job.items(): - setattr(self, k, v) - def get_output_targets(self): + self.id = job.get('id') + self.alias = job.get('submitterDID') if job.get('submitterDID') is not None else job.get('alias') + self._submitter_trust_id = job.get('submitterTrustId') if job.get('submitterTrustId') is not None else \ + job.get('_submitter_trust_id') + self._run_by_uuid = job.get('runByUUID') if job.get('runByUUID') is not None else job.get('_run_by_uuid') + self.submission_time = self._get_datetime(job.get('submissionTime') if job.get('submissionTime') is not None + else job.get('submission_time')) + self.start_time = self._get_datetime(job.get('startTime') if job.get('startTime') is not None + else job.get('start_time')) + self.end_time = self._get_datetime(job.get('endTime') if job.get('endTime') is not None + else job.get('end_time')) + self.duration = job.get('duration') + self.timeout = job.get('timeout') + self._messages = job.get('messages') if job.get('messages') is not None else job.get('_messages') + self.message_list = [m.get("content") for m in self._messages] if self._messages is not None else [] + self.status = job.get('status') + self.status_string = RDBJob.get_job_status(job.get('status')) + self._results_folder_uri = job.get('resultsFolderURI') if job.get('resultsFolderURI') is not None \ + else job.get('_results_folder_uri') + self._type = job.get('type') if job.get('type') is not None else job.get('_type') + self.user_name = job.get('username') if job.get('username') is not None else job.get('user_name') + self.input_sql = job.get('inputSql') if job.get('inputSql') is not None else job.get('input_sql') + self.targets = job.get('targets') + self.database_name = job.get('databaseContextName') \ + if job.get('databaseContextName') is not None else job.get('database_name') + self._rdb_resource_context_uuid = job.get('rdbResourceContextUUID') \ + if job.get('rdbResourceContextUUID') is not None else job.get('_rdb_resource_context_uuid') + self.rdb_compute_domain_name = job.get('rdbDomainName') if job.get('rdbDomainName') is not None \ + else job.get('rdb_compute_domain_name') + self.rdb_compute_domain_id = job.get('rdbDomainId') if job.get('rdbDomainId') is not None \ + else job.get('rdb_compute_domain_id') + self.outputs = self._get_outputs() + self.get_job_status = self._get_job_status_string + + def get_metadata(self, result_format="pandas") -> pd.DataFrame: + data = [] + column_names = [] + if result_format == "pandas": + for attr, value in self.__dict__.items(): + import inspect + if not attr.startswith("_") and not inspect.ismethod(value): # no private members nor methods + data.append(value) + column_names.append(attr) + df = pd.DataFrame([data], columns=column_names) + df.name = str(RDBJob) + return df + elif result_format == "dict": + return self.__dict__ + else: + raise Exception("Invalid value for input parameter 'result_format'.") + + @staticmethod + def get_job_status(status: int) -> str: + return RDBJob._JOB_STATUS_MAP.get(status) + + @staticmethod + def get_job(job_id: int): + return RDBJob(Jobs.getJobDescription(job_id)) + + def cancel(self): + Jobs.cancelJob(self.id) + + def refresh(self): + self.__init__(Jobs.getJobDescription(self.id)) + + def _get_job_status_string(self) -> str: + return RDBJob.get_job_status(self.status) + + def _get_outputs(self) -> Outputs: output_targets = {} for t in self.targets: i = (t['location'], t['type']) if i not in output_targets: output_targets[i] = [t['resultNumber']] else: - output_targets[i] = output_targets[i].append(t['resultNumber']) + output_targets[i].append(t['resultNumber']) - targets = [] + outputs = Outputs() for k in output_targets: - if k[1] == OutputTargetType.DATABASE_TABLE: + if k[1] == OutputType.DATABASE_TABLE: p = k[0].split(".") - targets.append(DatabaseTableOutput(table=p[3], database=p[1], rdb_domain=p[0], schema=p[2], + outputs.append(DatabaseTableOutput(table=p[3], database=p[1], rdb_compute_domain=p[0], schema=p[2], statement_indexes=output_targets[k])) else: - targets.append(FileOutput(target_name=k[0], target_type=k[1], statement_indexes=output_targets[k])) + file_parts = k[0].split(":") + file_service_identifier = file_parts[0] + name = file_parts[1] + outputs.append(FileOutput(name=name, output_type=k[1], statement_indexes=output_targets[k], + file_service=file_service_identifier)) + + return outputs + + def _get_output_from_index(self, ind: int): + if ind > len(self.outputs) - 1: + raise ValueError("Index is outside of the index range in the outputs list.") + return self.outputs[ind] + + def get_output_path(self, output: Union[Output, int] = 0) -> str: + out = self._get_output_from_index(output) if isinstance(output, int) else output + if out.output_type == OutputType.DATABASE_TABLE: + raise TypeError("Output is not a file but a database") + return out.get_path() + + def get_output_as_string(self, output: Union[Output, int, str] = None): + if not isinstance(output, str): + out = self._get_output_from_index(output) if isinstance(output, int) else output + file_path = self.get_output_path(out) + else: + file_path = output - # output_targets = [ OutputTarget(l,t,output_targets[(l,t)]) for l,t in output_targets] - # return output_targets - return targets + if Config.isSciServerComputeEnvironment(): + with open(file_path, ) as f: + data = f.read() + else: + if isinstance(output, str): + raise Exception(f"Cannot find file_path {output} in local file system.") + + fs = FileOutput.find_file_service(out.file_service_identifier) + path = out.file_service_path + data = Files.download(fs, path, format="txt", quiet=True) + + return data + + def get_json_output(self, output: Union[Output, int, str] = 0) -> dict: + data_dict = json.loads(self.get_output_as_string(output)) + return data_dict.get("Result") + + def get_dataframe_from_output(self, output: Union[Output, int] = 0, result_index: int = 0) -> list: + out = self._get_output_from_index(output) if isinstance(output, int) else output + if out.output_type == OutputType.FILE_JSON: + result = self.get_json_output(out)[result_index] + df = pd.DataFrame(result['Data'], columns=result['ColumnNames']) + df.name = result['TableName'] + elif out.output_type == OutputType.FILE_CSV: + df = pd.read_csv(out.get_path(), skiprows=1) + elif out.output_type == OutputType.DATABASE_TABLE: + sq = SciQuery(rdb_compute_domain=out.rdb_compute_domain_name, database=out.database) + query = f"select * from {out.table};" + df = sq.execute_query(query) + else: + raise Exception(f"Output type {out.output_type} not supported") + return df - def get_results_folder_path(self): - path = ":".join(self.resultsFolderURI.split(":")[1:]) - if not path.startswith(Config.ComputeWorkDir): - path = Config.ComputeWorkDir + path[1:] - return path + def _get_datetime(self, time): + return datetime.fromtimestamp(time / 1000.0) if time is not None else None - def get_output_target_path(self, output_target): - if output_target.target_type == OutputTargetType.DATABASE_TABLE: - raise ValueError("Output target is not a file but a database") - return self.get_results_folder_path() + output_target.target_name + def __str__(self): + return "RDB Job of id={}".format(self.id) - def get_fileservice_folder_path(self): - path = ":".join(self.resultsFolderURI.split(":")[1:]) - if path.startswith(Config.ComputeWorkDir): - path = "/" + path.replace(Config.ComputeWorkDir, "", 1) - return path + def __repr__(self): + return "RDBJob(id={})".format(self.id) - def get_start_time(self): - return datetime.fromtimestamp(self.startTime / 1000.0) - def get_end_time(self): - return datetime.fromtimestamp(self.endTime / 1000.0) +class Database: + """ + Defines a database context where users can run sql queries. + """ - def get_duration(self): - return self.duration + def __init__(self, rdb_compute_domain: Union[str, int, dict], database: Union[str, int, dict]): + """ + :param rdb_compute_domain: Parameter that identifies the relation database domain or environment that + contains the database. Could be either its name (string), ID (integer), or a dictionary containing + the attributes of the domain. + :param database: defines the database. Can be either the database name (string), ID (integer), or a dictionary + containing all the attributes of an object of class Database. + """ + if type(database) not in [str, int, dict]: + raise TypeError("Invalid type for input parameter 'database'.") - def __str__(self): - return "RDB Job of id = {}".format(self.id) + if type(rdb_compute_domain) not in [str, int, dict]: + raise TypeError("Invalid type for input parameter 'rdb_compute_domain'.") - def __repr__(self): - return "RDBJob(id = {})".format(self.id) + if type(rdb_compute_domain) != dict: + domain = RDBComputeDomain(rdb_compute_domain) + else: + domain = rdb_compute_domain + if type(database) != dict: + dbs = domain.get('databaseContexts') or domain.get('databases') + if type(database) == str: + database = [db for db in dbs if db.get('name') == database] + else: + database = [db for db in dbs if db.get('id') == database] + if len(database) == 0: + raise NameError("Unable to find database.") + else: + database = database[0] + + self.id = database.get('id') + self._racm_id = database.get('_racm_id') if database.get('_racm_id') is not None else database.get('racmId') + self.name = database.get('name') if database.get('name') is not None else database.get('contextName') + self.description = database.get('description') + self.vendor = database.get('vendor') + self.schemas = database.get('dbSchemas') if database.get('dbSchemas') is not None else database.get('schemas') + self.rdb_compute_domain_name = domain.get('name') if domain.get('name') is not None else \ + domain.get('displayName') + self.rdb_compute_domain_id = domain.get('id') + + def get_metadata(self) -> pd.DataFrame: + data = [] + column_names = ['database_name', 'database_description', 'database_vendor', 'database_id', + 'rdb_compute_domain_name', 'rdb_compute_domain_id'] + data.append([self.name, self.description, self.vendor, self.id, self.rdb_compute_domain_name, + self.rdb_compute_domain_id]) + df = pd.DataFrame(data=data, columns=column_names, index=[self.id]) + df = df.astype({"rdb_compute_domain_id": int, "database_id": int}) + return df -@lru_cache(128) -def _get_file_service(file_service_id=""): - file_services = Files.getFileServices(verbose=False) - for file_service in file_services: - if file_service["name"] == file_service_id or file_service["identifier"] == file_service_id: - return file_service + def __str__(self): + return "Database of name={}, id={} and rdb_compute_domain_name={}".format(self.name, self.id, + self.rdb_compute_domain_name) - if len(file_services) > 0: - return file_services[0] - else: - raise Exception("No fileservices available for the user") + def __repr__(self): + return "Database(name={}, id={}, rdb_compute_domain_name={})".format(self.name, self.id, + self.rdb_compute_domain_name) -def submitQueryJob(sqlQuery, - rdbComputeDomain=None, - databaseContextName=None, - outputTargets=FileOutput.get_default(), - resultsFolderPath="", - jobAlias="", - fileServiceName=""): +class RDBComputeDomain: """ - Submits a sql query for execution (as an asynchronous job) inside a relational database (RDB) compute domain. - - :param sqlQuery: sql query (string) - :param rdbComputeDomain: object (dictionary) that defines a relational database (RDB) compute domain. A list of - these kind of objects available to the user is returned by the function 'getRDBComputeDomains'. - :param databaseContextName: database context name (string) on which the sql query is executed. - :param outputTargets: object of type SciQuery.OutputTarget defining the output of one or multiple statements - within the input query. Could also be a list of OutputTarget objects. - :param resultsFolderPath: full path to results folder (string) where query output tables are written into. - E.g.: /home/idies/workspace/rootVOlume/username/userVolume/jobsFolder . If not set, - then a default folder will be set automatically. - :param jobAlias: alias (string) of job, defined by the user. - :param fileServiceName: name or uuid (string) of FileService where the results folder (resultsFolderPath) is - going to be created. If not defined, then the first available FileService is chosen by default. - :return: the ID (integer) that labels the job. - :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if - the HTTP request to the SciQuery API returns an error, or if the volumes defined by the user are not available in - the Docker compute domain. - :example: job_id = SciQuery.submitQueryJob('select 1;') - - .. seealso:: SciQuery.submitQueryJob, SciQuery.getJobStatus, SciQuery.getJob + Defines a domain or environment with databases that users are able to query. """ - token = Authentication.getToken() - if token is not None and token != "": + def __init__(self, rdb_compute_domain: Union[str, int, dict]): + """ + Creates an instance of an RDBComputeDomain, which defines a domain or environment with databases that users. + are able to query. + :param rdb_compute_domain: Parameter that identifies the domain. Could be either its name (string), + ID (integer), or a dictionary containing all the attributes of the domain. + """ + if type(rdb_compute_domain) not in [str, int, dict]: + raise TypeError("Invalid type for input parameter 'rdb_compute_domain'.") + + if type(rdb_compute_domain) != dict: + domains = SciQuery.get_rdb_compute_domains("dict") + if type(rdb_compute_domain) == str: + domain = [d for d in domains if d.get('name') == rdb_compute_domain] + elif type(rdb_compute_domain) == int: + domain = [d for d in domains if d.get('id') == rdb_compute_domain] + else: + raise TypeError("Invalid type for input parameter 'rdb_compute_domain'.") - if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.Sciquery.submitQueryJob" + if len(domain) > 0: + rdb_compute_domain = domain[0] + else: + raise NameError("Unable to find rdbComputeDomain {0}.".format(rdb_compute_domain)) + + self.id = rdb_compute_domain.get('id') + self._racm_id = rdb_compute_domain.get('_racm_id') if rdb_compute_domain.get('_racm_id') is not None else \ + rdb_compute_domain.get('racmId') + self.name = rdb_compute_domain.get('name') if rdb_compute_domain.get('name') is not None else \ + rdb_compute_domain.get('displayName') + self.description = rdb_compute_domain.get('description') + dbs = [] + databases = rdb_compute_domain.get('dbContexts') if \ + rdb_compute_domain.get('dbContexts') is not None else rdb_compute_domain.get('databases') + for db_name, db_dict in databases.items(): + dbs.append(Database(rdb_compute_domain, db_dict)) + self.databases = dbs + + def get_database_names(self) -> list: + """ + Gets a list of the names of databases in an RDBComputeDomain + + :return: list of database names (strings) + :example: dbnames = SciQuery.get_database_names(rdbComputeDomainName); + .. seealso:: SciQuery.get_databases_metadata + """ + return [db.name for db in self.databases] + + def get_database(self, database: Union[str, int, dict, Database]) -> Database: + if type(database) == str: + dbs = [db for db in self.databases if db.name == database] + elif type(database) == int: + dbs = [db for db in self.databases if db.id == database] + elif type(database) == dict: + dbs = [db for db in self.databases if db.id == database.get('id')] + elif isinstance(database, Database): + return self.get_database(database.id) + else: + raise TypeError("Invalid type for input parameter 'database'.") + if len(dbs) == 0: + raise NameError("Database not found in list of available databases.") + else: + return dbs[0] + + def get_default_database(self) -> Database: + dbs = [db for db in self.databases if db.name == SciQuery.get_mydb_name()] + if len(dbs) > 0: + return dbs[0] + elif len(self.databases) > 0: + return self.databases[0] else: - taskName = "SciScript-Python.Sciquery.submitQueryJob" + raise Exception("No default database available.") + + def get_metadata(self, do_include_databases: bool = False) -> pd.DataFrame: + + column_names = ['rdb_compute_domain_name', 'rdb_compute_domain_description', 'rdb_compute_domain_id'] + data = [[self.name, self.description, self.id]] + domain_metadata = pd.DataFrame(data=data, columns=column_names) + + if do_include_databases: + db_metadata = self.get_databases_metadata() + # https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.join.html + domain_metadata = pd.merge(domain_metadata, db_metadata, how="outer", + left_on=["rdb_compute_domain_id", "rdb_compute_domain_name"], + right_on=["rdb_compute_domain_id", "rdb_compute_domain_name"]) + domain_metadata.sort_values(by=['rdb_compute_domain_name', 'database_name'], inplace=True) + domain_metadata = domain_metadata.astype({"database_id": int}) + else: + domain_metadata.sort_values(by=['rdb_compute_domain_name'], inplace=True) - if rdbComputeDomain is None: - rdbComputeDomains = getRDBComputeDomains(); - if len(rdbComputeDomains) > 0: - rdbComputeDomain = rdbComputeDomains[0]; - else: - raise Exception("There are no rdbComputeDomains available for the user."); + domain_metadata = domain_metadata.astype({"rdb_compute_domain_id": int}) + return domain_metadata - if databaseContextName is None: - databaseContexts = rdbComputeDomain.get('databaseContexts'); - if len(databaseContexts) > 0: - databaseContextName = databaseContexts[0].get('name') - else: - raise Exception("rbdComputeDomain has no database contexts available for the user."); + def get_databases_metadata(self) -> pd.DataFrame: + """ + Gets metadata of the databases in this RDBComputeDomain. - if type(outputTargets) != list: - outputTargets = [outputTargets] + :return: pandas dataframe with associated metadata. + .. seealso:: SciQuery.get_database_names + """ + dfs = [db.get_metadata() for db in self.databases] + dfs = pd.concat(dfs, ignore_index=True) + dfs.sort_values(by="database_name", inplace=True) + return dfs - targets = [] - for target in outputTargets: - for index in target.statement_indexes: - targets.append({'location': target.target_name, 'type': target.target_type, 'resultNumber': index}) + def __str__(self): + return "RDBComputeDomain of name={} and id={}".format(self.name, self.id) - rdbDomainId = rdbComputeDomain.get('id'); + def __repr__(self): + return "RDBComputeDomain(name={}, id={})".format(self.name, self.id) - file_service = _get_file_service(fileServiceName) - resultsFolderPath = file_service['identifier'] + ":" + resultsFolderPath - dockerJobModel = { - "inputSql": sqlQuery, - "submitterDID": jobAlias, - "databaseContextName": databaseContextName, - "rdbDomainId": rdbDomainId, - "targets": targets, - "resultsFolderURI": resultsFolderPath - } +class RDBComputeDomains(list): + """ + Defines a list of RDBComputeDomains, which are domains or environments with databases that users are able to query. + """ + def __init__(self, rdb_compute_domains: Union[Iterable, RDBComputeDomain]): + """ + :param rdb_compute_domains: Parameter that identifies a list of RDBComputeDomain objects. + Could be either single RDBComputeDomain object, or an iterable containing multiple RDBComputeDomain objects. + """ + super().__init__() + domains = rdb_compute_domains if isinstance(rdb_compute_domains, Iterable) else [rdb_compute_domains] + for d in domains: + if isinstance(d, RDBComputeDomain): + self.append(d) + else: + raise NameError("Input object is not of class RDBComputeDomain.") + + def get_rdb_compute_domain(self, rdb_compute_domain: Union[str, int, dict, RDBComputeDomain]) -> RDBComputeDomain: + if type(rdb_compute_domain) == str: + domains = [d for d in self if d.name == rdb_compute_domain] + elif type(rdb_compute_domain) == int: + domains = [d for d in self if d.id == rdb_compute_domain] + elif type(rdb_compute_domain) == dict: + domains = [d for d in self if d.id == rdb_compute_domain.get('id')] + elif isinstance(rdb_compute_domain, RDBComputeDomain): + return self.get_rdb_compute_domain(rdb_compute_domain.id) + else: + raise TypeError("Invalid type for input parameter 'rdb_compute_domain'.") - data = json.dumps(dockerJobModel).encode() - url = Config.SciqueryURL + "/api/jobs/" + str(rdbDomainId) + "?TaskName=" + taskName; - headers = {'X-Auth-Token': token, "Content-Type": "application/json"} - res = requests.post(url, data=data, headers=headers, stream=True) - if res.status_code < 200 or res.status_code >= 300: - raise Exception("Error when submitting a job to the SciQuery API.\nHttp Response from SciQuery API " + - "returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + if len(domains) == 0: + raise NameError("RDBComputeDomain not found in list of available rdbComputeDomains") else: - return res.content.decode() - else: - raise Exception("User token is not defined. First log into SciServer.") + return domains[0] + + def get_default_rdb_compute_domain(self) -> RDBComputeDomain: + domains = [domain for domain in self if len(domain.databases) > 0] + if len(domains) > 0: + doms = [dom for dom in domains if dom.get_default_database().name == SciQuery.get_mydb_name()] + if len(doms) > 0: + return doms[0] + return domains[0] + else: + raise Exception("No RDBComputeDomain available with a database.") -def executeQuery(sqlQuery, - rdbComputeDomain=None, - databaseContextName=None, - resultsFolderPath="", - jobAlias="", - poll_time=0.2, - fileServiceName=""): +class SciQuery: """ - Returns the query result (as a Pandas data frame) of a sql query submitted as a job to a - relational database (RDB) compute domain. - - :param sqlQuery: sql query (string) - :param rdbComputeDomain: object (dictionary) that defines a relational database (RDB) compute domain. A list of - these kind of objects available to the user is returned by the function 'getRDBComputeDomains'. - :param databaseContextName: database context name (string) on which the sql query is executed. - :param resultsFolderPath: full path to results folder (string) where query output tables are written into. - E.g.: /home/idies/workspace/rootVOlume/username/userVolume/jobsFolder . If not set, - then a default folder will be set automatically. - :param jobAlias: alias (string) of job, defined by the user. - :param fileServiceName: name or uuid (string) of FileService where the results folder (resultsFolderPath) is - going to be created. If not defined, then the first available FileService is chosen by default. - :return: Pandas data frame containing the result of the query. - :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if - the HTTP request to the SciQuery API returns an error. - :example: df = SciQuery.executeQuery('select 1;') - - .. seealso:: SciQuery.submitQueryJob, SciQuery.getJobStatus, SciQuery.getJob - Jobs.cancelJob + Instance of the SciQuery app for querying relational databases. """ - output_target = FileOutput("result1.json", OutputTargetType.FILE_JSON).set_statement_indexes([1]) - - jobId = submitQueryJob(sqlQuery=sqlQuery, rdbComputeDomain=rdbComputeDomain, databaseContextName=databaseContextName, - outputTargets=output_target, - resultsFolderPath=resultsFolderPath, - jobAlias=jobAlias, fileServiceName=fileServiceName) - - job_status = Jobs.waitForJob(jobId, verbose=False, pollTime=poll_time) - job = RDBJob(jobId) - if job.status > 32: - messages = ". ".join([j.get('content') for j in job.messages if j.get('content') is not None]) - if (job.status == 64): - raise Exception("Query ended with an error. " + messages) - if (job.status == 128): - raise Exception("Query was cancelled. " + messages) - - if Config.isSciServerComputeEnvironment(): - file_path = job.get_output_target_path(output_target) - with open(file_path, ) as f: - j = json.load(f) - else: - file_service_id = job.resultsFolderURI.split(":")[0] - file_service = _get_file_service(file_service_id) - path = job.get_fileservice_folder_path(output_target) - s = Files.download(file_service, path, format="txt", quiet=True) - j = json.loads(s) + def __init__(self, + rdb_compute_domain: Union[str, int, dict, RDBComputeDomain] = None, + database: Union[str, int, dict, Database] = None, + file_service: Union[str, dict] = None, + results_base_path: str = None, + outputs: Outputs = None, + verbose: bool = True, + hard_fail: bool = False): + """ + Creates instance of SciQuery class. + + :param rdb_compute_domain: defines a domain or environment of multiple databases where users can run queries. + Can be either the domain's name (string), ID (integer), an object of class RDBComputeDomain, or a dictionary + containing all the attributes of an object of class RDBComputeDomain. If set to None, a default value will be + assigned to it. + :param database: defines the database where the queries are executed in. + Can be either the database name (string), ID (integer), an object of class Database, or a dictionary containing + all the attributes of an object of class Database. If set to None, a default value will be assigned to it. + :param file_service: a File Service defines an available file system where query result sets can be written + into. This parameter can be it name or identifier (string), or a dictionary defining a file service. + If set to None, a default value will be assigned to it. + :param results_base_path: base path (string) of the directory where the query results are written into. + Can be constructed by using FileOutput.build_file_base_path(). If set to None, a default value will be assigned + to it at the moment of running a sql query. + :param outputs: Defines the query(ies) output(s). Can be an object derived from the Output base class (such as + FileOutput or DatabaseTableOutput), or a list of those. If set to None, a default value (json file output) + will be assigned to it. + :param verbose: Boolean parameter. If True, warning messages will be written in case of errors, in the case when + the hard_fail parameter is set to False. If False, nothing will be written. + :param hard_fail: Boolean parameter. If True, exceptions will be raised in case of errors during instantiation. + If False, then no exceptions are raised, and warnings might be showed instead + (depending on the value of the verbose parameter). + """ - result=j['Result'][0] - df=pd.DataFrame(result['Data'],columns=result['ColumnNames']) - df.name = result['TableName'] - return df + self.user = SciQuery.get_user() + self.verbose = verbose + self.hard_fail = hard_fail + self._file_service = None + self._results_base_path = None + self._outputs = None + self._rdb_compute_domains = None + self._rdb_compute_domain = None + self._database = None + self.refresh_date = None + self.set(rdb_compute_domain, database, file_service, results_base_path, outputs, verbose, hard_fail) + + @staticmethod + def get_token() -> str: + token = Authentication.getToken() + if token is None or token == "": + raise Exception("User not has not logged into SciServer. Use 'Authentication.login'.") + return token + + @staticmethod + def get_user() -> Authentication.KeystoneUser: + return Authentication.getKeystoneUserWithToken(SciQuery.get_token()) + + def set(self, + rdb_compute_domain: Union[str, int, dict, RDBComputeDomain] = None, + database: Union[str, int, dict, Database] = None, + file_service: Union[str, dict] = None, + results_base_path: str = None, + outputs: Outputs = None, + verbose: bool = None, + hard_fail: bool = None): + """ + Sets or refreshes the parameters in the SciQuery object, all at once. + + :param rdb_compute_domain: defines a domain or environment of multiple databases where users can run queries. + Can be either the domain's name (string), ID (integer), an object of class RDBComputeDomain, or a dictionary + containing all the attributes of an object of class RDBComputeDomain. If set to None, the current value + is refreshed. + :param database: defines the database where the queries are executed in. + Can be either the database name (string), ID (integer), an object of class Database, or a dictionary containing + all the attributes of an object of class Database. If set to None, the current value is refreshed. + :param file_service: a File Service defines an available file system where query result sets can be written + into. This parameter can be it name or identifier (string), or a dictionary defining a file service. + If set to None, the current value is refreshed. + :param results_base_path: base path (string) of the directory where the query results are written into. + Can be constructed by using FileOutput.build_file_base_path(). + :param outputs: Defines the query(ies) output(s). Can be a list of Output objects, + or a single object of class Outputs. If set to None, a default value (json file output) will be assigned to it. + :param verbose: Boolean parameter. If True, warning messages will be written in case of errors, in the case when + the hard_fail parameter is set to False. If False, nothing will be written. + :param hard_fail: Boolean parameter. If True, exceptions will be raised in case of errors during instantiation. + If False, then no exceptions are raised, and warnings might be showed instead + (depending on the value of the verbose parameter). + """ + self.verbose = verbose if verbose else self.verbose + self.hard_fail = hard_fail if hard_fail else self.hard_fail + + # set or refresh current _rdb_compute_domains + try: + self.rdb_compute_domains = SciQuery.get_rdb_compute_domains('class') + except Exception as ex: + self._handle_exception(NameError(ex), "Unable to set or refresh rdb_compute_domains.") + # nothing else to do: + return + + try: + if self.rdb_compute_domain is None: + self.rdb_compute_domain = rdb_compute_domain if rdb_compute_domain else \ + self.get_default_rdb_compute_domain() + else: + self.rdb_compute_domain = rdb_compute_domain if rdb_compute_domain else self.rdb_compute_domain + except Exception as ex: + self._handle_exception(NameError(ex), "Unable to set or refresh rdb_compute_domain.") -def getRDBComputeDomains(): - """ - Gets a list of all registered Relational Database (RDB) compute domains that the user has access to. + try: + if self.database is None: + self.database = database if database else self.get_default_database() + else: + self.database = database if database else self.database + except Exception as ex: + self._handle_exception(NameError(ex), "Unable to set or refresh database.") - :return: a list of dictionaries, each one containing the definition of an RDB compute domain. - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). - Throws an exception if the HTTP request to the JOBM API returns an error. - :example: rdb_compute_domains = SciQuery.getRDBComputeDomains(); + try: + if self.file_service is None: + self.file_service = file_service if file_service else self.get_default_file_service() + else: + self.file_service = file_service if file_service else self.file_service + except Exception as ex: + self._handle_exception(NameError(ex), "Unable to set or refresh file_service.") - .. seealso:: SciQuery.executeQuery, SciQuery.submitQueryJob - """ - token = Authentication.getToken() - if token is not None and token != "": + try: + if self.outputs is None: + self.outputs = outputs if outputs else self.get_default_outputs() + else: + self.outputs = outputs if outputs else self.outputs + except Exception as ex: + self._handle_exception(NameError(ex), "Unable to set or refresh outputs.") + + try: + self.results_base_path = results_base_path + except Exception as ex: + self._handle_exception(NameError(ex), "Unable to set results_base_path.") + + self.refresh_date = datetime.now() + + def _handle_exception(self, exception: Exception, extra_message: str = ""): + + message = extra_message + " Error: " + str(exception) if extra_message else str(exception) + if self.hard_fail: + exception.message = message + raise exception + elif self.verbose: + warnings.warn(message) + + def refresh(self): + self.set(verbose=self.verbose, hard_fail=self.hard_fail) + + @staticmethod + def get_mydb_name(owner_name: str = None) -> str: + if not owner_name: + owner_name = SciQuery.get_user().userName + return "mydb:" + owner_name + + @staticmethod + def get_rdb_compute_domains(result_format: str = 'class') -> RDBComputeDomains: + """ + Gets a list of all registered Relational Database (RDB) compute domains that the user has access to. + + :param result_format: If set to "class", then the returned value will be of class RDBComputeDomains. + If set to "dict", then the return value will be a list of dictionaries, each of them containing the attributes + of an RDBComputeDomain object. + :return: an object of class RDBComputeDomains, or a list of dictionaries, each of them containing the attributes + of an RDBComputeDomain object. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that + purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: rdb_compute_domains = SciQuery.get_rdb_compute_domains(); + """ + token = SciQuery.get_user().token if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.SciQuery.get_compute_domains" + task_name = "Compute.SciScript-Python.SciQuery.get_rdb_compute_domains" else: - taskName = "SciScript-Python.SciQuery.get_compute_domains" + task_name = "SciScript-Python.SciQuery.get_rdb_compute_domains" - url = Config.RacmApiURL + "/jobm/rest/computedomains/rdb?TaskName=" + taskName + url = Config.SciqueryURL + "/api/info/domain?TaskName=" + task_name headers = {'X-Auth-Token': token, "Content-Type": "application/json"} res = requests.get(url, headers=headers, stream=True) if res.status_code != 200: - raise Exception("Error when getting RDB Compute Domains from JOBM API.\nHttp Response from JOBM API returned" - " status code " + str(res.status_code) + ":\n" + res.content.decode()); + raise Exception( + "Error when getting RDB Compute Domains from JOBM API.\nHttp Response from JOBM API returned" + " status code " + str(res.status_code) + ":\n" + res.content.decode()) else: - return json.loads(res.content.decode()) - else: - raise Exception("User token is not defined. First log into SciServer.") + arr = json.loads(res.content.decode()) + if result_format == 'class': + return RDBComputeDomains([RDBComputeDomain(d) for d in arr]) + else: + return arr + # rdb_compute_domains --------------------------------------------------- -def getRDBComputeDomainNames(rdbComputeDomains=None): - """ - Returns the names of the RDB compute domains available to the user. + @property + def rdb_compute_domains(self) -> RDBComputeDomains: + return self._rdb_compute_domains - :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by - Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. - :return: an array of strings, each being the name of a rdb compute domain available to the user. - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). - Throws an exception if the HTTP request to the RACM API returns an error. - :example: dockerComputeDomainsNames = Files.getDockerComputeDomainsNames(); + @rdb_compute_domains.setter + def rdb_compute_domains(self, rdb_compute_domains: RDBComputeDomains): + if not isinstance(rdb_compute_domains, RDBComputeDomains): + raise Exception("'rdb_compute_domains' should be of class RDBComputeDomains.") + self._rdb_compute_domains = rdb_compute_domains - .. seealso:: Files.getRDBComputeDomains - """ - if rdbComputeDomains is None: - rdbComputeDomains = getRDBComputeDomains(); + # rdb_compute_domain --------------------------------------------------- - rdbComputeDomainsNames = []; - for rdbComputeDomain in rdbComputeDomains: - rdbComputeDomainsNames.append(rdbComputeDomain.get('name')) + @property + def rdb_compute_domain(self) -> RDBComputeDomain: + return self._rdb_compute_domain - return rdbComputeDomainsNames; + @rdb_compute_domain.setter + def rdb_compute_domain(self, rdb_compute_domain: Union[str, int, dict, RDBComputeDomain]): + if rdb_compute_domain is None: + raise Exception("'rdb_compute_domain' cannot be set to None.") + self._rdb_compute_domain = self.get_rdb_compute_domain(rdb_compute_domain) -def getRDBComputeDomainFromName(rdbComputeDomainName, rdbComputeDomains = None): - """ - Returns an RDBComputeDomain object, given its registered name. - - :param rdbComputeDomainName: name of the RDBComputeDomainName, as shown within the results of - Jobs.getRDBComputeDomains() - :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by - Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. - :return: an RDBComputeDomain object (dictionary) that defines an RDB compute domain. A list of these kind of objects - available to the user is returned by the function Jobs.getRDBComputeDomains(). - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). - Throws an exception if the HTTP request to the JOBM API returns an error. - :example: rdbComputeDomain = SciQuery.getRDBComputeDomainFromName(rdbComputeDomainName); - - .. seealso:: SciQuery.getRDBComputeDomains - """ - if rdbComputeDomainName is None: - raise Exception("rdbComputeDomainName is not defined.") - else: - if rdbComputeDomains is None: - rdbComputeDomains = getRDBComputeDomains(); + def get_rdb_compute_domain(self, rdb_compute_domain: Union[str, int, dict, RDBComputeDomain] = None) \ + -> RDBComputeDomain: + """ + Returns an object of class RDBComputeDomain, either defined by the input name or identifiers, or that + which is set in the SciQuery instance. + + :param rdb_compute_domain: defines a domain or environment of multiple databases where users can run queries. + Can be either the domain's name (string), ID (integer), an object of class RDBComputeDomain, or a dictionary + containing all the attributes of an object of class RDBComputeDomain. If set to None, then the currently set + value of rdb_compute_domain in the SciQuery object is returned. + :return: Object of class RDBComputeDomain. + """ + if rdb_compute_domain is None: + return self._rdb_compute_domain + return self.rdb_compute_domains.get_rdb_compute_domain(rdb_compute_domain) - if rdbComputeDomains.__len__() > 0: - for rdbComputeDomain in rdbComputeDomains: - if rdbComputeDomainName == rdbComputeDomain.get('name'): - return rdbComputeDomain; - else: - raise Exception("There are no RDBComputeDomains available for the user."); + def get_default_rdb_compute_domain(self): + return self.rdb_compute_domains.get_default_rdb_compute_domain() - raise Exception("RDBComputeDomain of name '" + rdbComputeDomainName + "' is not available or does not exist."); + # database --------------------------------------------------- -def getJobsList(top=10, open=None, start=None, end=None): - """ - Gets the list of SciQuery Jobs submitted by the user. - - :param top: top number of jobs (integer) returned. If top=None, then all jobs are returned. - :param open: If set to 'True', then only returns jobs that have not finished executing and wrapped up - (status <= FINISHED). If set to 'False' then only returnes jobs that are still running. If set to 'None', - then returns both finished and unfinished jobs. - :param start: The earliest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. - If set to 'None', then there is no lower bound on date. - :param end: The latest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. - If set to 'None', then there is no upper bound on date. - :return: a list of dictionaries, each one containing the definition of a submitted job. - :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request - to the JOBM API returns an error. - :example: jobs = SciQuery.getJobsList(top=2); - - .. seealso:: SciQuery,getJob, SciQuery.getJobStatus, SciQuery.cancelJob - """ - job_dict_list = Jobs.getJobsList(top=top, open=open, start=start, end=end, type='rdb') - rdb_job_list = [] - for job_dict in job_dict_list: - rdb_job_list.append(RDBJob(job_dict)) - return rdb_job_list + @property + def database(self) -> Database: + return self._database -def getJob(jobId): - """ - Gets the definition of the job as a RDBJob object. + @database.setter + def database(self, database: Union[str, int, dict, Database]): + if database is None: + raise Exception("'database' cannot be set to None.") + self._database = self.get_database(database, self.rdb_compute_domain) - :param jobId: Id of job - :return: RDBJob object containing the description or definition of the job. - :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request - to the JOBM API returns an error. - :example: job = SciQuery.getJob(jobId) + def get_database(self, + database: Union[str, int, dict, Database] = None, + rdb_compute_domain: Union[str, int, dict, RDBComputeDomain] = None) -> Database: + """ + Returns an object of class Database, either defined by the input name or identifiers, or that + which is set in the SciQuery instance. + + + :param database: identifies the database, which this function returns as an object of class Database. + Can be either the database name (string), ID (integer), an object of class Database, or a dictionary containing + all the attributes of an object of class Database. If set to None, then the currently set value of database in + the SciQuery object is returned. + :param rdb_compute_domain: defines a domain or environment of multiple databases where users can run queries, + and that contains the database. Can be either the domain's name (string), ID (integer), an object of class + RDBComputeDomain, or a dictionary containing all the attributes of an object of class RDBComputeDomain. + If set to None, then the currently set value of rdb_compute_domain in the SciQuery object is internally used. + :return: Object of class Database + """ + if database is None: + return self._database + return self.get_rdb_compute_domain(rdb_compute_domain).get_database(database) - .. seealso:: SciQuery.getJob, SciQuery.cancelJob, SciQuery.submitQueryJob - """ - return RDBJob(Jobs.getJobDescription(jobId)) + def get_default_database(self, rdb_compute_domain: Union[str, int, dict, RDBComputeDomain] = None) -> Database: + domain = self.get_default_rdb_compute_domain() if rdb_compute_domain is None \ + else self.get_rdb_compute_domain(rdb_compute_domain) + return domain.get_default_database() -def getJobStatus(jobId): - """ - Gets a dictionary with the job status as an integer value, together with its semantic meaning. The integer value is - a power of 2, that is, 1:PENDING, 2:QUEUED, 4:ACCEPTED, 8:STARTED, 16:FINISHED, 32:SUCCESS, 64:ERROR, 128:CANCELED + # file_service --------------------------------------------------- - :param jobId: Id of job (integer). - :return: dictionary with the integer value of the job status, as well as its semantic meaning. - :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request - to the JOBM API returns an error. - :example: status = SciQuery.getJobStatus(jobId) + @property + def file_service(self) -> dict: + return self._file_service - .. seealso:: SciQuery.cancelJob, SciQuery.waitForJob, SciQuery.getJob, SciQuery.cancelJob - """ - return Jobs.getJobStatus(jobId) + @file_service.setter + def file_service(self, file_service: Union[str, dict]): + if file_service is None: + raise Exception("'file_service' cannot be set to None.") + self._file_service = self.get_file_service(file_service) -def cancelJob(jobId): - """ - Cancels the execution of a job. + def get_file_service(self, file_service: Union[str, dict] = None) -> dict: + """ + Returns the definition of a file service as a dictionary, either defined by the input name or identifiers, + or that which is set in the SciQuery instance. - :param jobId: Id of the job (integer) - :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if - the HTTP request to the JOBM API returns an error. - :example: SciQuery.cancelJob(jobId); + :param file_service: name or identifier (string) of a file service, or the dictionary with its definition. + If set to None, then the currently set value of file_service in the SciQuery object is returned. + :return: dictionary with the definition of a file service. + """ + if file_service is None: + return self._file_service + return FileOutput.find_file_service(file_service) - .. seealso:: SciQuery.getJobStatus, SciQuery.getJobDescription - """ - Jobs.cancelJob(jobId) + def get_default_file_service(self) -> dict: + return FileOutput.find_file_service() -def waitForJob(jobId, verbose=False, pollTime = 5): - """ - Queries the job status regularly and waits until the job is completed. - - :param jobId: id of job (integer) - :param verbose: if True, will print "wait" messages on the screen while the job is still running. If False, it will - suppress the printing of messages on the screen. - :param pollTime: idle time interval (integer, in seconds) before querying again for the job status. Minimum value - allowed is 5 seconds. - :return: After the job is finished, returns a dictionary object containing the job status. - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). - Throws an exception if the HTTP request to the JOBM API returns an error. - :example: jobStatus = SciQuery.waitForJob(jobId) - - .. seealso:: SciQuery.getJobStatus, SciQuery.getJobDescription - """ - return Jobs.waitForJob(jobId=jobId, verbose=verbose, pollTime = pollTime) + # results_base_path --------------------------------------------------- + @property + def results_base_path(self) -> str: + return self._results_base_path -def getDatabasesMetadata(rdbComputeDomain, format="pandas"): - """ - Gets metadata (name and description) of databases in an RDBComputeDomain. - - :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) that - defines a RDBComputeDomain. A list of these kind of objects available to the user is returned by - the function 'getRDBComputeDomains'. - :param format: String that defines the returned format. Either "pandas" for a pandas dataframe, or "dict" for - a dictionary object. - :return: pandas dataframe or dict (depending of the value of 'param') with associated metadata. - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). - Throws an exception if the HTTP request to the JOBM API returns an error. - :example: metadata = SciQuery.getDatabasesMetadata(rdbComputeDomainName); - - .. seealso:: SciQuery.getDatabaseNames - """ + @results_base_path.setter + def results_base_path(self, results_base_path: str): + # if results_base_path is None or not results_base_path.startswith(Config.ComputeWorkDir): + # raise Exception(f"The string 'results_base_path' must start with {Config.ComputeWorkDir}") + self._results_base_path = results_base_path - if type(rdbComputeDomain) == str: - rdbComputeDomain = getRDBComputeDomainFromName(rdbComputeDomain) - databaseContexts = rdbComputeDomain.get("databaseContexts") - if format == "dict": - return databaseContexts - columnNames = ['database_name', 'database_description'] - data = [] - for i in range(len(databaseContexts)): - data.append([databaseContexts[i]['name'], databaseContexts[i]['description']]) - df = pd.DataFrame(data=data, columns=columnNames) - return df + def get_results_base_path(self) -> str: + return self._results_base_path + def get_default_results_base_path(self, add_date_ending=True) -> str: + return FileOutput.build_file_base_path(add_date_ending=add_date_ending) -def getDatabaseNames(rdbComputeDomain): - """ - Gets a list of the names of databases in an RDBComputeDomain + # outputs ------------------------------------------------------------- - :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) that - defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by - the function 'getRDBComputeDomains'. - :return: array of database names (strings) - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). - Throws an exception if the HTTP request to the JOBM API returns an error. - :example: dbnames = SciQuery.getDatabaseNames(rdbComputeDomainName); + @property + def outputs(self) -> Outputs: + return self._outputs - .. seealso:: SciQuery.getDatabasesMetadata - """ - databases = getDatabasesMetadata(rdbComputeDomain, format="pandas") - return [name for name in databases['database_name']] + @outputs.setter + def outputs(self, outputs: Union[Outputs, Output]): + if outputs is None: + raise Exception("'outputs' cannot be set to None.") + self._outputs = self.get_outputs(outputs) -def getRDBComputeDomainsMetadata(format="pandas", includeDatabases=False): - """ - Gets metadata related to all relational database (RDB) compute domains (RDBComputeDomains) available to the user. - - :param format: String that defines the returned format. Either "pandas" for a pandas dataframe, or "dict" for - a dictionary object. - :param includeDatabases: Boolean parameter. If True, it will return metadata related to all available databases in - each RDBComputeDomain as well. - :return: pandas dataframe or dict (depending of the value of 'param') containing associated metadata. - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). - Throws an exception if the HTTP request to the JOBM API returns an error. - :example: metadata = SciQuery.getRDBComputeDomainsMetadata(); - - .. seealso:: SciQuery.getDatabasesMetadata, Sciquery.getDatabaseNames - """ - rdbComputeDomains = getRDBComputeDomains() - if format == "dict": - return rdbComputeDomains - columnNames = ['domain_name', 'domain_description'] - if includeDatabases: - columnNames.append("database_name") - columnNames.append("database_description") - data = [] - for i in range(len(rdbComputeDomains)): - domName = rdbComputeDomains[i].get('name') - domDescr = rdbComputeDomains[i].get('description') - dbs = rdbComputeDomains[i].get('databaseContexts') - if includeDatabases: - for j in range(len(dbs)): - data.append([domName, domDescr, dbs[j].get('name'), dbs[j].get('description')]) + def get_outputs(self, outputs: Union[Outputs, Output] = None) -> Outputs: + """ + Returns an object of class Outputs, either defined by the inputs parameters, or that + which is set in the SciQuery instance. + + :param outputs: object of class Outputs, or iterable of output objects. If set to None, then the currently + set value of outputs in the SciQuery object is returned. + :return: object of class Outputs. + """ + if outputs is None: + return self.outputs + return Outputs(outputs) + + def get_default_outputs(self) -> Outputs: + return Outputs.get_default() + + # --------------------------------------------------------------------------------------------- + # Running Queries ----------------------------------------------------------------------------- + # --------------------------------------------------------------------------------------------- + + def submit_query_job(self, + sql_query: str, + database: Union[str, int, dict, Database] = None, + outputs: Union[Outputs, Output] = None, + results_base_path: str = None, + rdb_compute_domain: Union[str, int, dict, RDBComputeDomain] = None, + file_service: str = None, + job_alias: str = "") -> int: + """ + Submits a sql query for execution (as an asynchronous job) inside a relational database (RDB) compute domain. + + :param sql_query: sql query (string) + :param database: defines the database where the sql query is executed in. + Can be either the database name (string), ID (integer), an object of class Database, or a dictionary containing + all the attributes of an object of class Database. If set to None, then the current value of the database field + in this SciQuery instance will be used. + :param outputs: Defines the query(ies) output(s). Can be an object derived from the Output base class (such as + FileOutput or DatabaseTableOutput), or a list of those. If set to None, then the current value of the outputs + field in this SciQuery instance will be used. + :param results_base_path: full path to results folder (string) where query output tables are written into. + E.g.: /home/idies/workspace/rootVOlume/username/userVolume/jobsFolder . If set to None, then its current value + in this SciQuery instance will be used. If that value is None, then a default folder will be set automatically. + :param rdb_compute_domain: defines a domain or environment of multiple databases where users can run queries, + and that contains the database. Can be either the domain's name (string), ID (integer), an object of class + RDBComputeDomain, or a dictionary containing all the attributes of an object of class RDBComputeDomain. + If set to None, then the currently set value of rdb_compute_domain in the SciQuery object is internally used. + :param file_service: a File Service defines an available file system where query result sets can be written + into. This parameter can be its name or identifier (string), or a dictionary defining a file service. + If set to None, then the currently set value of file_service in the SciQuery object is internally used. + :param job_alias: alias (string) of job, defined by the user. + :return: the ID (string) that labels the job. + """ + + domain = self.get_rdb_compute_domain(rdb_compute_domain) + db = self.get_database(database, domain) + fs = self.get_file_service(file_service) + outputs = self.get_outputs(outputs) + results_base_path = results_base_path if results_base_path else self.get_results_base_path() + if not results_base_path: + results_base_path = self.get_default_results_base_path() + + targets = outputs.get_target_list(results_base_path, fs.get('identifier')) + + job_model = { + "inputSql": sql_query, + "submitterDID": job_alias, + "databaseContextName": db.name, + "rdbDomainId": domain.id, + "targets": targets, + "resultsFolderURI": fs['identifier'] + ":" + results_base_path + } + + if Config.isSciServerComputeEnvironment(): + task_name = "Compute.SciScript-Python.SciQuery.submit_query_job" else: - data.append([domName, domDescr]) + task_name = "SciScript-Python.SciQuery.submit_query_job" - df = pd.DataFrame(data=data, columns=columnNames) - return df + data = json.dumps(job_model).encode() + url = Config.SciqueryURL + "/api/jobs/" + str(domain._racm_id) + "?TaskName=" + task_name; + headers = {'X-Auth-Token': self.user.token, "Content-Type": "application/json"} + res = requests.post(url, data=data, headers=headers, stream=True) + if res.status_code < 200 or res.status_code >= 300: + raise Exception("Error when submitting a job to the SciQuery API.\nHttp Response from SciQuery API " + + "returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return res.content.decode() + def execute_query(self, + sql_query, + database: Union[str, int, dict, Database] = None, + results_base_path: str = None, + rdb_compute_domain: Union[str, int, dict, RDBComputeDomain] = None, + job_alias: str = "", + poll_time: float = 1.0, + file_service: str = None) -> pd.DataFrame: + """ + Returns the query result (as a Pandas data frame) of a sql query submitted as a job to a + relational database (RDB) compute domain. + + :param sql_query: sql query (string) + :param database: defines the database where the sql query is executed in. + Can be either the database name (string), ID (integer), an object of class Database, or a dictionary containing + all the attributes of an object of class Database. If set to None, then the current value of the database field + in this SciQuery instance will be used. + :param results_base_path: full path to results folder (string) where query output tables are written into. + E.g.: /home/idies/workspace/rootVOlume/username/userVolume/jobsFolder . If set to None, then its current value in + this SciQuery instance will be used. If that value is None, then a default folder will be set automatically. + :param rdb_compute_domain: defines a domain or environment of multiple databases where users can run queries, + and that contains the database. Can be either the domain's name (string), ID (integer), an object of class + RDBComputeDomain, or a dictionary containing all the attributes of an object of class RDBComputeDomain. + If set to None, then the currently set value of rdb_compute_domain in the SciQuery object is internally used. + :param job_alias: alias (string) of job, defined by the user. + :param poll_time: time (float) in seconds between consecutive requests for updates in the jobs status. + :param file_service: a File Service defines an available file system where query result sets can be written + into. This parameter can be its name or identifier (string), or a dictionary defining a file service. + If set to None, then the currently set value of file_service in the SciQuery object is internally used. + :return: Pandas data frame containing the result of the query. + """ + output = FileOutput("result1.json", OutputType.FILE_JSON, 1) + job_alias = job_alias if job_alias else "synchronous query" + job_id = self.submit_query_job(sql_query=sql_query, rdb_compute_domain=rdb_compute_domain, database=database, + outputs=output, + results_base_path=results_base_path, + job_alias=job_alias, + file_service=file_service) + + job = self.wait_for_job(job_id, verbose=False, poll_time=poll_time) + if job.status > 32: + messages = ". ".join(job.message_list) + if (job.status == 64): + raise Exception("Query ended with an error. " + messages) + if (job.status == 128): + raise Exception("Query was cancelled. " + messages) + + df = job.get_dataframe_from_output(0) + return df + + @staticmethod + def get_jobs_list(top=5, open=None, start=None, end=None, result_format="pandas") \ + -> Union[pd.DataFrame, list]: + """ + Gets the list of SciQuery Jobs submitted by the user. + + :param top: top number of jobs (integer) returned. If top=None, then all jobs are returned. + :param open: If set to 'True', then only returns jobs that have not finished executing and wrapped up + (status <= FINISHED). If set to 'False' then only returns jobs that are still running. If set to 'None', + then returns both finished and unfinished jobs. + :param start: The earliest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. + If set to 'None', then there is no lower bound on date. + :param end: The latest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. + If set to 'None', then there is no upper bound on date. + :param result_format: string defining the return format. "pandas" for a pandas dataframe and "list" + for a list of RDBJob objects. + :return: pandas dataframe, or list of RDBJob objects or, each containing the definition of a submitted job. + """ + job_dict_list = Jobs.getJobsList(top=top, open=open, start=start, end=end, type='rdb') + jobs = [] + for job_dict in job_dict_list: + j = RDBJob(job_dict) + jobs.append(j if format == "list" else j.get_metadata()) + if result_format == "pandas": + jobs = pd.concat(jobs, ignore_index=True) + return jobs + + @staticmethod + def get_job(job_id): + """ + Gets the definition of the job as a RDBJob object. + :param job_id: Id of job + :return: RDBJob object containing the description or definition of the job. + """ + return RDBJob.get_job(job_id) + @staticmethod + def get_job_status(job_id): + """ + Gets a dictionary with the job status as an integer value, together with its semantic meaning. The integer value is + a power of 2, that is, 1:PENDING, 2:QUEUED, 4:ACCEPTED, 8:STARTED, 16:FINISHED, 32:SUCCESS, 64:ERROR, 128:CANCELED -class MetadataType: - """ - Contains a set of metadata types. - """ - TABLES = "TABLES" - VIEWS = "VIEWS" - COLUMNS = "COLUMNS" - ROUTINES = "ROUTINES" - CONSTRAINTS = "CONSTRAINTS" - PARAMETERS = "PARAMETERS" + :param job_id: Id of job (integer). + :return: dictionary with the integer value of the job status, as well as its semantic meaning. + """ + return Jobs.getJobStatus(job_id) + @staticmethod + def cancel_job(job): + """ + Cancels the execution of a job. -def _getMetadata(rdbComputeDomain, databaseContextName, resourceName="", metadataType=None, format="pandas"): - """ - Utility function for the use of other metadata functions. - - :param rdbComputeDomain: object (dictionary) that defines a relational database (RDB) compute domain. A list of - these kind of objects available to the user is returned by the function 'getRDBComputeDomains'. - :param databaseContextName: database context name (string) on which the sql query is executed. - :return: the ID (integer) that labels the job. - :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if - the HTTP request to the SciQuery API returns an error, or if the volumes defined by the user are not available in - the Docker compute domain. - :example: job_id = SciQuery.submitQueryJob('select 1;') - - .. seealso:: SciQuery.submitQueryJob, SciQuery.getJobStatus, SciQuery.getJob - """ + :param job_id: Id of the job (integer) + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if + the HTTP request to the JOBM API returns an error. + :example: SciQuery.cancelJob(jobId); - token = Authentication.getToken() - if token is not None and token != "": + .. seealso:: SciQuery.get_job_status, SciQuery.getJobDescription + """ + if isinstance(job, str): + Jobs.cancelJob(job) + elif isinstance(job, RDBJob): + job.cancel() + else: + raise NameError("Invalid type for input parameter 'job'.") - if metadataType not in [a for a in dir(MetadataType) if not a.startswith("__")]: - raise ValueError("Invalid value of metadataType paramter") + def wait_for_job(self, job_id, verbose = False, poll_time=1.0): + """ + Queries the job status regularly and waits until the job is completed. + + :param job_id: id of job (integer) + :param verbose: if True, will print "wait" messages on the screen while the job is still running. If False, it + will suppress the printing of messages on the screen. + :param poll_time: idle time interval (integer, in seconds) before querying again for the job status. Minimum + value allowed is 0.1 seconds. + :return: After the job is finished, returns a dictionary object containing the job status. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that + purpose). + Throws an exception if the HTTP request to the JOBM API returns an error. + :example: jobStatus = SciQuery.wait_for_job(jobId) + + .. seealso:: SciQuery.get_job_status, SciQuery.getJobDescription + """ + #return Jobs.waitForJob(jobId=jobId, verbose=verbose, pollTime=poll_time) + min_poll_time = 0.1 # in seconds + while True: + if verbose: + print("Waiting...") + job_desc = Jobs.getJobDescription(job_id) + if job_desc.get("status") >= 32: + if verbose: + print("Done!") + return RDBJob(job_desc) + else: + time.sleep(max(min_poll_time, poll_time)) - if format not in ["pandas", "dict"]: - raise ValueError("Invalid value of format paramter") + ### METADATA - if Config.isSciServerComputeEnvironment(): - taskName = "Compute.SciScript-Python.Sciquery.getMetadata_" + metadataType + def get_rdb_compute_domains_metadata(self, do_include_databases=False): + """ + Gets metadata related to all relational database (RDB) compute domains (RDBComputeDomains) available. + + :param do_include_databases: Boolean parameter. If True, it will return metadata related to all available + databases in each RDBComputeDomain as well. + :return: pandas dataframe containing associated metadata. + """ + dfs = [] + for domain in self.rdb_compute_domains: + dfs.append(domain.get_metadata(do_include_databases)) + + df = pd.concat(dfs, ignore_index=True) + df.sort_values(by="rdb_compute_domain_name", inplace=True) + return df + + def get_rdb_compute_domain_names(self): + """ + Returns the names of the RDB compute domains available to the user. + + :return: an array of strings, each being the name of a rdb compute domain available to the user. + """ + return [d.name for d in self.rdb_compute_domains] + + def get_rdb_compute_domain_metadata(self, rdb_compute_domain=None, do_include_databases=False): + return self.get_rdb_compute_domain(rdb_compute_domain).get_metadata(do_include_databases) + + def get_databases_metadata(self, rdb_compute_domain=None): + """ + Gets metadata (name and description) of databases in an RDBComputeDomain. + """ + rdb_compute_domain = self.get_rdb_compute_domain(rdb_compute_domain) + if isinstance(rdb_compute_domain, Iterable): + dfs = [d.get_databases_metadata() for d in rdb_compute_domain] + return pd.concat(dfs, ignore_index=True) else: - taskName = "SciScript-Python.Sciquery.getMetadata_" + metadataType + return rdb_compute_domain.get_databases_metadata() + + def get_database_metadata(self, database=None, rdb_compute_domain=None): + rdb_compute_domain = self.get_rdb_compute_domain(rdb_compute_domain) + database = self.get_database(database, rdb_compute_domain) + return database.get_metadata() - if type(rdbComputeDomain) == str: - rdbComputeDomain = getRDBComputeDomainFromName(rdbComputeDomain) + def get_database_names(self, rdb_compute_domain=None): + """ + Gets a list of the names of databases in an RDBComputeDomain - rdbComputeDomainId = rdbComputeDomain.get("id") + :return: array of database names (strings) + """ + rdb_compute_domain = self.get_rdb_compute_domain(rdb_compute_domain) + return rdb_compute_domain.get_database_names() - url = Config.SciqueryURL + "/api/metadata/{0}/{1}/".format(rdbComputeDomainId, databaseContextName); - if metadataType == MetadataType.TABLES: + def _get_metadata(self, rdb_compute_domain, database, resource_name="", metadata_type=""): + """ + Utility function for the use of other metadata functions. + """ + + rdb_compute_domain = self.get_rdb_compute_domain(rdb_compute_domain) + database = self.get_database(database, rdb_compute_domain.id) + + if metadata_type not in [t for t in dir(_MetadataType) if not t.startswith("__")]: + raise TypeError("Invalid type for input parameter 'metadata_type'.") + + if Config.isSciServerComputeEnvironment(): + task_name = "Compute.SciScript-Python.Sciquery.get_metadata_" + metadata_type + else: + task_name = "SciScript-Python.Sciquery.get_metadata_" + metadata_type + + url = Config.SciqueryURL + "/api/metadata/{0}/{1}/".format(rdb_compute_domain._racm_id, database.name); + if metadata_type == _MetadataType.TABLES: url += "tables" - elif metadataType == MetadataType.VIEWS: + elif metadata_type == _MetadataType.VIEWS: url += "views" - elif metadataType == MetadataType.ROUTINES: + elif metadata_type == _MetadataType.ROUTINES: url += "routines" - elif metadataType == MetadataType.COLUMNS: - url += "{0}/{1}".format(resourceName, "columns") - elif metadataType == MetadataType.PARAMETERS: - url += "{0}/{1}".format(resourceName, "parameters") - elif metadataType == MetadataType.CONSTRAINTS: - url += "{0}/{1}".format(resourceName, "constraints") + elif metadata_type == _MetadataType.COLUMNS: + url += "{0}/{1}".format(resource_name, "columns") + elif metadata_type == _MetadataType.PARAMETERS: + url += "{0}/{1}".format(resource_name, "parameters") + elif metadata_type == _MetadataType.CONSTRAINTS: + url += "{0}/{1}".format(resource_name, "constraints") else: - raise ValueError("Wrong metadataType parameter value of " + metadataType) + raise ValueError("Wrong metadata_type parameter value of " + metadata_type) - url += "?taskName=" + taskName - - headers = {'X-Auth-Token': token} + url += "?taskName=" + task_name + headers = {'X-Auth-Token': self.user.token} res = requests.get(url, headers=headers, stream=True) if res.status_code < 200 or res.status_code >= 300: raise Exception("Error when getting metadata from SciQuery API.\nHttp Response from SciQuery API " + - "returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + "returned status code " + str(res.status_code) + ":\n" + res.content.decode()) else: res = json.loads(res.content.decode()) result = res['Result'][0] - if format == "pandas": - df = pd.DataFrame(result['Data'], columns=[c.upper() for c in result['ColumnNames']]) - df.name = result['TableName'] - return df - else: - return result - else: - raise Exception("User token is not defined. First log into SciServer.") - - -def getTablesMetadata(rdbComputeDomain, databaseContextName, format="pandas"): - """ - Gets metadata related to tables in a particular database belonging to an RDBComputeDomain. - - :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) - that defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by - the function 'getRDBComputeDomains'. - :param databaseContextName: name (string) of the database. - :param format: String that defines the returned format. Either "pandas" for a pandas dataframe, - or "dict" for a dictionary object. - :return: pandas dataframe or dict (depending of the value of 'param') containing associated metadata. - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). - Throws an exception if the HTTP request to the JOBM API returns an error. - :example: metadata = SciQuery.getTablesMetadata(rdbComputeDomain, databaseContextName) - - .. seealso:: SciQuery.getTableNames, SciQuery.getDatabasesMetadata, Sciquery.getDatabaseNames - """ - return _getMetadata(rdbComputeDomain, databaseContextName, metadataType=MetadataType.TABLES, format=format) - -def getTableNames(rdbComputeDomain, databaseContextName): - """ - Gets a list of the names of tables in a particular database belonging to an RDBComputeDomain. - - :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) - that defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by - the function 'getRDBComputeDomains'. - :param databaseContextName: name (string) of the database. - :return: array of table names (strings) - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). - Throws an exception if the HTTP request to the JOBM API returns an error. - :example: tablenames = getTableNames(rdbComputeDomain, databaseContextName) - - .. seealso:: SciQuery.getTablesMetadata, SciQuery.getDatabasesMetadata, Sciquery.getDatabaseNames - """ - tables = getTablesMetadata(rdbComputeDomain, databaseContextName, format="pandas") - return [name for name in tables['TABLE_NAME']] - - -def getViewsMetadata(rdbComputeDomain, databaseContextName, format="pandas"): - """ - Gets metadata related to views in a particular database belonging to an RDBComputeDomain. - - :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) that - defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by - the function 'getRDBComputeDomains'. - :param databaseContextName: name (string) of the database. - :param format: String that defines the returned format. Either "pandas" for a pandas dataframe, or "dict" for - a dictionary object. - :return: pandas dataframe or dict (depending of the value of 'param') containing associated metadata. - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). - Throws an exception if the HTTP request to the JOBM API returns an error. - :example: metadata = SciQuery.getViewsMetadata(rdbComputeDomain, databaseContextName) - - .. seealso:: SciQuery.getTablesMetadata, SciQuery.getViewNames, SciQuery.getDatabasesMetadata - """ - return _getMetadata(rdbComputeDomain, databaseContextName, metadataType=MetadataType.VIEWS, format=format) - - -def getViewNames(rdbComputeDomain, databaseContextName): - """ - Gets a list of the names of views in a particular database belonging to an RDBComputeDomain. - - :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) - that defines an RDBComputeDomain. A list of - these kind of objects available to the user is returned by the function 'getRDBComputeDomains'. - :param databaseContextName: name (string) of the database. - :return: array of view names (strings) - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). - Throws an exception if the HTTP request to the JOBM API returns an error. - :example: viewnames = SciQuery.getViewNames(rdbComputeDomain, databaseContextName) - - .. seealso:: SciQuery.getViewsMetadata, SciQuery.getTablesMetadata, SciQuery.getDatabasesMetadata - """ - tables = getViewsMetadata(rdbComputeDomain, databaseContextName, format="pandas") - return [name for name in tables['TABLE_NAME']] + df = pd.DataFrame(result['Data'], columns=[c.upper() for c in result['ColumnNames']]) + df.name = result['TableName'] + return df + def get_tables_metadata(self, database=None, rdb_compute_domain=None): + """ + Gets metadata related to tables in a particular database belonging to an RDBComputeDomain. + """ + return self._get_metadata(rdb_compute_domain, database, metadata_type=_MetadataType.TABLES) -def getRoutinesMetadata(rdbComputeDomain, databaseContextName, format="pandas"): - """ - Gets metadata related to routines or functions in a particular database belonging to an RDBComputeDomain. - - :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) that - defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by - the function 'getRDBComputeDomains'. - :param databaseContextName: name (string) of the database. - :param format: String that defines the returned format. Either "pandas" for a pandas dataframe, or "dict" for - a dictionary object. - :return: pandas dataframe or dict (depending of the value of 'param') containing associated metadata. - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). - Throws an exception if the HTTP request to the JOBM API returns an error. - :example: metadata = SciQuery.getRoutinesMetadata(rdbComputeDomain, databaseContextName) - - .. seealso:: SciQuery.getTablesMetadata, SciQuery.getViewsMetadata, SciQuery.getDatabasesMetadata - """ - return _getMetadata(rdbComputeDomain, databaseContextName, metadataType=MetadataType.ROUTINES, format=format) + def get_table_names(self, database=None, rdb_compute_domain=None): + """ + Gets a list of the names of tables in a particular database belonging to an RDBComputeDomain. + """ + tables = self.get_tables_metadata(database, rdb_compute_domain) + return [name for name in tables['TABLE_NAME']] + def get_views_metadata(self, database=None, rdb_compute_domain=None): + """ + Gets metadata related to views in a particular database belonging to an RDBComputeDomain. + """ + return self._get_metadata(rdb_compute_domain, database, metadata_type=_MetadataType.VIEWS) -def getRoutineNames(rdbComputeDomain, databaseContextName): - """ - Gets a list of the names of routines or functions in a particular database belonging to an RDBComputeDomain. - - :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) - that defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by - the function 'getRDBComputeDomains'. - :param databaseContextName: name (string) of the database. - :return: array of routine names (strings) - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). - Throws an exception if the HTTP request to the JOBM API returns an error. - :example: routinenames = getRoutineNames(rdbComputeDomain, databaseContextName) - - .. seealso:: SciQuery.getTableNames, SciQuery.getDatabasesMetadata, Sciquery.getDatabaseNames - """ - routines = getRoutinesMetadata(rdbComputeDomain, databaseContextName, format="pandas") - return [routineName for routineName in routines['ROUTINE_NAME']] + def get_view_names(self, database=None, rdb_compute_domain=None): + """ + Gets a list of the names of views in a particular database belonging to an RDBComputeDomain. + """ + tables = self.get_views_metadata(database, rdb_compute_domain) + return [name for name in tables['TABLE_NAME']] + def get_routines_metadata(self, database=None, rdb_compute_domain=None): + """ + Gets metadata related to routines or functions in a particular database belonging to an RDBComputeDomain. + """ + return self._get_metadata(rdb_compute_domain, database, metadata_type=_MetadataType.ROUTINES) -def getColumnsMetadata(rdbComputeDomain, databaseContextName, tableName, format="pandas"): - """ - Gets metadata related to columns in a particular database table belonging to an RDBComputeDomain. - - :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) - that defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by - the function 'getRDBComputeDomains'. - :param databaseContextName: name (string) of the database. - :param tableName: name (string) of the database table. - :param format: String that defines the returned format. Either "pandas" for a pandas dataframe, or "dict" for - a dictionary object. - :return: pandas dataframe or dict (depending of the value of 'param') containing associated metadata. - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). - Throws an exception if the HTTP request to the JOBM API returns an error. - :example: metadata = SciQuery.getColumnsMetadata(rdbComputeDomain, databaseContextName, tableName) - - .. seealso:: SciQuery.getTablesMetadata, SciQuery.getViewsMetadata, SciQuery.getDatabasesMetadata - """ - return _getMetadata(rdbComputeDomain, databaseContextName, tableName, metadataType=MetadataType.COLUMNS, - format=format) + def get_routine_names(self, database=None, rdb_compute_domain=None): + """ + Gets a list of the names of routines or functions in a particular database belonging to an RDBComputeDomain. + """ + routines = self.get_routines_metadata(database, rdb_compute_domain) + return [routine_name for routine_name in routines['ROUTINE_NAME']] + def get_columns_metadata(self, table_name, database=None, rdb_compute_domain=None): + """ + Gets metadata related to columns in a particular database table belonging to an RDBComputeDomain. + """ + return self._get_metadata(rdb_compute_domain, database, table_name, metadata_type=_MetadataType.COLUMNS) -def getColumnNames(rdbComputeDomain, databaseContextName, tableName): - """ - Gets a list of the names of table columns in a particular database belonging to an RDBComputeDomain. - - :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) - that defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by - the function 'getRDBComputeDomains'. - :param databaseContextName: name (string) of the database. - :param tableName: name (string) of the database table - :return: array of column names (strings) - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). - Throws an exception if the HTTP request to the JOBM API returns an error. - :example: columnnames = SciQuery.getColumnNames(rdbComputeDomain, databaseContextName, tableName) - - .. seealso:: SciQuery.getTableNames, SciQuery.getDatabasesMetadata, Sciquery.getDatabaseNames - """ - columns = getColumnsMetadata(rdbComputeDomain, databaseContextName, tableName, format="pandas") - return [columnName for columnName in columns['COLUMN_NAME']] + def get_column_names(self, table_name, database=None, rdb_compute_domain=None): + """ + Gets a list of the names of table columns in a particular database belonging to an RDBComputeDomain. + """ + columns = self.get_columns_metadata(table_name, database, rdb_compute_domain) + return [columnName for columnName in columns['COLUMN_NAME']] + def get_constraints_metadata(self, table_name, database=None, rdb_compute_domain=None): + """ + Gets metadata related to table constraints in a particular database table belonging to an RDBComputeDomain. + """ + return self._get_metadata(rdb_compute_domain, database, table_name, metadata_type=_MetadataType.CONSTRAINTS) -def getConstraintsMetadata(rdbComputeDomain, databaseContextName, tableName, format="pandas"): - """ - Gets metadata related to table constraints in a particular database table belonging to an RDBComputeDomain. - - :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) - that defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by - the function 'getRDBComputeDomains'. - :param databaseContextName: name (string) of the database. - :param tableName: name (string) of the database table. - :param format: String that defines the returned format. Either "pandas" for a pandas dataframe, or "dict" for - a dictionary object. - :return: pandas dataframe or dict (depending of the value of 'param') containing associated metadata. - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). - Throws an exception if the HTTP request to the JOBM API returns an error. - :example: metadata = SciQuery.getConstraintsMetadata(rdbComputeDomain, databaseContextName, tableName) - - .. seealso:: SciQuery.getTablesMetadata, SciQuery.getColumnsMetadata, SciQuery.getDatabasesMetadata - """ - return _getMetadata(rdbComputeDomain, databaseContextName, tableName, metadataType=MetadataType.CONSTRAINTS, - format=format) + def get_constraint_names(self, table_name, database=None, rdb_compute_domain=None): + """ + Gets a list of the names of table constraints in a particular database belonging to an RDBComputeDomain. + """ + constraints = self.get_constraints_metadata(table_name, database, rdb_compute_domain) + return [constraintName for constraintName in constraints['CONSTRAINT_NAME']] + def get_routine_parameters_metadata(self, routine_name, database=None, rdb_compute_domain=None): + """ + Gets metadata related to routine parameters in a particular database belonging to an RDBComputeDomain. + """ + return self._get_metadata(rdb_compute_domain, database, routine_name, metadata_type=_MetadataType.PARAMETERS) -def getConstraintNames(rdbComputeDomain, databaseContextName, tableName): - """ - Gets a list of the names of table constraints in a particular database belonging to an RDBComputeDomain. - - :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) - that defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by - the function 'getRDBComputeDomains'. - :param databaseContextName: name (string) of the database. - :param tableName: name (string) of the database table. - :return: array of constraint names (strings) - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). - Throws an exception if the HTTP request to the JOBM API returns an error. - :example: names = SciQuery.getConstraintNames(rdbComputeDomain, databaseContextName, tableName) - - .. seealso:: SciQuery.getTableNames, SciQuery.getColumnNames, SciQuery.getDatabasesMetadata - """ - constraints = getConstraintsMetadata(rdbComputeDomain, databaseContextName, tableName, format="pandas") - return [constraintName for constraintName in constraints['CONSTRAINT_NAME']] + def get_routine_parameter_names(self, routine_name, database=None, rdb_compute_domain=None): + """ + Gets a list of the names of routine parameters in a particular database belonging to an RDBComputeDomain. + """ + parameters = self.get_routine_parameters_metadata(routine_name, database, rdb_compute_domain) + return [name for name in parameters['PARAMETER_NAME']] + def __str__(self): + return "SciQuery instance with rdb_compute_domains = {})".format(self._rdb_compute_domains) -def getRoutineParametersMetadata(rdbComputeDomain, databaseContextName, routineName, format="pandas"): - """ - Gets metadata related to routine parameters in a particular database belonging to an RDBComputeDomain. - - :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) - that defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by - the function 'getRDBComputeDomains'. - :param databaseContextName: name (string) of the database. - :param routineName: name (string) of the routine or function. - :param format: String that defines the returned format. Either "pandas" for a pandas dataframe, or "dict" - for a dictionary object. - :return: pandas dataframe or dict (depending of the value of 'param') containing associated metadata. - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). - Throws an exception if the HTTP request to the JOBM API returns an error. - :example: metadata = SciQuery.getRoutineParametersMetadata(rdbComputeDomain, databaseContextName, routineName) - - .. seealso:: SciQuery.getTablesMetadata, SciQuery.getColumnsMetadata, SciQuery.getDatabasesMetadata - """ - return _getMetadata(rdbComputeDomain, databaseContextName, routineName, metadataType=MetadataType.PARAMETERS, - format=format) + def __repr__(self): + return "SciQuery(rdb_compute_domains = {})".format(self._rdb_compute_domains) -def getRoutineParameterNames(rdbComputeDomain, databaseContextName, routineName): +class _MetadataType: """ - Gets a list of the names of routine parameters in a particular database belonging to an RDBComputeDomain. - - :param rdbComputeDomain: Name (string) of a relational database (RDB) compute domain, or an object (dictionary) - that defines an RDBComputeDomain. A list of these kind of objects available to the user is returned by - the function 'getRDBComputeDomains'. - :param databaseContextName: name (string) of the database. - :param routineName: name (string) of the routine or function. - :return: array of parameter names (strings) - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). - Throws an exception if the HTTP request to the JOBM API returns an error. - :example: names = SciQuery.getRoutineParameterNames(rdbComputeDomain, databaseContextName, routineName) - - .. seealso:: SciQuery.getTableNames, SciQuery.getDatabasesMetadata, Sciquery.getDatabaseNames + Contains a set of metadata types. """ - parameters = getRoutineParametersMetadata(rdbComputeDomain, databaseContextName, routineName, format="pandas") - return [parametersName for parametersName in parameters['SPECIFIC_NAME']] + TABLES = "TABLES" + VIEWS = "VIEWS" + COLUMNS = "COLUMNS" + ROUTINES = "ROUTINES" + CONSTRAINTS = "CONSTRAINTS" + PARAMETERS = "PARAMETERS" From a30d075853fc155c160e9fd9345d25897372dba8 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Fri, 3 Jun 2022 18:51:32 -0400 Subject: [PATCH 70/79] fixed missing token --- py3/SciServer/SciQuery.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/py3/SciServer/SciQuery.py b/py3/SciServer/SciQuery.py index cec1a25..4ce4f25 100644 --- a/py3/SciServer/SciQuery.py +++ b/py3/SciServer/SciQuery.py @@ -740,7 +740,10 @@ def get_token() -> str: @staticmethod def get_user() -> Authentication.KeystoneUser: - return Authentication.getKeystoneUserWithToken(SciQuery.get_token()) + token = SciQuery.get_token() + user = Authentication.getKeystoneUserWithToken(token) + user.token = token + return user def set(self, rdb_compute_domain: Union[str, int, dict, RDBComputeDomain] = None, From 72c427af456ac25b29f35139741db6ed0326e4f9 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Fri, 3 Jun 2022 18:51:57 -0400 Subject: [PATCH 71/79] assign missing token --- py3/SciServer/Authentication.py | 1 + 1 file changed, 1 insertion(+) diff --git a/py3/SciServer/Authentication.py b/py3/SciServer/Authentication.py index a02f6c7..e4af767 100644 --- a/py3/SciServer/Authentication.py +++ b/py3/SciServer/Authentication.py @@ -66,6 +66,7 @@ def getKeystoneUserWithToken(token): ksu = KeystoneUser() ksu.userName = responseJson["token"]["user"]["name"] ksu.id = responseJson["token"]["user"]["id"] + ksu.token = token keystoneUser.token = token; keystoneUser.userName = ksu.userName keystoneUser.id = ksu.id From cc02c0e378d05260bd4aaaf09e1a54c9b91e095d Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Sun, 5 Jun 2022 10:45:28 -0400 Subject: [PATCH 72/79] adding poll_time to SciQuery class. --- py3/SciServer/SciQuery.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/py3/SciServer/SciQuery.py b/py3/SciServer/SciQuery.py index 4ce4f25..a61ca8b 100644 --- a/py3/SciServer/SciQuery.py +++ b/py3/SciServer/SciQuery.py @@ -722,6 +722,7 @@ def __init__(self, self.user = SciQuery.get_user() self.verbose = verbose self.hard_fail = hard_fail + self.poll_time = 1.0 self._file_service = None self._results_base_path = None self._outputs = None @@ -1114,7 +1115,6 @@ def execute_query(self, results_base_path: str = None, rdb_compute_domain: Union[str, int, dict, RDBComputeDomain] = None, job_alias: str = "", - poll_time: float = 1.0, file_service: str = None) -> pd.DataFrame: """ Returns the query result (as a Pandas data frame) of a sql query submitted as a job to a @@ -1133,7 +1133,6 @@ def execute_query(self, RDBComputeDomain, or a dictionary containing all the attributes of an object of class RDBComputeDomain. If set to None, then the currently set value of rdb_compute_domain in the SciQuery object is internally used. :param job_alias: alias (string) of job, defined by the user. - :param poll_time: time (float) in seconds between consecutive requests for updates in the jobs status. :param file_service: a File Service defines an available file system where query result sets can be written into. This parameter can be its name or identifier (string), or a dictionary defining a file service. If set to None, then the currently set value of file_service in the SciQuery object is internally used. @@ -1146,8 +1145,9 @@ def execute_query(self, results_base_path=results_base_path, job_alias=job_alias, file_service=file_service) - - job = self.wait_for_job(job_id, verbose=False, poll_time=poll_time) + if self.verbose: + print("Query was submitted as a job with id = " + job_id) + job = self.wait_for_job(job_id, verbose=False, poll_time=self.poll_time) if job.status > 32: messages = ". ".join(job.message_list) if (job.status == 64): @@ -1242,7 +1242,6 @@ def wait_for_job(self, job_id, verbose = False, poll_time=1.0): .. seealso:: SciQuery.get_job_status, SciQuery.getJobDescription """ - #return Jobs.waitForJob(jobId=jobId, verbose=verbose, pollTime=poll_time) min_poll_time = 0.1 # in seconds while True: if verbose: From a3b6328ac9be292b22b2fbcf4859d857d35972ea Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Mon, 6 Jun 2022 10:55:04 -0400 Subject: [PATCH 73/79] Improve messages in wait_for_job --- py3/SciServer/SciQuery.py | 40 ++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/py3/SciServer/SciQuery.py b/py3/SciServer/SciQuery.py index a61ca8b..1eb2edc 100644 --- a/py3/SciServer/SciQuery.py +++ b/py3/SciServer/SciQuery.py @@ -429,7 +429,7 @@ def get_json_output(self, output: Union[Output, int, str] = 0) -> dict: data_dict = json.loads(self.get_output_as_string(output)) return data_dict.get("Result") - def get_dataframe_from_output(self, output: Union[Output, int] = 0, result_index: int = 0) -> list: + def get_dataframe_from_output(self, output: Union[Output, int] = 0, result_index: int = 0) -> pd.DataFrame: out = self._get_output_from_index(output) if isinstance(output, int) else output if out.output_type == OutputType.FILE_JSON: result = self.get_json_output(out)[result_index] @@ -692,7 +692,9 @@ def __init__(self, results_base_path: str = None, outputs: Outputs = None, verbose: bool = True, - hard_fail: bool = False): + hard_fail: bool = False, + poll_time: float = 1.0 + ): """ Creates instance of SciQuery class. @@ -717,6 +719,7 @@ def __init__(self, :param hard_fail: Boolean parameter. If True, exceptions will be raised in case of errors during instantiation. If False, then no exceptions are raised, and warnings might be showed instead (depending on the value of the verbose parameter). + :param poll_time: time (float) in seconds between consecutive requests for updates in the jobs status. """ self.user = SciQuery.get_user() @@ -753,7 +756,9 @@ def set(self, results_base_path: str = None, outputs: Outputs = None, verbose: bool = None, - hard_fail: bool = None): + hard_fail: bool = None, + poll_time: float = None + ): """ Sets or refreshes the parameters in the SciQuery object, all at once. @@ -776,10 +781,12 @@ def set(self, :param hard_fail: Boolean parameter. If True, exceptions will be raised in case of errors during instantiation. If False, then no exceptions are raised, and warnings might be showed instead (depending on the value of the verbose parameter). + :param poll_time: time (float) in seconds between consecutive requests for updates in the jobs status. """ self.verbose = verbose if verbose else self.verbose self.hard_fail = hard_fail if hard_fail else self.hard_fail + self.poll_time = poll_time if poll_time else self.poll_time # set or refresh current _rdb_compute_domains try: @@ -1147,12 +1154,12 @@ def execute_query(self, file_service=file_service) if self.verbose: print("Query was submitted as a job with id = " + job_id) - job = self.wait_for_job(job_id, verbose=False, poll_time=self.poll_time) + job = self.wait_for_job(job_id, verbose=False) if job.status > 32: messages = ". ".join(job.message_list) - if (job.status == 64): + if job.status == 64: raise Exception("Query ended with an error. " + messages) - if (job.status == 128): + if job.status == 128: raise Exception("Query was cancelled. " + messages) df = job.get_dataframe_from_output(0) @@ -1225,34 +1232,29 @@ def cancel_job(job): else: raise NameError("Invalid type for input parameter 'job'.") - def wait_for_job(self, job_id, verbose = False, poll_time=1.0): + def wait_for_job(self, job_id, verbose = False): """ Queries the job status regularly and waits until the job is completed. :param job_id: id of job (integer) :param verbose: if True, will print "wait" messages on the screen while the job is still running. If False, it will suppress the printing of messages on the screen. - :param poll_time: idle time interval (integer, in seconds) before querying again for the job status. Minimum - value allowed is 0.1 seconds. - :return: After the job is finished, returns a dictionary object containing the job status. + :return: After the job is finished, returns an object of class RDBJob, containing the job definition. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. - :example: jobStatus = SciQuery.wait_for_job(jobId) - - .. seealso:: SciQuery.get_job_status, SciQuery.getJobDescription """ - min_poll_time = 0.1 # in seconds + t = max(0.1, self.poll_time) + wait_message = "Waiting" while True: - if verbose: - print("Waiting...") job_desc = Jobs.getJobDescription(job_id) if job_desc.get("status") >= 32: - if verbose: - print("Done!") return RDBJob(job_desc) else: - time.sleep(max(min_poll_time, poll_time)) + if verbose: + wait_message += "." + print(wait_message, end="\r") + time.sleep(t) ### METADATA From db67c7a3ddd0db40095669a275353ad1df6b645b Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Wed, 8 Jun 2022 13:53:27 -0400 Subject: [PATCH 74/79] fixed docs, updated minimum polling time. --- py3/SciServer/SciQuery.py | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/py3/SciServer/SciQuery.py b/py3/SciServer/SciQuery.py index 1eb2edc..83cc690 100644 --- a/py3/SciServer/SciQuery.py +++ b/py3/SciServer/SciQuery.py @@ -725,7 +725,7 @@ def __init__(self, self.user = SciQuery.get_user() self.verbose = verbose self.hard_fail = hard_fail - self.poll_time = 1.0 + self.poll_time = poll_time self._file_service = None self._results_base_path = None self._outputs = None @@ -733,7 +733,7 @@ def __init__(self, self._rdb_compute_domain = None self._database = None self.refresh_date = None - self.set(rdb_compute_domain, database, file_service, results_base_path, outputs, verbose, hard_fail) + self.set(rdb_compute_domain, database, file_service, results_base_path, outputs, verbose, hard_fail, poll_time) @staticmethod def get_token() -> str: @@ -864,9 +864,6 @@ def get_rdb_compute_domains(result_format: str = 'class') -> RDBComputeDomains: of an RDBComputeDomain object. :return: an object of class RDBComputeDomains, or a list of dictionaries, each of them containing the attributes of an RDBComputeDomain object. - :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that - purpose). Throws an exception if the HTTP request to the JOBM API returns an error. - :example: rdb_compute_domains = SciQuery.get_rdb_compute_domains(); """ token = SciQuery.get_user().token @@ -880,8 +877,8 @@ def get_rdb_compute_domains(result_format: str = 'class') -> RDBComputeDomains: res = requests.get(url, headers=headers, stream=True) if res.status_code != 200: raise Exception( - "Error when getting RDB Compute Domains from JOBM API.\nHttp Response from JOBM API returned" - " status code " + str(res.status_code) + ":\n" + res.content.decode()) + "Error when getting RDB Compute Domains from the SciQuery API.\nHttp Response from the SciQuery API " + "returned status code " + str(res.status_code) + ":\n" + res.content.decode()) else: arr = json.loads(res.content.decode()) if result_format == 'class': @@ -1205,10 +1202,11 @@ def get_job(job_id): @staticmethod def get_job_status(job_id): """ - Gets a dictionary with the job status as an integer value, together with its semantic meaning. The integer value is - a power of 2, that is, 1:PENDING, 2:QUEUED, 4:ACCEPTED, 8:STARTED, 16:FINISHED, 32:SUCCESS, 64:ERROR, 128:CANCELED + Gets a dictionary with the job status as an integer value, together with its semantic meaning. The integer value + is a power of 2, that is, + 1:PENDING, 2:QUEUED, 4:ACCEPTED, 8:STARTED, 16:FINISHED, 32:SUCCESS, 64:ERROR, 128:CANCELED - :param job_id: Id of job (integer). + :param job_id: Id of job (string). :return: dictionary with the integer value of the job status, as well as its semantic meaning. """ return Jobs.getJobStatus(job_id) @@ -1218,12 +1216,7 @@ def cancel_job(job): """ Cancels the execution of a job. - :param job_id: Id of the job (integer) - :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if - the HTTP request to the JOBM API returns an error. - :example: SciQuery.cancelJob(jobId); - - .. seealso:: SciQuery.get_job_status, SciQuery.getJobDescription + :param job: id of the job (string), or object of class RDBJob. """ if isinstance(job, str): Jobs.cancelJob(job) @@ -1232,7 +1225,7 @@ def cancel_job(job): else: raise NameError("Invalid type for input parameter 'job'.") - def wait_for_job(self, job_id, verbose = False): + def wait_for_job(self, job_id, verbose=False): """ Queries the job status regularly and waits until the job is completed. @@ -1244,7 +1237,7 @@ def wait_for_job(self, job_id, verbose = False): purpose). Throws an exception if the HTTP request to the JOBM API returns an error. """ - t = max(0.1, self.poll_time) + t = max(0.5, self.poll_time) wait_message = "Waiting" while True: job_desc = Jobs.getJobDescription(job_id) @@ -1256,7 +1249,7 @@ def wait_for_job(self, job_id, verbose = False): print(wait_message, end="\r") time.sleep(t) - ### METADATA + # METADATA ------------------------------------------------------------------------------------------------- def get_rdb_compute_domains_metadata(self, do_include_databases=False): """ @@ -1326,7 +1319,7 @@ def _get_metadata(self, rdb_compute_domain, database, resource_name="", metadata else: task_name = "SciScript-Python.Sciquery.get_metadata_" + metadata_type - url = Config.SciqueryURL + "/api/metadata/{0}/{1}/".format(rdb_compute_domain._racm_id, database.name); + url = Config.SciqueryURL + "/api/metadata/{0}/{1}/".format(rdb_compute_domain._racm_id, database.name) if metadata_type == _MetadataType.TABLES: url += "tables" elif metadata_type == _MetadataType.VIEWS: From 4fbb613713bfe46e9789e056cef5f24d6f605022 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Wed, 19 Oct 2022 13:36:26 -0400 Subject: [PATCH 75/79] Adding SciQuery demo notebook. --- Examples/Sciquery_Demo(long).ipynb | 1175 ++++++++++++++++++++++++++++ 1 file changed, 1175 insertions(+) create mode 100644 Examples/Sciquery_Demo(long).ipynb diff --git a/Examples/Sciquery_Demo(long).ipynb b/Examples/Sciquery_Demo(long).ipynb new file mode 100644 index 0000000..1806cc8 --- /dev/null +++ b/Examples/Sciquery_Demo(long).ipynb @@ -0,0 +1,1175 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from SciServer.SciQuery import *" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1) Instantiation: different ways" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 1.0 Defining name of database and domain, for future queries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "database_name = 'BestDR16'\n", + "rdb_compute_domain_name = 'IDIES-SQLServer (Short queue)'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 1.1 Creating instance with default values/settings:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq = SciQuery()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 1.2 Ways of looking at preset instance values:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "print(f\"Default Domian = {sq.rdb_compute_domain}\" )\n", + "print(f\"Default Domian = {sq.get_rdb_compute_domain()}\" )\n", + "print(f\"Default Database = {sq.database}\" )\n", + "print(f\"Default Database = {sq.get_database()}\" )\n", + "print(f\"Default Output = {sq.outputs}\" )\n", + "print(f\"Default Output = {sq.get_outputs()}\" )\n", + "print(f\"Default File service = {sq.file_service.get('name')}\" )\n", + "print(f\"Default File service = {sq.get_file_service().get('name')}\" )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 1.3 Several ways of setting values:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### 1.3.1 By name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.rdb_compute_domain = rdb_compute_domain_name\n", + "sq.database = database_name" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### 1.3.2 By object" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "rdb_compute_domain = sq.get_rdb_compute_domain(rdb_compute_domain_name)\n", + "sq.rdb_compute_domain = rdb_compute_domain\n", + "database = sq.get_database(database_name)\n", + "sq.database = database" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### 1.3.3 Altogether by name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq = SciQuery()\n", + "sq.set(rdb_compute_domain_name, database_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### 1.3.4 Altogether by object" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq = SciQuery()\n", + "sq.set(rdb_compute_domain, database)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### 1.3.5 During instantiation, by name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq = SciQuery(rdb_compute_domain_name, database_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### 1.3.6 During instantiation, by object" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq = SciQuery(rdb_compute_domain, database)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 1.4 Updating the information stored in SciQuery if there are external changes, like being allowed to query new domains/databases:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.refresh()\n", + "print(sq.refresh_date)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2) Exploring stored objects and asociated metadata" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.get_rdb_compute_domain_names()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.get_rdb_compute_domains_metadata()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.get_rdb_compute_domains_metadata(do_include_databases=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "SciQuery.get_rdb_compute_domains()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.rdb_compute_domains" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "rdb_compute_domain = sq.get_rdb_compute_domain(rdb_compute_domain_name)\n", + "rdb_compute_domain" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "rdb_compute_domain.get_metadata()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "rdb_compute_domain.get_database_names()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "rdb_compute_domain.get_databases_metadata()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "rdb_compute_domain.databases" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "database = rdb_compute_domain.get_database(database_name)\n", + "database" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "database.get_metadata()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3) Detailed Database Metadata" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.rdb_compute_domain = rdb_compute_domain_name\n", + "sq.database = database_name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "table_names = sq.get_table_names()\n", + "#table_names = sq.get_table_names(database, rdb_compute_domain)\n", + "print(table_names)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.get_tables_metadata()\n", + "# or \n", + "sq.get_tables_metadata(database, rdb_compute_domain)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "table_name = table_names[0]\n", + "print(sq.get_column_names(table_name))\n", + "# or \n", + "print(sq.get_column_names(table_name, database, rdb_compute_domain))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sq.get_columns_metadata(table_name)\n", + "# or \n", + "sq.get_columns_metadata(table_name, database, rdb_compute_domain)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(sq.get_constraint_names(table_name))\n", + "# or \n", + "print(sq.get_constraint_names(table_name, database, rdb_compute_domain))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sq.get_constraints_metadata(table_name)\n", + "# or \n", + "sq.get_constraints_metadata(table_name, database, rdb_compute_domain)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "routine_names = sq.get_routine_names()\n", + "# or \n", + "routine_names = sq.get_routine_names(database, rdb_compute_domain)\n", + "print(routine_names)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sq.get_routines_metadata()\n", + "# or \n", + "sq.get_routines_metadata(database, rdb_compute_domain)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "routine_name = routine_names[0]\n", + "print(sq.get_routine_parameter_names(routine_name))\n", + "# or \n", + "print(sq.get_routine_parameter_names(routine_name, database, rdb_compute_domain))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sq.get_routine_parameters_metadata(routine_name)\n", + "# or \n", + "sq.get_routine_parameters_metadata(routine_name, database, rdb_compute_domain)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(sq.get_view_names())\n", + "# or \n", + "print(sq.get_view_names(database, rdb_compute_domain))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sq.get_views_metadata()\n", + "# or \n", + "sq.get_views_metadata(database, rdb_compute_domain)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4) Executing Synchronous Queries to Databases " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 4.1 Setting database and compute domain" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.set(database=database_name, rdb_compute_domain=rdb_compute_domain_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 4.2 Defining Single SQL Query" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sql_query = \"SELECT TOP 2 specObjID, ra, dec, z FROM specobj\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 4.3 Getting query result as pandas dataframe" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "df = sq.execute_query(sql_query)\n", + "# or\n", + "#df = sq.execute_query(sql_query, database_name, rdb_compute_domain=rdb_compute_domain_name)\n", + "df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5) Submitting query as a batch job" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 5.1 Using default JSON file output" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "jobid = sq.submit_query_job(sql_query=sql_query)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 5.2 Getting current job status" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.get_job_status(jobid)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 5.3 Waiting for job to finish before getting it back" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job = sq.wait_for_job(jobid, verbose=True)\n", + "job\n", + "\n", + "# or after job is " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 5.4 Getting job object" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job = sq.get_job(jobid)\n", + "job" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 5.5 Getting list of submitted jobs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.get_jobs_list(top = 2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 5.6 Getting job object and checking its properties" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job = sq.get_job(jobid)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.get_job_status()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.get_metadata()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "outputs = job.outputs\n", + "outputs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.get_output_path()\n", + "# or \n", + "job.get_output_path(0)\n", + "# or\n", + "job.get_output_path(job.outputs[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.get_json_output()\n", + "# or\n", + "job.get_json_output(0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.get_dataframe_from_output(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 5.7 Cancelling job" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "jobid = sq.submit_query_job(sql_query=sql_query)\n", + "sq.cancel_job(jobid)\n", + "\n", + "# or \n", + "\n", + "#jobid = sq.submit_query_job(sql_query=sql_query)\n", + "#job = sq.get_job(jobid)\n", + "#job.cancel()\n", + "\n", + "job = sq.wait_for_job(jobid)\n", + "print(job.status_string)\n", + "print(sq.get_job_status(jobid))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "results_base_path = '/home/idies/workspace/Temporary/{}/scratch/sciqueryjobs/dir/'.format(SciQuery.get_user().userName)\n", + "# or\n", + "results_base_path = FileOutput.build_file_base_path(top_volume=\"Temporary\", \n", + " user_volume=\"scratch\",\n", + " user_volume_owner_name=SciQuery.get_user().userName, \n", + " relative_path=\"sciqueryjobs/dir/\", \n", + " add_date_ending=False)\n", + "results_base_path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "name = results_base_path + \"result.json\"\n", + "output = FileOutput(name=name, output_type=OutputType.FILE_JSON, statement_indexes=1)\n", + "print(output.file)\n", + "print(output.file_base_path)\n", + "print(output.path)\n", + "print(output.get_path())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "jobid = sq.submit_query_job(sql_query=sql_query, outputs=output)\n", + "\n", + "# or \n", + "#jobid = sq.submit_query_job(sql_query=sql_query, results_base_path=results_base_path)\n", + "\n", + "# or \n", + "#sq.set_results_base_path(results_base_path)\n", + "#jobid = sq.submit_query_job(sql_query=sql_query)\n", + "\n", + "job = sq.wait_for_job(jobid, verbose=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.outputs[0].get_path()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "out = job.outputs[0]\n", + "out" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.get_dataframe_from_output(0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.get_json_output(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 5.9 Submitting query as batch job, with output to table in mydb database" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sql_query = \"SELECT TOP 2 specObjID, ra, dec, z FROM specobj\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "output_table = \"results12e\"\n", + "output_rdb_compute_domain_name = rdb_compute_domain_name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# setting output database directly to mydb:\n", + "\n", + "output_database_name = SciQuery.get_mydb_name() \n", + "print(output_database_name)\n", + "\n", + "# verifying that it is fact a database contained in the output rdb compute domian:\n", + "\n", + "print(output_database_name in sq.get_database_names(output_rdb_compute_domain_name))\n", + "print(sq.get_rdb_compute_domain(output_rdb_compute_domain_name).get_database(output_database_name).name)\n", + "\n", + "# or verifying that it is fact the default database in the output rdb compute domian:\n", + "\n", + "output_database_name = sq.get_rdb_compute_domain(output_rdb_compute_domain_name).get_default_database().name\n", + "print(output_database_name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "output = DatabaseTableOutput(table=output_table, database=output_database_name, rdb_compute_domain=output_rdb_compute_domain_name)\n", + "output" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "jobid = sq.submit_query_job(sql_query=sql_query, outputs=output)\n", + "job = sq.wait_for_job(jobid)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "print(job.message_list)\n", + "print(job.targets)\n", + "job.get_metadata()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# looking at the contents of the output table\n", + "\n", + "job.get_dataframe_from_output(0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# or directly making a query to see the contents:\n", + "\n", + "sql_query = \"SELECT * FROM \" + output_table\n", + "sq.execute_query(sql_query, database= output_database_name, rdb_compute_domain= output_rdb_compute_domain_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 5.10 Submitting a multi-query sql statement as a batch job, with multiple outputs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sql_query = \"SELECT TOP 2 specObjID, ra,dec, z FROM specobj; SELECT TOP 2 specobjid, petromag_u, petromag_r FROM specphoto;\"\n", + "job_alias = \"multi-query\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "out1 = FileOutput(name=\"query1.json\", output_type=OutputType.FILE_JSON, statement_indexes=1)\n", + "out2 = FileOutput(name=\"query2.json\", output_type=OutputType.FILE_JSON, statement_indexes=2)\n", + "out3 = FileOutput(name=\"query12.json\", output_type=OutputType.FILE_JSON, statement_indexes=[1,2])\n", + "\n", + "outputs = Outputs(out1, out2, out3) \n", + "# or\n", + "outputs = Outputs([out1, out2, out3])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "results_base_path = '/home/idies/workspace/Temporary/{}/scratch/sciqueryjobs/'.format(SciQuery.get_user().userName)\n", + "# or\n", + "sq.results_base_path = results_base_path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "jobid = sq.submit_query_job(sql_query=sql_query, outputs=outputs, results_base_path=results_base_path, job_alias=job_alias)\n", + "job = sq.wait_for_job(jobid)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "outputs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.outputs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.get_json_output(0)\n", + "# or\n", + "job.get_json_output(job.outputs[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.get_json_output(1)\n", + "# or\n", + "job.get_json_output(job.outputs[1])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.get_json_output(2)\n", + "# or\n", + "job.get_json_output(job.outputs[2])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8 (py38)", + "language": "python", + "name": "py38" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 076f2439d56337d0d13b7857fc2e3449f8c5b990 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Wed, 26 Oct 2022 15:34:12 -0400 Subject: [PATCH 76/79] Add write_job_id parameter in execute_query function. Updated setting of path in FileOutput. --- py3/SciServer/SciQuery.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/py3/SciServer/SciQuery.py b/py3/SciServer/SciQuery.py index 83cc690..66245fe 100644 --- a/py3/SciServer/SciQuery.py +++ b/py3/SciServer/SciQuery.py @@ -6,7 +6,7 @@ from collections.abc import Iterable from datetime import datetime import warnings -from pathlib import Path +from pathlib import PurePosixPath from typing import Union, List import time @@ -105,7 +105,7 @@ def __init__(self, if not name: raise NameError("Input parameter name cannot be empty or None") name = name.rstrip("/") - file_path = Path(name) + file_path = PurePosixPath(name) if name == file_path.name: # means no path included in 'name' input parameter self.file_base_path = None self.file = name @@ -113,7 +113,7 @@ def __init__(self, self.file_service_path = None else: if not name.startswith(Config.ComputeWorkDir): - file_path = Path(Config.ComputeWorkDir + name) # in case it is relative path + file_path = PurePosixPath(Config.ComputeWorkDir + name) # in case it is relative path self.file_base_path = str(file_path.parent) self.file_base_path = self.file_base_path if self.file_base_path.endswith("/") \ else self.file_base_path + "/" @@ -157,9 +157,9 @@ def build_file_base_path(top_volume: str = "Temporary", if user_volume: if not user_volume_owner_name: user_volume_owner_name = SciQuery.get_user().userName - path = str(Path(Config.ComputeWorkDir, top_volume, user_volume_owner_name, user_volume, relative_path)) + path = str(PurePosixPath(Config.ComputeWorkDir, top_volume, user_volume_owner_name, user_volume, relative_path)) else: - path = str(Path(Config.ComputeWorkDir, top_volume, relative_path)) + path = str(PurePosixPath(Config.ComputeWorkDir, top_volume, relative_path)) return path if path.endswith("/") else path + "/" @staticmethod @@ -440,7 +440,7 @@ def get_dataframe_from_output(self, output: Union[Output, int] = 0, result_index elif out.output_type == OutputType.DATABASE_TABLE: sq = SciQuery(rdb_compute_domain=out.rdb_compute_domain_name, database=out.database) query = f"select * from {out.table};" - df = sq.execute_query(query) + df = sq.execute_query(query, write_job_id=False) else: raise Exception(f"Output type {out.output_type} not supported") return df @@ -1119,7 +1119,8 @@ def execute_query(self, results_base_path: str = None, rdb_compute_domain: Union[str, int, dict, RDBComputeDomain] = None, job_alias: str = "", - file_service: str = None) -> pd.DataFrame: + file_service: str = None, + write_job_id = True) -> pd.DataFrame: """ Returns the query result (as a Pandas data frame) of a sql query submitted as a job to a relational database (RDB) compute domain. @@ -1140,6 +1141,10 @@ def execute_query(self, :param file_service: a File Service defines an available file system where query result sets can be written into. This parameter can be its name or identifier (string), or a dictionary defining a file service. If set to None, then the currently set value of file_service in the SciQuery object is internally used. + :param write_job_id: if True, the job id will be written on the screen, just before returning the result. + The job id won;t be written if write_job_id = False. + into. This parameter can be its name or identifier (string), or a dictionary defining a file service. + If set to None, then the currently set value of file_service in the SciQuery object is internally used. :return: Pandas data frame containing the result of the query. """ output = FileOutput("result1.json", OutputType.FILE_JSON, 1) @@ -1149,7 +1154,7 @@ def execute_query(self, results_base_path=results_base_path, job_alias=job_alias, file_service=file_service) - if self.verbose: + if write_job_id: print("Query was submitted as a job with id = " + job_id) job = self.wait_for_job(job_id, verbose=False) if job.status > 32: From ec85792cdfcdcc2bde390b1d146403caa404a388 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Tue, 25 Jun 2024 08:58:41 -0400 Subject: [PATCH 77/79] Updating documentation. --- README.md | 4 + docs_sphinx/SciServer.rst | 8 + docs_sphinx/conf.py | 9 +- py2/SciServer/Config.py | 2 +- py2/SciServer/__init__.py | 2 +- py2/setup.py | 2 +- py3/SciServer/Config.py | 2 +- py3/SciServer/SciQuery.py | 370 +++++++++++++++++++++++++------------- py3/SciServer/__init__.py | 7 +- py3/setup.py | 2 +- requirements.txt | 1 + 11 files changed, 277 insertions(+), 132 deletions(-) create mode 100644 requirements.txt diff --git a/README.md b/README.md index 6b7b5af..c23868c 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ Some SciServer tools you can access with this package: * [CasJobs](http://skyserver.sdss.org/CasJobs): Database storage and querying. +* SciQuery: Advanced Database storage and querying. + * [SciDrive](http://www.scidrive.org/): Drag-and-drop file storage and sharing. * [SkyServer](http://skyserver.sdss.org/): Access to the SDSS astronomical survey. @@ -37,6 +39,8 @@ Authors: Gerard Lemson, Manuchehr Taghizadeh-Popp. ## 3) Installation: +First, add required packages with `pip install -r requirements.txt` + There are 2 possibilities: automatic or manual installation. ### a) Automatic Installation and Update: diff --git a/docs_sphinx/SciServer.rst b/docs_sphinx/SciServer.rst index fa71e05..e42ad9e 100644 --- a/docs_sphinx/SciServer.rst +++ b/docs_sphinx/SciServer.rst @@ -14,6 +14,14 @@ SciServer.CasJobs module :undoc-members: :show-inheritance: +SciServer.SciQuery module +------------------------- + +.. automodule:: SciServer.SciQuery + :members: + :undoc-members: + :show-inheritance: + SciServer.Config module ----------------------- diff --git a/docs_sphinx/conf.py b/docs_sphinx/conf.py index 63600e8..381fc37 100644 --- a/docs_sphinx/conf.py +++ b/docs_sphinx/conf.py @@ -48,16 +48,16 @@ # General information about the project. project = 'SciScript-Python' -copyright = '2018, SciServer' +copyright = '2024, SciServer' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '2.1.0' +version = '2.2.0' # The full version, including alpha/beta/rc tags. -release = '2.1.0' +release = '2.2.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -182,6 +182,7 @@ # Output file base name for HTML help builder. htmlhelp_basename = 'sphinxdoc' +autoclass_content = 'both' # -- Options for LaTeX output --------------------------------------------- @@ -268,7 +269,7 @@ epub_title = '.' epub_author = 'Gerard Lemson, Manuchehr Taghizadeh-Popp' epub_publisher = 'Gerard Lemson, Manuchehr Taghizadeh-Popp' -epub_copyright = '2018, SciServer' +epub_copyright = '2024, SciServer' # The basename for the epub file. It defaults to the project name. #epub_basename = '.' diff --git a/py2/SciServer/Config.py b/py2/SciServer/Config.py index 3c78c03..c2b0c70 100644 --- a/py2/SciServer/Config.py +++ b/py2/SciServer/Config.py @@ -31,7 +31,7 @@ RacmApiURL = "https://apps.sciserver.org/racm" DataRelease = "DR14" KeystoneTokenPath = "/home/idies/keystone.token" #the path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment -version = "sciserver-v2.1.0" #sciserver release version +version = "sciserver-v2.2.0" #sciserver release version ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. def _load_config(filename): diff --git a/py2/SciServer/__init__.py b/py2/SciServer/__init__.py index bf3877c..79179af 100644 --- a/py2/SciServer/__init__.py +++ b/py2/SciServer/__init__.py @@ -41,7 +41,7 @@ **Authors**: Gerard Lemson , Manuchehr Taghizadeh-Popp -**Version**: sciserver-v2.1.0 +**Version**: sciserver-v2.2.0 """ diff --git a/py2/setup.py b/py2/setup.py index b61abb2..26e8abf 100644 --- a/py2/setup.py +++ b/py2/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages setup( name = "SciServer", - version = "2.1.0", + version = "2.2.0", packages = find_packages(), ) \ No newline at end of file diff --git a/py3/SciServer/Config.py b/py3/SciServer/Config.py index b241298..557c195 100644 --- a/py3/SciServer/Config.py +++ b/py3/SciServer/Config.py @@ -35,7 +35,7 @@ RacmApiURL = "https://apps.sciserver.org/racm" DataRelease = "DR15" KeystoneTokenPath = "/home/idies/keystone.token" #the path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment -version = "sciserver-v2.1.0" #sciserver release version +version = "sciserver-v2.2.0" #sciserver release version ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. ComputeUrl = "https://apps.sciserver.org/compute" SciqueryURL = "https://apps.sciserver.org/sciquery-api" diff --git a/py3/SciServer/SciQuery.py b/py3/SciServer/SciQuery.py index 66245fe..76edf39 100644 --- a/py3/SciServer/SciQuery.py +++ b/py3/SciServer/SciQuery.py @@ -35,14 +35,14 @@ def _get_file_service(file_service: str = None): class Output: - """ - Base class for output objects - """ + def __init__(self, name: str = "output.json", output_type: str = OutputType.FILE_JSON, statement_indexes: Union[int, List[int]] = 1): - + """ + Base class for output objects, including files or database tables. + """ if type(name) != str: raise TypeError("Invalid type for input parameter 'name'.") if type(output_type) != str: @@ -58,8 +58,9 @@ def set_statement_indexes(self, statement_indexes: Union[int, List[int]] = 1): """ Sets the index(es) of the sql statement(s) within the input query, whose result-set(s) is(are) going to be written into this Output. + :param statement_indexes: integer or list of integers, which are the indices (starting with 1) of the sql - statements within the input query, whose resultsets are going to be written into this Output. + statements within the input query, whose resultsets are going to be written into this Output. """ if not isinstance(statement_indexes, Iterable): statement_indexes = [statement_indexes] @@ -77,9 +78,6 @@ def __repr__(self): class FileOutput(Output): - """ - Defines the output of a database query into a file. - """ def __init__(self, name: str = "output.json", @@ -87,11 +85,13 @@ def __init__(self, statement_indexes: Union[int, List[int]] = 1, file_service: str = None): """ + Defines the output of a database query into a file. + :param name: name of the file (string), such as "result.json" :param output_type: type (string) of the file containing the query result(s) (e.g., "FILE_JSON"). - As set of possible values is given by the static members of class 'SciQuery.OutputTargetType' + As set of possible values is given by the static members of class 'SciQuery.OutputTargetType' :param statement_indexes: list of integers or integer. Each integer value denotes the index or position (>=1) - of the sql statements within the input query, whose resultset is going to be written into this OutputTarget + of the sql statements within the input query, whose resultset is going to be written into this OutputTarget :param file_service: string denoting name or identifier of file service where the output file is written into. """ if file_service: @@ -125,6 +125,9 @@ def __init__(self, super().__init__(name, output_type, statement_indexes) def get_path(self) -> str: + """ + Get file output path on SciServer's FileSystem + """ if self.path: return self.path else: @@ -134,7 +137,7 @@ def get_path(self) -> str: @classmethod def get_default(cls): """ - Gets an Output object filled with default values: JSON output file where only the 1st SQL statement of + Gets a feault FileOutput object filled with default values: JSON output file where only the 1st SQL statement of the query is written in it. """ return cls("result.json", OutputType.FILE_JSON, 1) @@ -145,6 +148,16 @@ def build_file_base_path(top_volume: str = "Temporary", user_volume_owner_name: str = "", relative_path: str = "sciqueryjobs", add_date_ending: bool = False) -> str: + """ + Builds the base path for an output file located in the SciServer filesystem. + + :param top_volume: name (str) of top volume in SciServer's filesystem, such as "Temporary" or "Storage". + :param user_volume: name (str) of user volume in SciServer's filesystem. + :param user_volume_owner_name: name (str) of user volume owner in SciServer's filesystem. + :param relative_path: relative path (str) after /// + :param add_date_ending: if True, then the relative path is built from the current date. + :return: path string + """ if not top_volume: raise NameError("Input parameter top_volume cannot be empty or None") @@ -164,6 +177,9 @@ def build_file_base_path(top_volume: str = "Temporary", @staticmethod def find_file_service(file_service: Union[str, dict] = None) -> dict: + """ + Gets FileService info + """ if isinstance(file_service, dict): file_service = file_service.get("identifier") return _get_file_service(file_service) @@ -177,9 +193,6 @@ def __repr__(self): class DatabaseTableOutput(Output): - """ - Defines the output of a database query into a database table - """ def __init__(self, table: str = "resultTable", @@ -189,15 +202,17 @@ def __init__(self, schema: str = ""): """ + Defines the output of a database query into a database table. + :param table: name of the output database table (string), such as "resultTable" :param database: name of the database (string) where the output table in created. If it is owned explicitly by - a user, then it should follow the pattern "mydb:" + a user, then it should follow the pattern "mydb:" :param statement_indexes: list of integers or integer. Each integer value denotes the index or position (>=1) :param rdb_compute_domain: name (string) of the relational database (RDB) compute domain that contains the - database, or object of class RDBComputeDomain corresponding to it. - Name of such domains available to the user is returned by the function Jobs.getRDBComputeDomainNames(). + database, or object of class RDBComputeDomain corresponding to it. + Name of such domains available to the user is returned by the function Jobs.getRDBComputeDomainNames(). :param schema: database schema (string) - of the sql statements within the input query, whose resultset is going to be written into this OutputTarget + of the sql statements within the input query, whose resultset is going to be written into this OutputTarget """ if type(table) != str or type(schema) != str: raise TypeError("Input parameter(s) 'table' or 'schema' should be of type string.") @@ -242,11 +257,11 @@ def __repr__(self): class Outputs(list): - """ - Contains a list of output objects, defining database query resultset outputs. - """ def __init__(self, *outputs): + """ + Contains a list of output objects, defining database query result outputs. + """ super().__init__() for output in outputs: outs = output if isinstance(output, Iterable) else [output] @@ -254,12 +269,18 @@ def __init__(self, *outputs): self.append(out) def append(self, obj): + """ + Appends an Output object to this list. + """ if isinstance(obj, Output): super().append(obj) else: raise NameError("Input object is not a subclass of the 'Output' class.") def get_target_list(self, file_base_path: str = None, file_service: str = None): + """ + Gets list of output targets. + """ targets = [] fs = FileOutput.find_file_service(file_service) for output in self: @@ -285,18 +306,21 @@ def get_target_list(self, file_base_path: str = None, file_service: str = None): @staticmethod def get_default(): + """ + Gets an OutputList with one element, consisting of a JSON FileOutput object. + """ return Outputs(FileOutput(name="result.json", output_type=OutputType.FILE_JSON, statement_indexes = [1])) class RDBJob: - """ - Contains the definition of a job consisting on a query run in a Relational Database (RDB) - """ + _JOB_STATUS_MAP = {1: "PENDING", 2: "QUEUED", 4: "ACCEPTED", 8: "STARTED", 16: "FINISHED", 32: "SUCCESS", 64: "ERROR", 128: "CANCELED"} def __init__(self, job): """ + Contains the definition of a job consisting on a query run in a Relational Database (RDB). + :param job: can be the job ID (string), or a dictionary containing all the attributes of an RDBJob object. """ if type(job) != dict: @@ -337,6 +361,9 @@ def __init__(self, job): self.get_job_status = self._get_job_status_string def get_metadata(self, result_format="pandas") -> pd.DataFrame: + """ + Gets this RDBJob's metadata as a Pandas DataFrame. + """ data = [] column_names = [] if result_format == "pandas": @@ -355,16 +382,28 @@ def get_metadata(self, result_format="pandas") -> pd.DataFrame: @staticmethod def get_job_status(status: int) -> str: + """ + Gets job status string from its integer representation. + """ return RDBJob._JOB_STATUS_MAP.get(status) @staticmethod def get_job(job_id: int): + """ + Gets RDBJob object from its Id. + """ return RDBJob(Jobs.getJobDescription(job_id)) def cancel(self): + """ + Cancels this RDBJob. + """ Jobs.cancelJob(self.id) def refresh(self): + """ + Refreshes metadata and info of this RDBJob. + """ self.__init__(Jobs.getJobDescription(self.id)) def _get_job_status_string(self) -> str: @@ -400,12 +439,18 @@ def _get_output_from_index(self, ind: int): return self.outputs[ind] def get_output_path(self, output: Union[Output, int] = 0) -> str: + """ + Gets output path on SciServer's filesystem, if output type is a file. + """ out = self._get_output_from_index(output) if isinstance(output, int) else output if out.output_type == OutputType.DATABASE_TABLE: raise TypeError("Output is not a file but a database") return out.get_path() def get_output_as_string(self, output: Union[Output, int, str] = None): + """ + Gets content of output file in SciServer's filesystem as a string. + """ if not isinstance(output, str): out = self._get_output_from_index(output) if isinstance(output, int) else output file_path = self.get_output_path(out) @@ -426,10 +471,16 @@ def get_output_as_string(self, output: Union[Output, int, str] = None): return data def get_json_output(self, output: Union[Output, int, str] = 0) -> dict: + """ + Gets content of output file in SciServer's filesystem as a dictionary. + """ data_dict = json.loads(self.get_output_as_string(output)) return data_dict.get("Result") def get_dataframe_from_output(self, output: Union[Output, int] = 0, result_index: int = 0) -> pd.DataFrame: + """ + Gets query output as a Pandas DatFrame. + """ out = self._get_output_from_index(output) if isinstance(output, int) else output if out.output_type == OutputType.FILE_JSON: result = self.get_json_output(out)[result_index] @@ -456,17 +507,16 @@ def __repr__(self): class Database: - """ - Defines a database context where users can run sql queries. - """ def __init__(self, rdb_compute_domain: Union[str, int, dict], database: Union[str, int, dict]): """ + Defines a database context where users can run sql queries. + :param rdb_compute_domain: Parameter that identifies the relation database domain or environment that - contains the database. Could be either its name (string), ID (integer), or a dictionary containing - the attributes of the domain. + contains the database. Could be either its name (string), ID (integer), or a dictionary containing + the attributes of the domain. :param database: defines the database. Can be either the database name (string), ID (integer), or a dictionary - containing all the attributes of an object of class Database. + containing all the attributes of an object of class Database. """ if type(database) not in [str, int, dict]: raise TypeError("Invalid type for input parameter 'database'.") @@ -502,6 +552,9 @@ def __init__(self, rdb_compute_domain: Union[str, int, dict], database: Union[st self.rdb_compute_domain_id = domain.get('id') def get_metadata(self) -> pd.DataFrame: + """ + Gets Database metadata. + """ data = [] column_names = ['database_name', 'database_description', 'database_vendor', 'database_id', 'rdb_compute_domain_name', 'rdb_compute_domain_id'] @@ -521,16 +574,14 @@ def __repr__(self): class RDBComputeDomain: - """ - Defines a domain or environment with databases that users are able to query. - """ def __init__(self, rdb_compute_domain: Union[str, int, dict]): """ Creates an instance of an RDBComputeDomain, which defines a domain or environment with databases that users. are able to query. + :param rdb_compute_domain: Parameter that identifies the domain. Could be either its name (string), - ID (integer), or a dictionary containing all the attributes of the domain. + ID (integer), or a dictionary containing all the attributes of the domain. """ if type(rdb_compute_domain) not in [str, int, dict]: raise TypeError("Invalid type for input parameter 'rdb_compute_domain'.") @@ -564,15 +615,19 @@ def __init__(self, rdb_compute_domain: Union[str, int, dict]): def get_database_names(self) -> list: """ - Gets a list of the names of databases in an RDBComputeDomain + Gets a list of the names of databases in an RDBComputeDomain. :return: list of database names (strings) :example: dbnames = SciQuery.get_database_names(rdbComputeDomainName); + .. seealso:: SciQuery.get_databases_metadata """ return [db.name for db in self.databases] def get_database(self, database: Union[str, int, dict, Database]) -> Database: + """ + Gets Database + """ if type(database) == str: dbs = [db for db in self.databases if db.name == database] elif type(database) == int: @@ -589,6 +644,9 @@ def get_database(self, database: Union[str, int, dict, Database]) -> Database: return dbs[0] def get_default_database(self) -> Database: + """ + Gets default database. + """ dbs = [db for db in self.databases if db.name == SciQuery.get_mydb_name()] if len(dbs) > 0: return dbs[0] @@ -598,6 +656,9 @@ def get_default_database(self) -> Database: raise Exception("No default database available.") def get_metadata(self, do_include_databases: bool = False) -> pd.DataFrame: + """ + Gets metadata of this RDBComputeDomain. + """ column_names = ['rdb_compute_domain_name', 'rdb_compute_domain_description', 'rdb_compute_domain_id'] data = [[self.name, self.description, self.id]] @@ -622,6 +683,7 @@ def get_databases_metadata(self) -> pd.DataFrame: Gets metadata of the databases in this RDBComputeDomain. :return: pandas dataframe with associated metadata. + .. seealso:: SciQuery.get_database_names """ dfs = [db.get_metadata() for db in self.databases] @@ -637,13 +699,14 @@ def __repr__(self): class RDBComputeDomains(list): - """ - Defines a list of RDBComputeDomains, which are domains or environments with databases that users are able to query. - """ + def __init__(self, rdb_compute_domains: Union[Iterable, RDBComputeDomain]): """ + Defines a list of RDBComputeDomains, which are domains or environments with databases that users are able to + query. + :param rdb_compute_domains: Parameter that identifies a list of RDBComputeDomain objects. - Could be either single RDBComputeDomain object, or an iterable containing multiple RDBComputeDomain objects. + Could be either single RDBComputeDomain object, or an iterable containing multiple RDBComputeDomain objects. """ super().__init__() domains = rdb_compute_domains if isinstance(rdb_compute_domains, Iterable) else [rdb_compute_domains] @@ -654,6 +717,9 @@ def __init__(self, rdb_compute_domains: Union[Iterable, RDBComputeDomain]): raise NameError("Input object is not of class RDBComputeDomain.") def get_rdb_compute_domain(self, rdb_compute_domain: Union[str, int, dict, RDBComputeDomain]) -> RDBComputeDomain: + """ + Gets RDBComputeDomain from this list + """ if type(rdb_compute_domain) == str: domains = [d for d in self if d.name == rdb_compute_domain] elif type(rdb_compute_domain) == int: @@ -671,6 +737,9 @@ def get_rdb_compute_domain(self, rdb_compute_domain: Union[str, int, dict, RDBCo return domains[0] def get_default_rdb_compute_domain(self) -> RDBComputeDomain: + """ + Gets default RDBComputeDomain from this list. + """ domains = [domain for domain in self if len(domain.databases) > 0] if len(domains) > 0: doms = [dom for dom in domains if dom.get_default_database().name == SciQuery.get_mydb_name()] @@ -682,9 +751,7 @@ def get_default_rdb_compute_domain(self) -> RDBComputeDomain: class SciQuery: - """ - Instance of the SciQuery app for querying relational databases. - """ + def __init__(self, rdb_compute_domain: Union[str, int, dict, RDBComputeDomain] = None, database: Union[str, int, dict, Database] = None, @@ -696,29 +763,30 @@ def __init__(self, poll_time: float = 1.0 ): """ - Creates instance of SciQuery class. + Created an instance of the SciQuery app for querying relational databases. :param rdb_compute_domain: defines a domain or environment of multiple databases where users can run queries. - Can be either the domain's name (string), ID (integer), an object of class RDBComputeDomain, or a dictionary - containing all the attributes of an object of class RDBComputeDomain. If set to None, a default value will be - assigned to it. + Can be either the domain's name (string), ID (integer), an object of class RDBComputeDomain, or a dictionary + containing all the attributes of an object of class RDBComputeDomain. If set to None, a default value will + be assigned to it. :param database: defines the database where the queries are executed in. - Can be either the database name (string), ID (integer), an object of class Database, or a dictionary containing - all the attributes of an object of class Database. If set to None, a default value will be assigned to it. + Can be either the database name (string), ID (integer), an object of class Database, or a dictionary + containing all the attributes of an object of class Database. If set to None, a default value will be + assigned to it. :param file_service: a File Service defines an available file system where query result sets can be written - into. This parameter can be it name or identifier (string), or a dictionary defining a file service. - If set to None, a default value will be assigned to it. + into. This parameter can be it name or identifier (string), or a dictionary defining a file service. + If set to None, a default value will be assigned to it. :param results_base_path: base path (string) of the directory where the query results are written into. - Can be constructed by using FileOutput.build_file_base_path(). If set to None, a default value will be assigned - to it at the moment of running a sql query. + Can be constructed by using FileOutput.build_file_base_path(). If set to None, a default value will be + assigned to it at the moment of running a sql query. :param outputs: Defines the query(ies) output(s). Can be an object derived from the Output base class (such as - FileOutput or DatabaseTableOutput), or a list of those. If set to None, a default value (json file output) - will be assigned to it. + FileOutput or DatabaseTableOutput), or a list of those. If set to None, a default value (json file output) + will be assigned to it. :param verbose: Boolean parameter. If True, warning messages will be written in case of errors, in the case when - the hard_fail parameter is set to False. If False, nothing will be written. + the hard_fail parameter is set to False. If False, nothing will be written. :param hard_fail: Boolean parameter. If True, exceptions will be raised in case of errors during instantiation. - If False, then no exceptions are raised, and warnings might be showed instead - (depending on the value of the verbose parameter). + If False, then no exceptions are raised, and warnings might be showed instead + (depending on the value of the verbose parameter). :param poll_time: time (float) in seconds between consecutive requests for updates in the jobs status. """ @@ -737,6 +805,9 @@ def __init__(self, @staticmethod def get_token() -> str: + """ + Gets user's auth token. + """ token = Authentication.getToken() if token is None or token == "": raise Exception("User not has not logged into SciServer. Use 'Authentication.login'.") @@ -744,6 +815,9 @@ def get_token() -> str: @staticmethod def get_user() -> Authentication.KeystoneUser: + """ + Gets logged Keystone user info. + """ token = SciQuery.get_token() user = Authentication.getKeystoneUserWithToken(token) user.token = token @@ -763,24 +837,26 @@ def set(self, Sets or refreshes the parameters in the SciQuery object, all at once. :param rdb_compute_domain: defines a domain or environment of multiple databases where users can run queries. - Can be either the domain's name (string), ID (integer), an object of class RDBComputeDomain, or a dictionary - containing all the attributes of an object of class RDBComputeDomain. If set to None, the current value - is refreshed. + Can be either the domain's name (string), ID (integer), an object of class RDBComputeDomain, or a dictionary + containing all the attributes of an object of class RDBComputeDomain. If set to None, the current value + is refreshed. :param database: defines the database where the queries are executed in. - Can be either the database name (string), ID (integer), an object of class Database, or a dictionary containing - all the attributes of an object of class Database. If set to None, the current value is refreshed. + Can be either the database name (string), ID (integer), an object of class Database, or a dictionary + containing all the attributes of an object of class Database. If set to None, the current value is + refreshed. :param file_service: a File Service defines an available file system where query result sets can be written - into. This parameter can be it name or identifier (string), or a dictionary defining a file service. - If set to None, the current value is refreshed. + into. This parameter can be it name or identifier (string), or a dictionary defining a file service. + If set to None, the current value is refreshed. :param results_base_path: base path (string) of the directory where the query results are written into. - Can be constructed by using FileOutput.build_file_base_path(). + Can be constructed by using FileOutput.build_file_base_path(). :param outputs: Defines the query(ies) output(s). Can be a list of Output objects, - or a single object of class Outputs. If set to None, a default value (json file output) will be assigned to it. + or a single object of class Outputs. If set to None, a default value (json file output) will be assigned to + it. :param verbose: Boolean parameter. If True, warning messages will be written in case of errors, in the case when - the hard_fail parameter is set to False. If False, nothing will be written. + the hard_fail parameter is set to False. If False, nothing will be written. :param hard_fail: Boolean parameter. If True, exceptions will be raised in case of errors during instantiation. - If False, then no exceptions are raised, and warnings might be showed instead - (depending on the value of the verbose parameter). + If False, then no exceptions are raised, and warnings might be showed instead + (depending on the value of the verbose parameter). :param poll_time: time (float) in seconds between consecutive requests for updates in the jobs status. """ @@ -846,10 +922,16 @@ def _handle_exception(self, exception: Exception, extra_message: str = ""): warnings.warn(message) def refresh(self): + """ + Refreshes SciQuery instance. + """ self.set(verbose=self.verbose, hard_fail=self.hard_fail) @staticmethod def get_mydb_name(owner_name: str = None) -> str: + """ + Returns name of mydb based on the owner's name. + """ if not owner_name: owner_name = SciQuery.get_user().userName return "mydb:" + owner_name @@ -860,10 +942,10 @@ def get_rdb_compute_domains(result_format: str = 'class') -> RDBComputeDomains: Gets a list of all registered Relational Database (RDB) compute domains that the user has access to. :param result_format: If set to "class", then the returned value will be of class RDBComputeDomains. - If set to "dict", then the return value will be a list of dictionaries, each of them containing the attributes - of an RDBComputeDomain object. - :return: an object of class RDBComputeDomains, or a list of dictionaries, each of them containing the attributes - of an RDBComputeDomain object. + If set to "dict", then the return value will be a list of dictionaries, each of them containing the + attributes of an RDBComputeDomain object. + :return: an object of class RDBComputeDomains, or a list of dictionaries, each of them containing the + attributes of an RDBComputeDomain object. """ token = SciQuery.get_user().token @@ -890,6 +972,9 @@ def get_rdb_compute_domains(result_format: str = 'class') -> RDBComputeDomains: @property def rdb_compute_domains(self) -> RDBComputeDomains: + """ + Property defining the RDBComputeDomains available in this SciQuery instance. + """ return self._rdb_compute_domains @rdb_compute_domains.setter @@ -902,6 +987,9 @@ def rdb_compute_domains(self, rdb_compute_domains: RDBComputeDomains): @property def rdb_compute_domain(self) -> RDBComputeDomain: + """ + Property defining the selected RDBComputeDomain in this SciQuery instance. + """ return self._rdb_compute_domain @rdb_compute_domain.setter @@ -917,9 +1005,9 @@ def get_rdb_compute_domain(self, rdb_compute_domain: Union[str, int, dict, RDBCo which is set in the SciQuery instance. :param rdb_compute_domain: defines a domain or environment of multiple databases where users can run queries. - Can be either the domain's name (string), ID (integer), an object of class RDBComputeDomain, or a dictionary - containing all the attributes of an object of class RDBComputeDomain. If set to None, then the currently set - value of rdb_compute_domain in the SciQuery object is returned. + Can be either the domain's name (string), ID (integer), an object of class RDBComputeDomain, or a dictionary + containing all the attributes of an object of class RDBComputeDomain. If set to None, then the currently set + value of rdb_compute_domain in the SciQuery object is returned. :return: Object of class RDBComputeDomain. """ if rdb_compute_domain is None: @@ -927,12 +1015,18 @@ def get_rdb_compute_domain(self, rdb_compute_domain: Union[str, int, dict, RDBCo return self.rdb_compute_domains.get_rdb_compute_domain(rdb_compute_domain) def get_default_rdb_compute_domain(self): + """ + Gets default RDBComputeDomain. + """ return self.rdb_compute_domains.get_default_rdb_compute_domain() # database --------------------------------------------------- @property def database(self) -> Database: + """ + Property defining the selected database in this SciQuery instance. + """ return self._database @database.setter @@ -948,15 +1042,15 @@ def get_database(self, Returns an object of class Database, either defined by the input name or identifiers, or that which is set in the SciQuery instance. - :param database: identifies the database, which this function returns as an object of class Database. - Can be either the database name (string), ID (integer), an object of class Database, or a dictionary containing - all the attributes of an object of class Database. If set to None, then the currently set value of database in - the SciQuery object is returned. + Can be either the database name (string), ID (integer), an object of class Database, or a dictionary + containing all the attributes of an object of class Database. If set to None, then the currently set value + of database in the SciQuery object is returned. :param rdb_compute_domain: defines a domain or environment of multiple databases where users can run queries, - and that contains the database. Can be either the domain's name (string), ID (integer), an object of class - RDBComputeDomain, or a dictionary containing all the attributes of an object of class RDBComputeDomain. - If set to None, then the currently set value of rdb_compute_domain in the SciQuery object is internally used. + and that contains the database. Can be either the domain's name (string), ID (integer), an object of class + RDBComputeDomain, or a dictionary containing all the attributes of an object of class RDBComputeDomain. + If set to None, then the currently set value of rdb_compute_domain in the SciQuery object is internally + used. :return: Object of class Database """ if database is None: @@ -964,6 +1058,9 @@ def get_database(self, return self.get_rdb_compute_domain(rdb_compute_domain).get_database(database) def get_default_database(self, rdb_compute_domain: Union[str, int, dict, RDBComputeDomain] = None) -> Database: + """ + Gets default database. + """ domain = self.get_default_rdb_compute_domain() if rdb_compute_domain is None \ else self.get_rdb_compute_domain(rdb_compute_domain) return domain.get_default_database() @@ -972,6 +1069,9 @@ def get_default_database(self, rdb_compute_domain: Union[str, int, dict, RDBComp @property def file_service(self) -> dict: + """ + Property defining the selected FileService available in this SciQuery instance. + """ return self._file_service @file_service.setter @@ -983,10 +1083,10 @@ def file_service(self, file_service: Union[str, dict]): def get_file_service(self, file_service: Union[str, dict] = None) -> dict: """ Returns the definition of a file service as a dictionary, either defined by the input name or identifiers, - or that which is set in the SciQuery instance. + or that which is set in the SciQuery instance. :param file_service: name or identifier (string) of a file service, or the dictionary with its definition. - If set to None, then the currently set value of file_service in the SciQuery object is returned. + If set to None, then the currently set value of file_service in the SciQuery object is returned. :return: dictionary with the definition of a file service. """ if file_service is None: @@ -994,12 +1094,18 @@ def get_file_service(self, file_service: Union[str, dict] = None) -> dict: return FileOutput.find_file_service(file_service) def get_default_file_service(self) -> dict: + """ + Gets default FiLeService + """ return FileOutput.find_file_service() # results_base_path --------------------------------------------------- @property def results_base_path(self) -> str: + """ + Property defining the file output base path in SciServer's filesystem for this SciQuery instance. + """ return self._results_base_path @results_base_path.setter @@ -1009,15 +1115,24 @@ def results_base_path(self, results_base_path: str): self._results_base_path = results_base_path def get_results_base_path(self) -> str: + """ + Gets results base path on SciServer's filesystem. + """ return self._results_base_path def get_default_results_base_path(self, add_date_ending=True) -> str: + """ + Gets default results base path on SciServer's filesystem. + """ return FileOutput.build_file_base_path(add_date_ending=add_date_ending) # outputs ------------------------------------------------------------- @property def outputs(self) -> Outputs: + """ + Property defining a list of query result Output objects. + """ return self._outputs @outputs.setter @@ -1032,7 +1147,7 @@ def get_outputs(self, outputs: Union[Outputs, Output] = None) -> Outputs: which is set in the SciQuery instance. :param outputs: object of class Outputs, or iterable of output objects. If set to None, then the currently - set value of outputs in the SciQuery object is returned. + set value of outputs in the SciQuery object is returned. :return: object of class Outputs. """ if outputs is None: @@ -1040,6 +1155,9 @@ def get_outputs(self, outputs: Union[Outputs, Output] = None) -> Outputs: return Outputs(outputs) def get_default_outputs(self) -> Outputs: + """ + Gets default query outputs. + """ return Outputs.get_default() # --------------------------------------------------------------------------------------------- @@ -1059,22 +1177,24 @@ def submit_query_job(self, :param sql_query: sql query (string) :param database: defines the database where the sql query is executed in. - Can be either the database name (string), ID (integer), an object of class Database, or a dictionary containing - all the attributes of an object of class Database. If set to None, then the current value of the database field - in this SciQuery instance will be used. + Can be either the database name (string), ID (integer), an object of class Database, or a dictionary + containing all the attributes of an object of class Database. If set to None, then the current value of + the database field in this SciQuery instance will be used. :param outputs: Defines the query(ies) output(s). Can be an object derived from the Output base class (such as - FileOutput or DatabaseTableOutput), or a list of those. If set to None, then the current value of the outputs - field in this SciQuery instance will be used. + FileOutput or DatabaseTableOutput), or a list of those. If set to None, then the current value of the + outputs field in this SciQuery instance will be used. :param results_base_path: full path to results folder (string) where query output tables are written into. - E.g.: /home/idies/workspace/rootVOlume/username/userVolume/jobsFolder . If set to None, then its current value - in this SciQuery instance will be used. If that value is None, then a default folder will be set automatically. + E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder . If set to None, then its current + value in this SciQuery instance will be used. If that value is None, then a default folder will be set + automatically. :param rdb_compute_domain: defines a domain or environment of multiple databases where users can run queries, - and that contains the database. Can be either the domain's name (string), ID (integer), an object of class - RDBComputeDomain, or a dictionary containing all the attributes of an object of class RDBComputeDomain. - If set to None, then the currently set value of rdb_compute_domain in the SciQuery object is internally used. + and that contains the database. Can be either the domain's name (string), ID (integer), an object of class + RDBComputeDomain, or a dictionary containing all the attributes of an object of class RDBComputeDomain. + If set to None, then the currently set value of rdb_compute_domain in the SciQuery object is internally + used. :param file_service: a File Service defines an available file system where query result sets can be written - into. This parameter can be its name or identifier (string), or a dictionary defining a file service. - If set to None, then the currently set value of file_service in the SciQuery object is internally used. + into. This parameter can be its name or identifier (string), or a dictionary defining a file service. + If set to None, then the currently set value of file_service in the SciQuery object is internally used. :param job_alias: alias (string) of job, defined by the user. :return: the ID (string) that labels the job. """ @@ -1127,24 +1247,26 @@ def execute_query(self, :param sql_query: sql query (string) :param database: defines the database where the sql query is executed in. - Can be either the database name (string), ID (integer), an object of class Database, or a dictionary containing - all the attributes of an object of class Database. If set to None, then the current value of the database field - in this SciQuery instance will be used. + Can be either the database name (string), ID (integer), an object of class Database, or a dictionary + containing all the attributes of an object of class Database. If set to None, then the current value of + the database field in this SciQuery instance will be used. :param results_base_path: full path to results folder (string) where query output tables are written into. - E.g.: /home/idies/workspace/rootVOlume/username/userVolume/jobsFolder . If set to None, then its current value in - this SciQuery instance will be used. If that value is None, then a default folder will be set automatically. + E.g.: /home/idies/workspace/rootVOlume/username/userVolume/jobsFolder . If set to None, then its current + value in this SciQuery instance will be used. If that value is None, then a default folder will be set + automatically. :param rdb_compute_domain: defines a domain or environment of multiple databases where users can run queries, - and that contains the database. Can be either the domain's name (string), ID (integer), an object of class - RDBComputeDomain, or a dictionary containing all the attributes of an object of class RDBComputeDomain. - If set to None, then the currently set value of rdb_compute_domain in the SciQuery object is internally used. + and that contains the database. Can be either the domain's name (string), ID (integer), an object of class + RDBComputeDomain, or a dictionary containing all the attributes of an object of class RDBComputeDomain. + If set to None, then the currently set value of rdb_compute_domain in the SciQuery object is internally used. :param job_alias: alias (string) of job, defined by the user. :param file_service: a File Service defines an available file system where query result sets can be written - into. This parameter can be its name or identifier (string), or a dictionary defining a file service. - If set to None, then the currently set value of file_service in the SciQuery object is internally used. + into. This parameter can be its name or identifier (string), or a dictionary defining a file service. + If set to None, then the currently set value of file_service in the SciQuery object is internally used. :param write_job_id: if True, the job id will be written on the screen, just before returning the result. - The job id won;t be written if write_job_id = False. - into. This parameter can be its name or identifier (string), or a dictionary defining a file service. - If set to None, then the currently set value of file_service in the SciQuery object is internally used. + The job id won;t be written if write_job_id = False. + into. This parameter can be its name or identifier (string), or a dictionary defining a file service. + If set to None, then the currently set value of file_service in the SciQuery object is internally used. + :return: Pandas data frame containing the result of the query. """ output = FileOutput("result1.json", OutputType.FILE_JSON, 1) @@ -1175,14 +1297,14 @@ def get_jobs_list(top=5, open=None, start=None, end=None, result_format="pandas" :param top: top number of jobs (integer) returned. If top=None, then all jobs are returned. :param open: If set to 'True', then only returns jobs that have not finished executing and wrapped up - (status <= FINISHED). If set to 'False' then only returns jobs that are still running. If set to 'None', - then returns both finished and unfinished jobs. + (status <= FINISHED). If set to 'False' then only returns jobs that are still running. If set to 'None', + then returns both finished and unfinished jobs. :param start: The earliest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. - If set to 'None', then there is no lower bound on date. + If set to 'None', then there is no lower bound on date. :param end: The latest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. - If set to 'None', then there is no upper bound on date. + If set to 'None', then there is no upper bound on date. :param result_format: string defining the return format. "pandas" for a pandas dataframe and "list" - for a list of RDBJob objects. + for a list of RDBJob objects. :return: pandas dataframe, or list of RDBJob objects or, each containing the definition of a submitted job. """ job_dict_list = Jobs.getJobsList(top=top, open=open, start=start, end=end, type='rdb') @@ -1236,11 +1358,11 @@ def wait_for_job(self, job_id, verbose=False): :param job_id: id of job (integer) :param verbose: if True, will print "wait" messages on the screen while the job is still running. If False, it - will suppress the printing of messages on the screen. + will suppress the printing of messages on the screen. :return: After the job is finished, returns an object of class RDBJob, containing the job definition. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that - purpose). - Throws an exception if the HTTP request to the JOBM API returns an error. + purpose). + Throws an exception if the HTTP request to the JOBM API returns an error. """ t = max(0.5, self.poll_time) wait_message = "Waiting" @@ -1261,7 +1383,7 @@ def get_rdb_compute_domains_metadata(self, do_include_databases=False): Gets metadata related to all relational database (RDB) compute domains (RDBComputeDomains) available. :param do_include_databases: Boolean parameter. If True, it will return metadata related to all available - databases in each RDBComputeDomain as well. + databases in each RDBComputeDomain as well. :return: pandas dataframe containing associated metadata. """ dfs = [] @@ -1281,6 +1403,9 @@ def get_rdb_compute_domain_names(self): return [d.name for d in self.rdb_compute_domains] def get_rdb_compute_domain_metadata(self, rdb_compute_domain=None, do_include_databases=False): + """ + Gets metadata of an rdb_compute_domain. + """ return self.get_rdb_compute_domain(rdb_compute_domain).get_metadata(do_include_databases) def get_databases_metadata(self, rdb_compute_domain=None): @@ -1295,13 +1420,16 @@ def get_databases_metadata(self, rdb_compute_domain=None): return rdb_compute_domain.get_databases_metadata() def get_database_metadata(self, database=None, rdb_compute_domain=None): + """ + Gets database metadata. + """ rdb_compute_domain = self.get_rdb_compute_domain(rdb_compute_domain) database = self.get_database(database, rdb_compute_domain) return database.get_metadata() def get_database_names(self, rdb_compute_domain=None): """ - Gets a list of the names of databases in an RDBComputeDomain + Gets a list of the names of the databases in this RDBComputeDomain. :return: array of database names (strings) """ diff --git a/py3/SciServer/__init__.py b/py3/SciServer/__init__.py index bf3877c..87f1fcb 100644 --- a/py3/SciServer/__init__.py +++ b/py3/SciServer/__init__.py @@ -9,7 +9,10 @@ \t\tAlthough some tools accept anonymous access, you can use Authentication.login to login and access the tools and your own data and environment (after registering in the Login Portal). If you are running this package in a Jupyter Notebook in the SciServer-Compute environment, the use of Authentication.login is not necessary since it's done automatically. * `CasJobs `_: Database storage and querying. -\t\tYou can have access big databases, as well as save your data tables in your own database called 'MyDB'. The user can run synchronous or asynchronous SQL queries and get the result back as an R data-frame (using CasJobs.executeQuery or CasJobs.submitJob, respectively). Uploading of CSV files or R data-frames into a database table can be done using CasJobs.uploadCSVToTable and CasJobs.uploadDataFrameToTable, respectively. +\t\tYou can have access big databases, as well as saving your data tables in your own database called 'MyDB'. The user can run synchronous or asynchronous SQL queries and get the result back as a Pandas data-frame (using CasJobs.executeQuery or CasJobs.submitJob, respectively). Uploading of CSV files or data-frames into a database table can be done using CasJobs.uploadCSVToTable and CasJobs.uploadDataFrameToTable, respectively. + +* `SciQuery`: Advanced database storage and querying. +\t\tSciQuery expands the capabilities of CasJobs, as it is designed to work with databases loaded in several database management systems beyond MS SQL Server, such as PostgreSQL and MySQL. Query result tables can also be directly written as JSON or CSV files in the SciServer filesystem. * `SciDrive `_: Drag-and-drop file storage and sharing. \t\tYou can create directories in SciDrive using SciDrive.createContainer, upload a file to SciDrive using SciDrive.upload, and share its URL with your collaborators by using SciDrive.publicUrl. @@ -41,7 +44,7 @@ **Authors**: Gerard Lemson , Manuchehr Taghizadeh-Popp -**Version**: sciserver-v2.1.0 +**Version**: sciserver-v2.2.0 """ diff --git a/py3/setup.py b/py3/setup.py index b61abb2..26e8abf 100644 --- a/py3/setup.py +++ b/py3/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages setup( name = "SciServer", - version = "2.1.0", + version = "2.2.0", packages = find_packages(), ) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..cb22f83 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +cachetools \ No newline at end of file From 5dca376a91954b22b04ecfe8a96f53be92ef50fa Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Tue, 2 Jul 2024 08:27:08 -0400 Subject: [PATCH 78/79] Adding API documentation link --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c23868c..aefe442 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # SciScript-Python -## Python libraries for Jupyter Notebooks +## SciServer Python libraries for Jupyter Notebooks This python package provides functions for quick access to [SciServer](http://www.sciserver.org) APIs (web services) and tools. [SciServer](http://www.sciserver.org) provides a new online framework for data-intensive scientifc computing in the cloud, @@ -24,6 +24,8 @@ Some SciServer tools you can access with this package: * [Files](http://apps.sciserver.org/dashboard/files): Interact with the SciServer file system. +[API Documentation](https://www.sciserver.org/docs/sciscript-python/) + Maintainer: Manuchehr Taghizadeh-Popp. Authors: Gerard Lemson, Manuchehr Taghizadeh-Popp. From 2081fc4cf04a23edfde146997264cc37e26ea3db Mon Sep 17 00:00:00 2001 From: Manuchehr Taghizadeh-Popp Date: Wed, 2 Oct 2024 08:43:07 -0400 Subject: [PATCH 79/79] Fix URL in SkyServer module --- py3/SciServer/SkyServer.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/py3/SciServer/SkyServer.py b/py3/SciServer/SkyServer.py index ca20934..ad05c27 100644 --- a/py3/SciServer/SkyServer.py +++ b/py3/SciServer/SkyServer.py @@ -36,9 +36,9 @@ def sqlSearch(sql, dataRelease=None): url = url + 'cmd=' + sql + '&' if Config.isSciServerComputeEnvironment(): - url = url + "TaskName=Compute.SciScript-Python.SkyServer.sqlSearch&" + url = url + "TaskName=Compute.SciScript-Python.SkyServer.sqlSearch" else: - url = url + "TaskName=SciScript-Python.SkyServer.sqlSearch&" + url = url + "TaskName=SciScript-Python.SkyServer.sqlSearch" #url = urllib.quote_plus(url) acceptHeader = "text/plain" @@ -113,9 +113,9 @@ def getJpegImgCutout(ra, dec, scale=0.7, width=512, height=512, opt="", query="" url = url + 'query=' + query + '&' if Config.isSciServerComputeEnvironment(): - url = url + "TaskName=Compute.SciScript-Python.SkyServer.getJpegImgCutout&" + url = url + "TaskName=Compute.SciScript-Python.SkyServer.getJpegImgCutout" else: - url = url + "TaskName=SciScript-Python.SkyServer.getJpegImgCutout&" + url = url + "TaskName=SciScript-Python.SkyServer.getJpegImgCutout" #url = urllib.quote_plus(url) acceptHeader = "text/plain" @@ -171,9 +171,9 @@ def radialSearch(ra, dec, radius=1, coordType="equatorial", whichPhotometry="opt url = url + 'limit=' + limit + '&' if Config.isSciServerComputeEnvironment(): - url = url + "TaskName=Compute.SciScript-Python.SkyServer.radialSearch&" + url = url + "TaskName=Compute.SciScript-Python.SkyServer.radialSearch" else: - url = url + "TaskName=SciScript-Python.SkyServer.radialSearch&" + url = url + "TaskName=SciScript-Python.SkyServer.radialSearch" #url = urllib.quote_plus(url) acceptHeader = "text/plain" @@ -230,9 +230,9 @@ def rectangularSearch(min_ra, max_ra, min_dec, max_dec, coordType="equatorial", url = url + 'limit=' + limit + '&' if Config.isSciServerComputeEnvironment(): - url = url + "TaskName=Compute.SciScript-Python.SkyServer.rectangularSearch&" + url = url + "TaskName=Compute.SciScript-Python.SkyServer.rectangularSearch" else: - url = url + "TaskName=SciScript-Python.SkyServer.rectangularSearch&" + url = url + "TaskName=SciScript-Python.SkyServer.rectangularSearch" #url = urllib.quote_plus(url) acceptHeader = "text/plain" @@ -318,9 +318,9 @@ def objectSearch(objId=None, specObjId=None, apogee_id=None, apstar_id=None, ra= url = url + 'obj=' + str(obj) + '&'; if Config.isSciServerComputeEnvironment(): - url = url + "TaskName=Compute.SciScript-Python.SkyServer.objectSearch&" + url = url + "TaskName=Compute.SciScript-Python.SkyServer.objectSearch" else: - url = url + "TaskName=SciScript-Python.SkyServer.objectSearch&" + url = url + "TaskName=SciScript-Python.SkyServer.objectSearch" #url = urllib.quote_plus(url) acceptHeader = "text/plain"