From f69f7ab8d79da65ed77dab96b4fbb63d1e59ccac Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Thu, 28 Feb 2019 12:59:48 -0500 Subject: [PATCH 1/4] add 2.0.13 tags --- DESCRIPTION | 4 ++-- R/Config.r | 4 ++-- man/SciServer-package.Rd | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 7c4ae96..939cfd4 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,10 +1,10 @@ Package: SciServer Type: Package Title: Package for accessing SciServer tools. -Version: 2.0.0 +Version: 2.0.13 Author: Manuchehr Taghizadeh-Popp, Gerard Lemson Maintainer: Manuchehr Taghizadeh-Popp -Description: Provides access to CasJobs, SciDrive, Authentication, and SkyServer +Description: Provides access to CasJobs, SciDrive, Authentication, Jobs, FileService, SkyQuery and SkyServer tools in SciServer. License: TBD Imports: diff --git a/R/Config.r b/R/Config.r index f36e640..dac87c1 100644 --- a/R/Config.r +++ b/R/Config.r @@ -12,11 +12,11 @@ Config.SkyQueryUrl = 'https://voservices.net/skyquery/Api/V1' #Config.SkyServerWSurl: defines the base URL of the SkyServer web service API Config.SkyServerWSurl = 'https://skyserver.sdss.org' #Config.DataRelease: defines the SDSS data release, to be used to build the full SkyServer API url along with Config.SkyServerWSurl -Config.DataRelease = 'DR14' # SDSS data release. E.g., DR13 +Config.DataRelease = 'DR15' # SDSS data release. E.g., DR13 #Config.KeystoneTokenPath: defines the local path (string) to the file containing the user's authentication token in the SciServer-Compute environment Config.KeystoneTokenFilePath = "/home/idies/keystone.token" #this path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment #Config.version: defines the SciServer release tag, to which this package belongs -Config.Version = "sciserver-v2.0.0" #sciserver release version +Config.Version = "sciserver-v2.0.13" #sciserver release version #Config.ComputeJobDirectoryFile: defines the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. Config.ComputeJobDirectoryFile = "/home/idies/jobs.path" #Config.RacmApiURL: defines the base URL of the multiple APIs in RACM diff --git a/man/SciServer-package.Rd b/man/SciServer-package.Rd index c617b87..1096261 100644 --- a/man/SciServer-package.Rd +++ b/man/SciServer-package.Rd @@ -34,7 +34,7 @@ Some SciServer tools you can access with this package: \tabular{ll}{ Package: \tab SciServer\cr Type: \tab Package\cr -Version: \tab 2.0.0\cr +Version: \tab 2.0.13\cr }} \author{ Manuchehr Taghizadeh-Popp, Gerard Lemson \cr From 1c10b77a16a06e12b8899534653243687d14d1c2 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Fri, 26 Apr 2019 11:24:44 -0400 Subject: [PATCH 2/4] added quick docker jobs endpoint --- Examples/Examples_SciScript-R.ipynb | 84 ++++++++++++----------------- NAMESPACE | 2 +- R/Jobs.r | 35 ++++++++++++ man/Jobs.getDockerJobsListQuick.Rd | 33 ++++++++++++ man/Jobs.getJobsList.Rd | 2 +- 5 files changed, 104 insertions(+), 52 deletions(-) create mode 100644 man/Jobs.getDockerJobsListQuick.Rd diff --git a/Examples/Examples_SciScript-R.ipynb b/Examples/Examples_SciScript-R.ipynb index cde4b85..5dc6155 100644 --- a/Examples/Examples_SciScript-R.ipynb +++ b/Examples/Examples_SciScript-R.ipynb @@ -22,14 +22,10 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# *******************************************************************************************************\n", - "# Authentication section\n", - "# *******************************************************************************************************" + "# Authentication section" ] }, { @@ -96,14 +92,10 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# *******************************************************************************************************\n", - "# LoginPortal section:\n", - "# *******************************************************************************************************" + "# LoginPortal section:" ] }, { @@ -169,14 +161,10 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# *******************************************************************************************************\n", - "# CasJobs section:\n", - "# *******************************************************************************************************" + "# CasJobs section:" ] }, { @@ -373,14 +361,10 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# *******************************************************************************************************\n", - "# SkyServer section:\n", - "# *******************************************************************************************************" + "# SkyServer section:" ] }, { @@ -468,14 +452,10 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# *******************************************************************************************************\n", - "# SciDrive section:\n", - "# *******************************************************************************************************" + "# SciDrive section:" ] }, { @@ -623,14 +603,10 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# *******************************************************************************************************\n", - "# SkyQuery section:\n", - "# *******************************************************************************************************" + "# SkyQuery section:" ] }, { @@ -868,14 +844,10 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# *******************************************************************************************************\n", - "# Files section:\n", - "# *******************************************************************************************************" + "# Files section:" ] }, { @@ -1169,14 +1141,10 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# *******************************************************************************************************\n", - "# Jobs section:\n", - "# *******************************************************************************************************" + "# Jobs section:" ] }, { @@ -1442,7 +1410,23 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "# get list of jobs\n", + "jobs_list = Jobs.getJobsList(top=2)\n", + "jobs_list\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get list of docker jobs\n", + "jobs_list = Jobs.getDockerJobsListQuick(top=2)\n", + "jobs_list" + ] } ], "metadata": { @@ -1461,5 +1445,5 @@ } }, "nbformat": 4, - "nbformat_minor": 1 + "nbformat_minor": 2 } diff --git a/NAMESPACE b/NAMESPACE index 5eaa520..e02a863 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -6,5 +6,5 @@ export(SciDrive.createContainer,SciDrive.upload,SciDrive.publicUrl,SciDrive.down export(Config.isSciServerComputeEnvironment,Config.CasJobsRESTUri,Config.AuthenticationURL,Config.SciDriveHost,Config.SkyQueryUrl,Config.DataRelease,Config.SkyServerWSurl,Config.Version,Config.KeystoneTokenFilePath,Config.ComputeJobDirectoryFile,Config.RacmApiURL) export(SkyQuery.getJobStatus,SkyQuery.cancelJob,SkyQuery.listQueues,SkyQuery.getQueueInfo,SkyQuery.submitJob,SkyQuery.waitForJob,SkyQuery.listJobs,SkyQuery.listAllDatasets,SkyQuery.getDatasetInfo,SkyQuery.listDatasetTables,SkyQuery.getTableInfo,SkyQuery.listTableColumns,SkyQuery.getTable,SkyQuery.dropTable,SkyQuery.uploadTable) export(Files.getFileServices,Files.getFileServicesNames,Files.getFileServiceFromName,Files.__getFileServiceAPIUrl,Files.getRootVolumesInfo,Files.getDataVolumesInfo,Files.getUserVolumesInfo,Files.splitPath,Files.createUserVolume,Files.deleteUserVolume,Files.createDir,Files.delete,Files.upload,Files.download,Files.dirList,Files.move,Files.shareUserVolume) -export(Jobs.getDockerComputeDomains,Jobs.getDockerComputeDomainsNames,Jobs.getDockerComputeDomainFromName,Jobs.getRDBComputeDomains,Jobs.getRDBComputeDomainsNames,Jobs.getRDBComputeDomainFromName,Jobs.getJobsList,Jobs.getJobDescription,Jobs.getJobStatus,Jobs.submitNotebookJob,Jobs.submitShellCommandJob,Jobs.submitRDBQueryJob,Jobs.cancelJob,Jobs.waitForJob) +export(Jobs.getDockerComputeDomains,Jobs.getDockerComputeDomainsNames,Jobs.getDockerComputeDomainFromName,Jobs.getRDBComputeDomains,Jobs.getRDBComputeDomainsNames,Jobs.getRDBComputeDomainFromName,Jobs.getJobsList,Jobs.getDockerJobsListQuick,Jobs.getJobDescription,Jobs.getJobStatus,Jobs.submitNotebookJob,Jobs.submitShellCommandJob,Jobs.submitRDBQueryJob,Jobs.cancelJob,Jobs.waitForJob) import(httr,jsonlite,utils,data.table) diff --git a/R/Jobs.r b/R/Jobs.r index 78b9b8f..b1acce5 100644 --- a/R/Jobs.r +++ b/R/Jobs.r @@ -177,6 +177,41 @@ Jobs.getJobsList <- function(top=10, open=NULL, start=NULL, end=NULL, type="all" } } +Jobs.getDockerJobsListQuick <- function(top=10, open=NULL, start=NULL, end=NULL, labelReg=NULL){ + + token = Authentication.getToken() + if(!is.null(token) && token != "") + { + + if(Config.isSciServerComputeEnvironment()){ + taskName = "Compute.SciScript-R.Jobs.getJobList" + }else{ + taskName = "SciScript-R.Jobs.getJobList" + } + + topString <- if(!is.null(top)) paste("top=",top,"&",sep="") else "" + startString <- if(!is.null(start)) paste("start=",start,"&",sep="") else "" + endString <- if(!is.null(end)) paste("end=",end,"&",sep="") else "" + + openString <- if(!is.null(open)) paste("open=",tolower(as.character(open)),"&",sep="") else "" + labRegString <- if(!is.null(labelReg)) paste("labelReg=",labelReg,"&",sep="") else "" + + + url = paste(Config.RacmApiURL,"/jobm/rest/dockerjobs/quick?",sep="") + url = paste(url,topString,startString,endString,"TaskName=",taskName,sep="") + + r = GET(url,add_headers('X-Auth-Token'=token),accept("application/json")) + + if(r$status_code != 200) { + stop(paste("Error when getting Jobs list from JOBM API.\nHttp Response from JOBM API returned status code ", r$status_code, ":\n", content(r, as="text", encoding="UTF-8"))) + } else { + return(content(r)) + } + }else{ + stop(paste("User token is not defined. First log into SciServer.")) + } +} + Jobs.getJobDescription <- function(jobId){ diff --git a/man/Jobs.getDockerJobsListQuick.Rd b/man/Jobs.getDockerJobsListQuick.Rd new file mode 100644 index 0000000..435d5e6 --- /dev/null +++ b/man/Jobs.getDockerJobsListQuick.Rd @@ -0,0 +1,33 @@ +\name{Jobs.getDockerJobsListQuick} +\alias{Jobs.getDockerJobsListQuick} +\title{ +Get Docker Jobs List +} +\description{ +Quickly gets the list of Docker Jobs submitted by the user. +} +\usage{Jobs.getDockerJobsListQuick(top=10, open=NULL, start=NULL, end=NULL, labelReg=None)} +\arguments{ + \item{top}{top number of jobs (integer) returned. If top=NULL, then all jobs are returned.} + \item{open}{Boolean variable. If set to TRUE, then only returns jobs that have not finished executing and wrapped up (status <= FINISHED). If set to FALSE then only returnes jobs that are still running. If set to NULL, then returns both finished and unfinished jobs.} +\item{start}{The earliest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to NULL, then there is no lower bound on date.} +\item{end}{The latest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to NULL, then there is no upper bound on date.} +\item{labelReg}{A string occurring in the submitterDIDtype of jobs returned.} +} +\value{a list containing the definitions of submitted jobs.} +\author{ +Manuchehr Taghizadeh-Popp \cr +Maintainer: Manuchehr Taghizadeh-Popp +} +\seealso{ +\code{\link[SciServer]{Jobs.getJobsList}},\code{\link[SciServer]{Jobs.submitNotebookJob}},\code{\link[SciServer]{Jobs.submitShellCommandJob}},\code{\link[SciServer]{Jobs.getJobStatus}},\code{\link[SciServer]{Jobs.getDockerComputeDomains}},\code{\link[SciServer]{Jobs.cancelJob}} +} +\references{ +\href{http://www.sciserver.org}{http://www.sciserver.org}\cr +\href{http://apps.sciserver.org/}{http://apps.sciserver.org}\cr +\href{http://www.github.com/sciserver/SciScript-R}{http://www.github.com/sciserver/SciScript-R} +} +\examples{ +jobs = Jobs.getDockerJobsListQuick(top=2) +} +\keyword{SciServer} \ No newline at end of file diff --git a/man/Jobs.getJobsList.Rd b/man/Jobs.getJobsList.Rd index ced9da3..72e1d65 100644 --- a/man/Jobs.getJobsList.Rd +++ b/man/Jobs.getJobsList.Rd @@ -21,7 +21,7 @@ Manuchehr Taghizadeh-Popp \cr Maintainer: Manuchehr Taghizadeh-Popp } \seealso{ -\code{\link[SciServer]{Jobs.submitNotebookJob}},\code{\link[SciServer]{Jobs.submitShellCommandJob}},\code{\link[SciServer]{Jobs.getJobStatus}},\code{\link[SciServer]{Jobs.getDockerComputeDomains}},\code{\link[SciServer]{Jobs.cancelJob}} +\code{\link[SciServer]{Jobs.getDockerJobsListQuick}},\code{\link[SciServer]{Jobs.submitNotebookJob}},\code{\link[SciServer]{Jobs.submitShellCommandJob}},\code{\link[SciServer]{Jobs.getJobStatus}},\code{\link[SciServer]{Jobs.getDockerComputeDomains}},\code{\link[SciServer]{Jobs.cancelJob}} } \references{ \href{http://www.sciserver.org}{http://www.sciserver.org}\cr From 72a43d8ab54323e5b97de24a4d5e7129f3118613 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Tue, 7 May 2019 09:53:48 -0400 Subject: [PATCH 3/4] fix bug in casbobs.executequery: do not assign column names when dataset has no rows --- R/CasJobs.r | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/R/CasJobs.r b/R/CasJobs.r index 1fb4c9f..02837f9 100644 --- a/R/CasJobs.r +++ b/R/CasJobs.r @@ -108,30 +108,28 @@ CasJobs.executeQuery <- function(sql, context="MyDB", format="dataframe") { if(format == "list"){ return(content(r)) }else if(format == "dataframe"){ - tables = content(r, encoding="UTF-8") - if((tables == "\n")[1]){ - return(data.frame(NULL)) - }else{ - response = content(r, "text", encoding="UTF-8") - response = fromJSON(response) - - if(length(response$Result$Data) > 1 ){ - result = list() - for( i in 1:length(response$Result$Data)){ - df = data.frame(response$Result$Data[[i]]) - if(dim(df)[2] >0){ - colnames(df) <- response$Result$Columns[[i]] - } - result[[length(result)+1]] <- df + response = content(r, "text", encoding="UTF-8") + response = fromJSON(response) + + if(length(response$Result$Data) > 1 ){ + result = list() + for( i in 1:length(response$Result$Data)){ + df = data.frame(response$Result$Data[[i]]) + if(dim(df)[2] >0){ + colnames(df) <- response$Result$Columns[[i]] } - }else{ - result = data.frame(response$Result$Data[[1]]) + result[[length(result)+1]] <- df + } + }else{ + result = data.frame(response$Result$Data[[1]]) + if(dim(result)[2] >0){ colnames(result) <- response$Result$Columns[[1]] } - - return(result) } + + return(result) + }else if(format == "json"){ return((content(r, "text", encoding="UTF-8"))) }else if(format == "csv"){ From a54f7db5471357ad24f3df115f50070321f28519 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Fri, 3 Apr 2020 09:33:54 -0400 Subject: [PATCH 4/4] allow writing into Data Volume in job submission --- R/Jobs.r | 52 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/R/Jobs.r b/R/Jobs.r index b1acce5..23a4364 100644 --- a/R/Jobs.r +++ b/R/Jobs.r @@ -345,8 +345,12 @@ Jobs.submitNotebookJob <- function(notebookPath, dockerComputeDomain=NULL, docke dataVols = list() if( is.null(dataVolumes)){ for( i in 1:length(dockerComputeDomain$volumes)){ - vol = dockerComputeDomain$volumes[[i]] - dataVols[[length(dataVols)+1]] <- list(name= vol$name) + vol = dockerComputeDomain$volumes[[i]] + if("write" %in% vol$allowedActions){ + dataVols[[length(dataVols)+1]] <- list(id=vol$id, name= vol$name, writable=TRUE) + }else{ + dataVols[[length(dataVols)+1]] <- list(id=vol$id, name= vol$name, writable=FALSE) + } } }else{ for(i in 1:length(dataVolumes)){ @@ -356,7 +360,24 @@ Jobs.submitNotebookJob <- function(notebookPath, dockerComputeDomain=NULL, docke vol = dockerComputeDomain$volumes[[j]] if( vol$name == dVol$name ){ found = TRUE; - dataVols[[length(dataVols)+1]] <- list(name=vol$name) + if(dVol$needsWriteAccess){ + + if(dVol$needsWriteAccess == TRUE && ('write' %in% vol$allowedActions) ){ + dataVols[[length(dataVols)+1]] <- list(id= vol$id, name=vol$name, writable= TRUE) + }else{ + dataVols[[length(dataVols)+1]] <- list(id= vol$id, name=vol$name, writable= FALSE) + } + + }else{ + + if('write' %in% vol$allowedActions ){ + dataVols[[length(dataVols)+1]] <- list(id= vol$id, name=vol$name, writable= TRUE) + }else{ + dataVols[[length(dataVols)+1]] <- list(id= vol$id, name=vol$name, writable= FALSE) + } + + } + } } @@ -474,7 +495,11 @@ Jobs.submitShellCommandJob <- function(shellCommand, dockerComputeDomain = NULL, if( is.null(dataVolumes)){ for( i in 1:length(dockerComputeDomain$volumes)){ vol = dockerComputeDomain$volumes[[i]] - dataVols[[length(dataVols)+1]] <- list(name= vol$name) + if("write" %in% vol$allowedActions){ + dataVols[[length(dataVols)+1]] <- list(id=vol$id, name= vol$name, writable=TRUE) + }else{ + dataVols[[length(dataVols)+1]] <- list(id=vol$id, name= vol$name, writable=FALSE) + } } }else{ for(i in 1:length(dataVolumes)){ @@ -484,7 +509,24 @@ Jobs.submitShellCommandJob <- function(shellCommand, dockerComputeDomain = NULL, vol = dockerComputeDomain$volumes[[j]] if( vol$name == dVol$name ){ found = TRUE; - dataVols[[length(dataVols)+1]] <- list(name=vol$name) + if(dVol$needsWriteAccess){ + + if(dVol$needsWriteAccess == TRUE && ('write' %in% vol$allowedActions) ){ + dataVols[[length(dataVols)+1]] <- list(id= vol$id, name=vol$name, writable= TRUE) + }else{ + dataVols[[length(dataVols)+1]] <- list(id= vol$id, name=vol$name, writable= FALSE) + } + + }else{ + + if('write' %in% vol$allowedActions ){ + dataVols[[length(dataVols)+1]] <- list(id= vol$id, name=vol$name, writable= TRUE) + }else{ + dataVols[[length(dataVols)+1]] <- list(id= vol$id, name=vol$name, writable= FALSE) + } + + } + } }