diff --git a/.gitignore b/.gitignore index f7b1acc..d75e6eb 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,6 @@ /py3/dist /py3/SciServer.egg-info /py3/.idea -/Examples/.ipynb_checkpoints/ \ No newline at end of file +/Examples/.ipynb_checkpoints/ +/docs_sphinx/SciServer.egg-info +/docs_sphinx/dist \ No newline at end of file diff --git a/Examples/Examples_SciScript-Python3.ipynb b/Examples/Examples_SciScript-Python.ipynb similarity index 53% rename from Examples/Examples_SciScript-Python3.ipynb rename to Examples/Examples_SciScript-Python.ipynb index 0be144c..c2d90ae 100644 --- a/Examples/Examples_SciScript-Python3.ipynb +++ b/Examples/Examples_SciScript-Python.ipynb @@ -4,13 +4,12 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": false, "scrolled": true }, "outputs": [], "source": [ "import SciServer\n", - "from SciServer import Authentication, LoginPortal, Config, CasJobs, SkyQuery, SciDrive, SkyServer\n", + "from SciServer import Authentication, LoginPortal, Config, CasJobs, SkyQuery, SciDrive, SkyServer, Files, Jobs\n", "import os;\n", "import pandas;\n", "import sys;\n", @@ -23,15 +22,16 @@ "\n", "# Define login Name and password before running these examples\n", "Authentication_loginName = '***';\n", - "Authentication_loginPassword = '***'" + "Authentication_loginPassword = '***'\n", + "\n", + "Authentication_login_sharedWithName = '***'\n", + "Authentication_login_sharedWithPassword = '***'" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#help(SciServer)" @@ -40,9 +40,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# *******************************************************************************************************\n", @@ -53,9 +51,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#help(Authentication)" @@ -64,9 +60,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#logging in and getting current token from different ways\n", @@ -84,9 +78,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#getting curent user info\n", @@ -101,9 +93,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#reseting the current token to another value:\n", @@ -120,9 +110,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "#logging-in again\n", @@ -133,9 +121,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# *******************************************************************************************************\n", @@ -146,9 +132,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "#help(LoginPortal)" @@ -157,9 +141,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#logging in and getting current token from different ways\n", @@ -175,9 +157,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#getting curent user info\n", @@ -192,9 +172,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#reseting the current token to another value:\n", @@ -207,9 +185,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#logging-in again\n", @@ -220,9 +196,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# *******************************************************************************************************\n", @@ -233,9 +207,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "#help(CasJobs)" @@ -244,9 +216,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "#Defining databse context and query, and other variables\n", @@ -263,9 +233,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#get user schema info\n", @@ -277,9 +245,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#get info about tables inside MyDB database context:\n", @@ -291,9 +257,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#execute a quick SQL query:\n", @@ -305,9 +269,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#submit a job, which inserts the query results into a table in the MyDB database context. \n", @@ -322,9 +284,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# drop or delete table in MyDB database context\n", @@ -336,9 +296,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#get job status\n", @@ -352,9 +310,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#cancel a job\n", @@ -368,9 +324,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#execute a query and write a local Fits file containing the query results:\n", @@ -382,9 +336,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#delete local FITS file just created:\n", @@ -395,9 +347,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#get a Pandas dataframe containing the results of a query\n", @@ -409,9 +359,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# get numpy array containing the results of a query\n", @@ -423,9 +371,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#uploads a Pandas dataframe into a Database table\n", @@ -440,9 +386,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# drop or delete table just created:\n", @@ -454,9 +398,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#upload csv data string into a database table:\n", @@ -470,9 +412,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# drop or delete table just created:\n", @@ -484,9 +424,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# *******************************************************************************************************\n", @@ -497,9 +435,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#help(SkyServer)" @@ -508,9 +444,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "#defining sql query and SDSS data relelease:\n", @@ -522,9 +456,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#Exectute sql query\n", @@ -536,9 +468,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#get an image cutout\n", @@ -552,9 +482,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# do a radial search of objects:\n", @@ -566,9 +494,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#do rectangular search of objects:\n", @@ -580,9 +506,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#do an object search based on RA,Dec coordinates:\n", @@ -594,9 +518,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# *******************************************************************************************************\n", @@ -607,9 +529,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "#help(SciDrive)" @@ -618,9 +538,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#list content and metadata of top level directory in SciDrive\n", @@ -632,9 +550,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "#define name of directory to be created in SciDrive:\n", @@ -648,9 +564,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#create a folder or container in SciDrive\n", @@ -662,9 +576,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#list content and metadata of directory in SciDrive\n", @@ -676,9 +588,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#get the public url to access the directory content in SciDrive\n", @@ -690,9 +600,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#Delete folder or container in SciDrive:\n", @@ -704,9 +612,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#create a local file:\n", @@ -719,9 +625,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#uploading a file to SciDrive:\n", @@ -733,9 +637,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#download file:\n", @@ -748,9 +650,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#upload string data:\n", @@ -763,9 +663,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#delete folder in SciDrive:\n", @@ -777,9 +675,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#delete local file:\n", @@ -790,9 +686,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# *******************************************************************************************************\n", @@ -803,9 +697,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "#help(SkyQuery)" @@ -814,9 +706,16 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, + "outputs": [], + "source": [ + "token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "#list all databses or datasets available\n", @@ -828,9 +727,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#get info about the user's personal database or dataset\n", @@ -842,9 +739,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#list tables inside dataset\n", @@ -856,9 +751,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#list available job queues\n", @@ -870,9 +763,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#list available job queues and related info\n", @@ -886,9 +777,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "#Define query\n", @@ -899,9 +788,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#submit a query as a job\n", @@ -913,9 +800,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#get status of a submitted job\n", @@ -928,9 +813,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# wait for a job to be finished and then get the status\n", @@ -944,9 +827,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# cancel a job that is running, and then get its status\n", @@ -961,9 +842,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# get list of jobs\n", @@ -977,9 +856,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "#define csv table to be uploaded to into MyDB in SkyQuery\n", @@ -991,9 +868,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#uploading the csv table:\n", @@ -1005,9 +880,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#downloading table:\n", @@ -1019,9 +892,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#list tables inside dataset\n", @@ -1033,9 +904,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#get dataset table info:\n", @@ -1047,9 +916,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#get dataset table columns info\n", @@ -1061,9 +928,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "#drop (or delete) table from dataset.\n", @@ -1071,27 +936,617 @@ "result = SkyQuery.dropTable(tableName=SkyQuery_TestTableName, datasetName=\"MyDB\");\n", "print(result)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# *******************************************************************************************************\n", + "# Files section:\n", + "# *******************************************************************************************************" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#help(Files)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# defining the FileService name\n", + "\n", + "Files_FileServiceName = \"FileServiceJHU\"\n", + "Files_RootVolumeName1 = \"Storage\"\n", + "Files_UserVolumeName1 = \"UserVolume5\"\n", + "Files_RootVolumeName2 = \"Storage\"\n", + "Files_UserVolumeName2 = \"UserVolume9\"\n", + "Files_NewDirectoryName1 = \"myNewDirectory5\"\n", + "Files_NewDirectoryName2 = \"myNewDirectory9\"\n", + "Files_LocalFileName = \"MyNewFile.txt\"\n", + "Files_LocalFileContent = \"#ID,Column1,Column2\\n1,4.5,5.5\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get available FileServices\n", + "\n", + "fileServices = Files.getFileServices();\n", + "print(fileServices)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get names of file services available to the user.\n", + "\n", + "fileServiceNames = Files.getFileServicesNames();\n", + "print(fileServiceNames)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get FileService from Name\n", + "fileService = Files.getFileServiceFromName(Files_FileServiceName);\n", + "fileService" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get the API endpoint URL of a FileService\n", + "\n", + "fileServiceAPIUrl = Files.__getFileServiceAPIUrl(fileService);\n", + "print(fileServiceAPIUrl)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get root volumes\n", + "\n", + "rootVolumes = Files.getRootVolumesInfo(fileService)\n", + "rootVolumes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get user volumes\n", + "\n", + "userVolumes = Files.getUserVolumesInfo(fileService)\n", + "userVolumes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get data volumes\n", + "\n", + "dataVolumes = Files.getDataVolumesInfo(fileService)\n", + "dataVolumes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# create user volume\n", + "\n", + "Files.createUserVolume(fileService,\n", + " \"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1]))\n", + "Files.createUserVolume(fileService, \n", + " \"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName2]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# create a directory in the persistent volume\n", + "\n", + "Files.createDir(fileService, \n", + " \"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1]));\n", + "Files.createDir(fileService, \n", + " \"/\".join([Files_RootVolumeName2,Authentication_loginName,Files_UserVolumeName2,Files_NewDirectoryName2]));" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# upload a text string into a file in the remote directory\n", + "\n", + "Files.upload(fileService, \n", + " \"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1,Files_LocalFileName]), \n", + " data=Files_LocalFileContent);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# List content of remote directory\n", + "\n", + "dirList = Files.dirList(fileService, \n", + " \"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1]),\n", + " level=4)\n", + "dirList" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# download a remote text file into a local directory\n", + "\n", + "Files.download(fileService, \n", + " \"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1,Files_LocalFileName]) \n", + " ,localFilePath=Files_LocalFileName);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Delete remote file\n", + "\n", + "Files.delete(fileService, \n", + "\"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1,Files_LocalFileName]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# upload a local file into a remote directory\n", + "\n", + "Files.upload(fileService, \n", + "\"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1,Files_LocalFileName])\n", + ", localFilePath=Files_LocalFileName, quiet=False);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# copy remote file into remote directory\n", + "\n", + "Files.move(fileService, \n", + "\"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1,Files_LocalFileName]), \n", + " fileService, \n", + "\"/\".join([Files_RootVolumeName2,Authentication_loginName,Files_UserVolumeName2,Files_NewDirectoryName2 ,Files_LocalFileName])\n", + ");" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# sharing user volume with another user\n", + "\n", + "Files.shareUserVolume(fileService, \n", + "\"/\".join([Files_RootVolumeName2,Authentication_loginName,Files_UserVolumeName2]), \n", + " sharedWith=Authentication_login_sharedWithName, sharee_type=\"USER\",allowedActions=[\"read\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# let the other user log-in\n", + "\n", + "token1 = Authentication.login(Authentication_login_sharedWithName, Authentication_login_sharedWithPassword);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# the other user downloads the remote text file (from the shared user volume) into a local string variable\n", + "\n", + "string = Files.download(fileService, \n", + " \"/\".join([Files_RootVolumeName2,Authentication_loginName,Files_UserVolumeName2,Files_NewDirectoryName2,Files_LocalFileName]), \n", + " format=\"txt\");\n", + "print(string)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# the original user logs in again\n", + "\n", + "token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# delete folders\n", + "\n", + "Files.delete(fileService, \n", + " \"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1]))\n", + "Files.delete(fileService, \n", + " \"/\".join([Files_RootVolumeName2,Authentication_loginName,Files_UserVolumeName2,Files_NewDirectoryName2]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# delete user volumes\n", + "\n", + "Files.deleteUserVolume(fileService, \n", + " \"/\".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1]))\n", + "Files.deleteUserVolume(fileService, \n", + " \"/\".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName2]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# *******************************************************************************************************\n", + "# Jobs section:\n", + "# *******************************************************************************************************" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#help(Jobs)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Jobs_DockerComputeDomainName = 'Small Jobs Domain'\n", + "\n", + "Jobs_FileServiceName = \"FileServiceJHU\"\n", + "Jobs_RootVolumeName = \"Storage\"\n", + "Jobs_UserVolumeName = \"JobsTestVolume\"\n", + "Jobs_DirectoryName = \"JobsTestDirectory\"\n", + "Jobs_NotebookName = 'TestNotebook.ipynb'\n", + "Jobs_ShellCommand = 'ls -laht'\n", + "Jobs_DockerImageName = 'Python + R'\n", + "\n", + "Jobs_UserVolumes = [{'name':Jobs_UserVolumeName, 'rootVolumeName':Jobs_RootVolumeName, 'owner':Authentication_loginName, 'needsWriteAccess':True},\n", + " {'name':'scratch', 'rootVolumeName':'Temporary', 'owner':Authentication_loginName, 'needsWriteAccess':True}]\n", + "Jobs_DataVolumes = [{'name':'SDSS DAS'}]\n", + "Jobs_Parameters = 'param1=1\\nparam2=2\\nparam3=3'\n", + "Jobs_Alias = 'MyNewJob'\n", + "\n", + "Jobs_SqlQuery = \"select 1;\"\n", + "Jobs_RBBComputeDomainName = 'Manga (long)'\n", + "Jobs_DatabaseContextName = \"manga\"\n", + "Jobs_QueryResultsFile = 'myQueryResults'\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get docker compute domains\n", + "\n", + "dockerComputeDomains = Jobs.getDockerComputeDomains();\n", + "print(dockerComputeDomains)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get names of docker compute domains available to the user\n", + "\n", + "dockerComputeDomainsNames = Jobs.getDockerComputeDomainsNames()\n", + "print(dockerComputeDomainsNames)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get docker compute domain from name\n", + "\n", + "dockerComputeDomain = Jobs.getDockerComputeDomainFromName(Jobs_DockerComputeDomainName)\n", + "dockerComputeDomain" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# uploading Jupyter notebook to remote directory\n", + "\n", + "fileService = Files.getFileServiceFromName(Jobs_FileServiceName);\n", + "Files.createUserVolume(fileService, \n", + " \"/\".join([Jobs_RootVolumeName, Authentication_loginName, Jobs_UserVolumeName]))\n", + "Files.upload(fileService, \n", + " \"/\".join([Jobs_RootVolumeName,Authentication_loginName,Jobs_UserVolumeName,Jobs_DirectoryName,Jobs_NotebookName]), \n", + " localFilePath=Jobs_NotebookName);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# submit a Jupyter notebook job.\n", + "\n", + "jobId = Jobs.submitNotebookJob(notebookPath='/home/idies/workspace/' + \n", + "\"/\".join([Jobs_RootVolumeName,Authentication_loginName,Jobs_UserVolumeName,Jobs_DirectoryName,Jobs_NotebookName]),\n", + " dockerComputeDomain=dockerComputeDomain,\n", + " dockerImageName=Jobs_DockerImageName,\n", + " userVolumes=Jobs_UserVolumes,\n", + " dataVolumes=Jobs_DataVolumes,\n", + " resultsFolderPath=\"\",\n", + " parameters=Jobs_Parameters,\n", + " jobAlias=Jobs_Alias)\n", + "print(jobId)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get job status\n", + "\n", + "jobStatus = Jobs.getJobStatus(jobId)\n", + "print(jobStatus)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get job description\n", + "\n", + "job = Jobs.getJobDescription(jobId)\n", + "print(job)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# wait until job is finished and get job status\n", + "\n", + "jobId = Jobs.submitNotebookJob(notebookPath='/home/idies/workspace/' + \n", + "\"/\".join([Jobs_RootVolumeName,Authentication_loginName,Jobs_UserVolumeName,Jobs_DirectoryName,Jobs_NotebookName]),\n", + " dockerComputeDomain=dockerComputeDomain,\n", + " dockerImageName=Jobs_DockerImageName, \n", + " userVolumes=Jobs_UserVolumes, dataVolumes=Jobs_DataVolumes,\n", + " resultsFolderPath=\"\",\n", + " parameters=Jobs_Parameters, jobAlias=Jobs_Alias)\n", + "jobStatus = Jobs.waitForJob(jobId)\n", + "print(jobStatus)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# cancel job\n", + "\n", + "jobId = Jobs.submitNotebookJob('/home/idies/workspace/' + Jobs_UserVolumeName + '/' + Jobs_DirectoryName + '/' + Jobs_NotebookName,\n", + " dockerComputeDomain=dockerComputeDomain,\n", + " dockerImageName=Jobs_DockerImageName, \n", + " userVolumes=Jobs_UserVolumes, dataVolumes=Jobs_DataVolumes,\n", + " resultsFolderPath=\"\",\n", + " parameters=Jobs_Parameters, jobAlias=Jobs_Alias)\n", + "Jobs.cancelJob(jobId)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Files.deleteUserVolume(fileService, \"/\".join([Jobs_RootVolumeName,Authentication_loginName,Jobs_UserVolumeName]), quiet=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# submit shell command job\n", + "\n", + "jobId = Jobs.submitShellCommandJob(shellCommand=Jobs_ShellCommand,\n", + " dockerComputeDomain=dockerComputeDomain,\n", + " dockerImageName=Jobs_DockerImageName,\n", + " userVolumes=Jobs_UserVolumes, dataVolumes=Jobs_DataVolumes,\n", + " resultsFolderPath = \"\",\n", + " jobAlias=Jobs_Alias)\n", + "jobStatus = Jobs.waitForJob(jobId)\n", + "print(jobStatus)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get relational database (RDB) compute domains\n", + "\n", + "rdbComputeDomains = Jobs.getRDBComputeDomains();\n", + "print(rdbComputeDomains)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get names of relational database (RDB) compute domains\n", + "\n", + "rdbComputeDomainsNames = Jobs.getRDBComputeDomainsNames()\n", + "print(rdbComputeDomainsNames)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get relational database (RDB) compute domain from name\n", + "\n", + "rdbComputeDomain = Jobs.getRDBComputeDomainFromName(Jobs_RBBComputeDomainName);\n", + "print(rdbComputeDomain)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# submit RDB (relatinal database) query job\n", + "\n", + "jobId = Jobs.submitRDBQueryJob(Jobs_SqlQuery, rdbComputeDomain, Jobs_DatabaseContextName, Jobs_QueryResultsFile, Jobs_Alias)\n", + "print(jobId)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get job description\n", + "\n", + "job = Jobs.getJobDescription(jobId)\n", + "print(job)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 2", + "display_name": "Python 3", "language": "python", - "name": "python2" + "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" + "pygments_lexer": "ipython3", + "version": "3.6.5" } }, "nbformat": 4, - "nbformat_minor": 0 + "nbformat_minor": 1 } diff --git a/Examples/Examples_SciScript-Python3.py b/Examples/Examples_SciScript-Python.py similarity index 58% rename from Examples/Examples_SciScript-Python3.py rename to Examples/Examples_SciScript-Python.py index 2e94851..9cdc89b 100644 --- a/Examples/Examples_SciScript-Python3.py +++ b/Examples/Examples_SciScript-Python.py @@ -4,7 +4,7 @@ # In[ ]: import SciServer -from SciServer import Authentication, LoginPortal, Config, CasJobs, SkyQuery, SciDrive, SkyServer +from SciServer import Authentication, LoginPortal, Config, CasJobs, SkyQuery, SciDrive, SkyServer, Files, Jobs import os; import pandas; import sys; @@ -19,6 +19,9 @@ Authentication_loginName = '***'; Authentication_loginPassword = '***' +Authentication_login_sharedWithName = '***' +Authentication_login_sharedWithPassword = '***' + # In[ ]: @@ -475,6 +478,11 @@ #help(SkyQuery) +# In[ ]: + +token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword); + + # In[ ]: #list all databses or datasets available @@ -627,3 +635,362 @@ result = SkyQuery.dropTable(tableName=SkyQuery_TestTableName, datasetName="MyDB"); print(result) + +# In[ ]: + +# ******************************************************************************************************* +# Files section: +# ******************************************************************************************************* + + +# In[ ]: + +#help(Files) + + +# In[ ]: + +# defining the FileService name + +Files_FileServiceName = "FileServiceJHU" +Files_RootVolumeName1 = "volumes" +Files_UserVolumeName1 = Authentication_loginName + "_UserVolume555" +Files_RootVolumeName2 = "volumes" +Files_UserVolumeName2 = Authentication_loginName + "_UserVolume999" +Files_NewDirectoryName1 = "myNewDirectory555" +Files_NewDirectoryName2 = "myNewDirectory999" +Files_LocalFileName = "MyNewFile.txt" +Files_LocalFileContent = "#ID,Column1,Column2\n1,4.5,5.5" + + +# In[ ]: + +token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword); + + +# In[ ]: + +# get available FileServices + +fileServices = Files.getFileServices(); +print(fileServices) + + +# In[ ]: + +# get names of file services available to the user. + +fileServiceNames = Files.getFileServicesNames(); +print(fileServiceNames) + + +# In[ ]: + +# get FileService from Name + +fileService = Files.getFileServiceFromName(Files_FileServiceName); +print(fileService) + + +# In[ ]: + +# get the API endpoint URL of a FileService + +fileServiceAPIUrl = Files.__getFileServiceAPIUrl(fileService); +print(fileServiceAPIUrl) + + +# In[ ]: + +# get root volumes + +rootVolumes = Files.getRootVolumes(fileService) +print(rootVolumes) + + +# In[ ]: + +# create user volume + +Files.createUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName1) +Files.createUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName2) + + +# In[ ]: + +# create a directory in the persistent volume + +Files.createDir(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1); +Files.createDir(fileService, Files_RootVolumeName2, Files_UserVolumeName2, Files_NewDirectoryName2); + + +# In[ ]: + +# upload a text string into a file in the remote directory + +Files.upload(fileService, Files_RootVolumeName1, Files_UserVolumeName1, + Files_NewDirectoryName1 + "/" + Files_LocalFileName, + data=Files_LocalFileContent); + + +# In[ ]: + +# List content of remote directory + +dirList = Files.dirList(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1, level=2) +print(dirList) + + +# In[ ]: + +# download a remote text file into a local directory + +Files.download(fileService, Files_RootVolumeName1, Files_UserVolumeName1, + Files_NewDirectoryName1 + "/" + Files_LocalFileName, + localFilePath=Files_LocalFileName); + + +# In[ ]: + +# Delete remote file + +Files.delete(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1 + "/" + Files_LocalFileName) + + +# In[ ]: + +# upload a local file into a remote directory + +Files.upload(fileService, Files_RootVolumeName1, Files_UserVolumeName1, + Files_NewDirectoryName1 + "/" + Files_LocalFileName, localFilePath=Files_LocalFileName, quiet=False); + + +# In[ ]: + +# copy remote file into remote directory + +Files.move(fileService, Files_RootVolumeName1, Files_UserVolumeName1,Files_NewDirectoryName1 + "/" + Files_LocalFileName, + fileService, Files_RootVolumeName2, Files_UserVolumeName2,Files_UserVolumeName2 + "/" + Files_NewDirectoryName2 + "/" + Files_LocalFileName + ); + + +# In[ ]: + +# sharing user volume with another user + +Files.shareUserVolume(fileService, Files_RootVolumeName2, Files_UserVolumeName2, + sharedWith=Authentication_login_sharedWithName, type="USER", + userVolumeOwner=Authentication_loginName, + allowedActions=["read"]) + + +# In[ ]: + +# let the other user log-in + +token1 = Authentication.login(Authentication_login_sharedWithName, Authentication_login_sharedWithPassword); + + +# In[ ]: + +# the other user downloads the remote text file (from the shared user volume) into a local string variable + +string = Files.download(fileService, Files_RootVolumeName2, Files_UserVolumeName2, + Files_NewDirectoryName2 + "/" + Files_LocalFileName, + format="txt", userVolumeOwner=Authentication_loginName); +print(string) + + +# In[ ]: + +# the original user logs in again + +token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword); + + +# In[ ]: + +# delete folders + +Files.delete(fileService, Files_RootVolumeName1, Files_UserVolumeName1, Files_NewDirectoryName1) +Files.delete(fileService, Files_RootVolumeName2, Files_UserVolumeName2, Files_NewDirectoryName2) + + +# In[ ]: + +# delete user volumes + +Files.deleteUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName1) +Files.deleteUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName2) + + +# In[ ]: + +# ******************************************************************************************************* +# Jobs section: +# ******************************************************************************************************* + + +# In[ ]: + +#help(Jobs) + + +# In[ ]: + +Jobs_DockerComputeDomainName = 'Small Jobs Batch Domain' + +Jobs_FileServiceName = "FileServiceJHU" +Jobs_RootVolumeName = "volumes" +Jobs_UserVolumeName = Authentication_loginName + "_JobsTestVolume" +Jobs_DirectoryName = "JobsTestDirectory" +Jobs_NotebookName = 'TestNotebook.ipynb' +Jobs_ShellCommand = 'ls -laht' +Jobs_DockerImageName = 'Python (astro)' + +Jobs_UserVolumes = [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}] +Jobs_DataVolumes = [{'name':'SDSS_DAS'}] +Jobs_Parameters = 'param1=1\nparam2=2\nparam3=3' +Jobs_Alias = 'MyNewJob' + +Jobs_SqlQuery = "select 1;" +Jobs_RBBComputeDomainName = 'Manga (long)' +Jobs_DatabaseContextName = "manga" +Jobs_QueryResultsFile = 'myQueryResults' + + +# In[ ]: + +# get docker compute domains + +dockerComputeDomains = Jobs.getDockerComputeDomains(); +print(dockerComputeDomains) + + +# In[ ]: + +# get names of docker compute domains available to the user + +dockerComputeDomainsNames = Jobs.getDockerComputeDomainsNames() +print(dockerComputeDomainsNames) + + +# In[ ]: + +# get docker compute domain from name + +dockerComputeDomain = Jobs.getDockerComputeDomainFromName(Jobs_DockerComputeDomainName) +print(dockerComputeDomain) + + +# In[ ]: + +# uploading Jupyter notebook to remote directory + +fileService = Files.getFileServiceFromName(Jobs_FileServiceName); +Files.createUserVolume(fileService, Jobs_RootVolumeName, Jobs_UserVolumeName) +Files.upload(fileService, Jobs_RootVolumeName, Jobs_UserVolumeName, + Jobs_DirectoryName + "/" + Jobs_NotebookName, + localFilePath=Jobs_NotebookName); + + +# In[ ]: + +# submit a Jupyter notebook job. + +jobId = Jobs.submitNotebookJob('/home/idies/workspace/' + Jobs_UserVolumeName + '/' + Jobs_DirectoryName + '/' + Jobs_NotebookName, + dockerComputeDomain, + Jobs_DockerImageName, + Jobs_UserVolumes, Jobs_DataVolumes, + Jobs_Alias) +print(jobId) + + +# In[ ]: + +# get job status + +jobStatus = Jobs.getJobStatus(jobId) +print(jobStatus) + + +# In[ ]: + +# get job description + +job = Jobs.getJobDescription(jobId) +print(job) + + +# In[ ]: + +# wait until job is finsihed and get job status + +jobId = Jobs.submitNotebookJob('/home/idies/workspace/' + Jobs_UserVolumeName + '/' + Jobs_DirectoryName + '/' + Jobs_NotebookName, + dockerComputeDomain, + Jobs_DockerImageName, + Jobs_UserVolumes, Jobs_DataVolumes, + Jobs_Alias) +jobStatus = Jobs.waitForJob(jobId) +print(jobStatus) + + +# In[ ]: + +# cancel job + +jobId = Jobs.submitNotebookJob('/home/idies/workspace/' + Jobs_UserVolumeName + '/' + Jobs_DirectoryName + '/' + Jobs_NotebookName, + dockerComputeDomain, + Jobs_DockerImageName, + Jobs_UserVolumes, Jobs_DataVolumes, + Jobs_Alias) +Jobs.cancelJob(jobId) + + +# In[ ]: + +Files.deleteUserVolume(fileService, Jobs_RootVolumeName, Jobs_UserVolumeName, quiet=True) + + +# In[ ]: + +# submit shell command job + +jobId = Jobs.submitShellCommandJob(Jobs_ShellCommand, + dockerComputeDomain, + Jobs_DockerImageName, + Jobs_UserVolumes, Jobs_DataVolumes, + Jobs_Alias) + + +# In[ ]: + +# get relational database (RDB) compute domains + +rdbComputeDomains = Jobs.getRDBComputeDomains(); +print(rdbComputeDomains) + + +# In[ ]: + +# get names of relational database (RDB) compute domains + +rdbComputeDomainsNames = Jobs.getRDBComputeDomainsNames() +print(rdbComputeDomainsNames) + + +# In[ ]: + +# get relational database (RDB) compute domain from name + +rdbComputeDomain = Jobs.getRDBComputeDomainFromName(Jobs_RBBComputeDomainName); +print(rdbComputeDomain) + + +# In[ ]: + +# submit RDB (relatinal database) query job + +jobId = Jobs.submitRDBQueryJob(Jobs_SqlQuery, rdbComputeDomain, Jobs_DatabaseContextName, Jobs_QueryResultsFile, Jobs_Alias) +print(jobId) + diff --git a/Examples/Sciquery_Demo(long).ipynb b/Examples/Sciquery_Demo(long).ipynb new file mode 100644 index 0000000..1806cc8 --- /dev/null +++ b/Examples/Sciquery_Demo(long).ipynb @@ -0,0 +1,1175 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from SciServer.SciQuery import *" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1) Instantiation: different ways" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 1.0 Defining name of database and domain, for future queries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "database_name = 'BestDR16'\n", + "rdb_compute_domain_name = 'IDIES-SQLServer (Short queue)'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 1.1 Creating instance with default values/settings:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq = SciQuery()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 1.2 Ways of looking at preset instance values:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "print(f\"Default Domian = {sq.rdb_compute_domain}\" )\n", + "print(f\"Default Domian = {sq.get_rdb_compute_domain()}\" )\n", + "print(f\"Default Database = {sq.database}\" )\n", + "print(f\"Default Database = {sq.get_database()}\" )\n", + "print(f\"Default Output = {sq.outputs}\" )\n", + "print(f\"Default Output = {sq.get_outputs()}\" )\n", + "print(f\"Default File service = {sq.file_service.get('name')}\" )\n", + "print(f\"Default File service = {sq.get_file_service().get('name')}\" )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 1.3 Several ways of setting values:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### 1.3.1 By name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.rdb_compute_domain = rdb_compute_domain_name\n", + "sq.database = database_name" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### 1.3.2 By object" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "rdb_compute_domain = sq.get_rdb_compute_domain(rdb_compute_domain_name)\n", + "sq.rdb_compute_domain = rdb_compute_domain\n", + "database = sq.get_database(database_name)\n", + "sq.database = database" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### 1.3.3 Altogether by name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq = SciQuery()\n", + "sq.set(rdb_compute_domain_name, database_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### 1.3.4 Altogether by object" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq = SciQuery()\n", + "sq.set(rdb_compute_domain, database)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### 1.3.5 During instantiation, by name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq = SciQuery(rdb_compute_domain_name, database_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### 1.3.6 During instantiation, by object" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq = SciQuery(rdb_compute_domain, database)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 1.4 Updating the information stored in SciQuery if there are external changes, like being allowed to query new domains/databases:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.refresh()\n", + "print(sq.refresh_date)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2) Exploring stored objects and asociated metadata" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.get_rdb_compute_domain_names()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.get_rdb_compute_domains_metadata()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.get_rdb_compute_domains_metadata(do_include_databases=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "SciQuery.get_rdb_compute_domains()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.rdb_compute_domains" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "rdb_compute_domain = sq.get_rdb_compute_domain(rdb_compute_domain_name)\n", + "rdb_compute_domain" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "rdb_compute_domain.get_metadata()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "rdb_compute_domain.get_database_names()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "rdb_compute_domain.get_databases_metadata()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "rdb_compute_domain.databases" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "database = rdb_compute_domain.get_database(database_name)\n", + "database" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "database.get_metadata()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3) Detailed Database Metadata" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.rdb_compute_domain = rdb_compute_domain_name\n", + "sq.database = database_name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "table_names = sq.get_table_names()\n", + "#table_names = sq.get_table_names(database, rdb_compute_domain)\n", + "print(table_names)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.get_tables_metadata()\n", + "# or \n", + "sq.get_tables_metadata(database, rdb_compute_domain)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "table_name = table_names[0]\n", + "print(sq.get_column_names(table_name))\n", + "# or \n", + "print(sq.get_column_names(table_name, database, rdb_compute_domain))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sq.get_columns_metadata(table_name)\n", + "# or \n", + "sq.get_columns_metadata(table_name, database, rdb_compute_domain)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(sq.get_constraint_names(table_name))\n", + "# or \n", + "print(sq.get_constraint_names(table_name, database, rdb_compute_domain))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sq.get_constraints_metadata(table_name)\n", + "# or \n", + "sq.get_constraints_metadata(table_name, database, rdb_compute_domain)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "routine_names = sq.get_routine_names()\n", + "# or \n", + "routine_names = sq.get_routine_names(database, rdb_compute_domain)\n", + "print(routine_names)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sq.get_routines_metadata()\n", + "# or \n", + "sq.get_routines_metadata(database, rdb_compute_domain)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "routine_name = routine_names[0]\n", + "print(sq.get_routine_parameter_names(routine_name))\n", + "# or \n", + "print(sq.get_routine_parameter_names(routine_name, database, rdb_compute_domain))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sq.get_routine_parameters_metadata(routine_name)\n", + "# or \n", + "sq.get_routine_parameters_metadata(routine_name, database, rdb_compute_domain)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(sq.get_view_names())\n", + "# or \n", + "print(sq.get_view_names(database, rdb_compute_domain))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sq.get_views_metadata()\n", + "# or \n", + "sq.get_views_metadata(database, rdb_compute_domain)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4) Executing Synchronous Queries to Databases " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 4.1 Setting database and compute domain" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.set(database=database_name, rdb_compute_domain=rdb_compute_domain_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 4.2 Defining Single SQL Query" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sql_query = \"SELECT TOP 2 specObjID, ra, dec, z FROM specobj\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 4.3 Getting query result as pandas dataframe" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "df = sq.execute_query(sql_query)\n", + "# or\n", + "#df = sq.execute_query(sql_query, database_name, rdb_compute_domain=rdb_compute_domain_name)\n", + "df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5) Submitting query as a batch job" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 5.1 Using default JSON file output" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "jobid = sq.submit_query_job(sql_query=sql_query)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 5.2 Getting current job status" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.get_job_status(jobid)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 5.3 Waiting for job to finish before getting it back" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job = sq.wait_for_job(jobid, verbose=True)\n", + "job\n", + "\n", + "# or after job is " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 5.4 Getting job object" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job = sq.get_job(jobid)\n", + "job" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 5.5 Getting list of submitted jobs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sq.get_jobs_list(top = 2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 5.6 Getting job object and checking its properties" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job = sq.get_job(jobid)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.get_job_status()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.get_metadata()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "outputs = job.outputs\n", + "outputs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.get_output_path()\n", + "# or \n", + "job.get_output_path(0)\n", + "# or\n", + "job.get_output_path(job.outputs[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.get_json_output()\n", + "# or\n", + "job.get_json_output(0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.get_dataframe_from_output(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 5.7 Cancelling job" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "jobid = sq.submit_query_job(sql_query=sql_query)\n", + "sq.cancel_job(jobid)\n", + "\n", + "# or \n", + "\n", + "#jobid = sq.submit_query_job(sql_query=sql_query)\n", + "#job = sq.get_job(jobid)\n", + "#job.cancel()\n", + "\n", + "job = sq.wait_for_job(jobid)\n", + "print(job.status_string)\n", + "print(sq.get_job_status(jobid))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "results_base_path = '/home/idies/workspace/Temporary/{}/scratch/sciqueryjobs/dir/'.format(SciQuery.get_user().userName)\n", + "# or\n", + "results_base_path = FileOutput.build_file_base_path(top_volume=\"Temporary\", \n", + " user_volume=\"scratch\",\n", + " user_volume_owner_name=SciQuery.get_user().userName, \n", + " relative_path=\"sciqueryjobs/dir/\", \n", + " add_date_ending=False)\n", + "results_base_path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "name = results_base_path + \"result.json\"\n", + "output = FileOutput(name=name, output_type=OutputType.FILE_JSON, statement_indexes=1)\n", + "print(output.file)\n", + "print(output.file_base_path)\n", + "print(output.path)\n", + "print(output.get_path())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "jobid = sq.submit_query_job(sql_query=sql_query, outputs=output)\n", + "\n", + "# or \n", + "#jobid = sq.submit_query_job(sql_query=sql_query, results_base_path=results_base_path)\n", + "\n", + "# or \n", + "#sq.set_results_base_path(results_base_path)\n", + "#jobid = sq.submit_query_job(sql_query=sql_query)\n", + "\n", + "job = sq.wait_for_job(jobid, verbose=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.outputs[0].get_path()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "out = job.outputs[0]\n", + "out" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.get_dataframe_from_output(0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.get_json_output(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 5.9 Submitting query as batch job, with output to table in mydb database" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sql_query = \"SELECT TOP 2 specObjID, ra, dec, z FROM specobj\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "output_table = \"results12e\"\n", + "output_rdb_compute_domain_name = rdb_compute_domain_name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# setting output database directly to mydb:\n", + "\n", + "output_database_name = SciQuery.get_mydb_name() \n", + "print(output_database_name)\n", + "\n", + "# verifying that it is fact a database contained in the output rdb compute domian:\n", + "\n", + "print(output_database_name in sq.get_database_names(output_rdb_compute_domain_name))\n", + "print(sq.get_rdb_compute_domain(output_rdb_compute_domain_name).get_database(output_database_name).name)\n", + "\n", + "# or verifying that it is fact the default database in the output rdb compute domian:\n", + "\n", + "output_database_name = sq.get_rdb_compute_domain(output_rdb_compute_domain_name).get_default_database().name\n", + "print(output_database_name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "output = DatabaseTableOutput(table=output_table, database=output_database_name, rdb_compute_domain=output_rdb_compute_domain_name)\n", + "output" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "jobid = sq.submit_query_job(sql_query=sql_query, outputs=output)\n", + "job = sq.wait_for_job(jobid)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "print(job.message_list)\n", + "print(job.targets)\n", + "job.get_metadata()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# looking at the contents of the output table\n", + "\n", + "job.get_dataframe_from_output(0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# or directly making a query to see the contents:\n", + "\n", + "sql_query = \"SELECT * FROM \" + output_table\n", + "sq.execute_query(sql_query, database= output_database_name, rdb_compute_domain= output_rdb_compute_domain_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 5.10 Submitting a multi-query sql statement as a batch job, with multiple outputs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sql_query = \"SELECT TOP 2 specObjID, ra,dec, z FROM specobj; SELECT TOP 2 specobjid, petromag_u, petromag_r FROM specphoto;\"\n", + "job_alias = \"multi-query\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "out1 = FileOutput(name=\"query1.json\", output_type=OutputType.FILE_JSON, statement_indexes=1)\n", + "out2 = FileOutput(name=\"query2.json\", output_type=OutputType.FILE_JSON, statement_indexes=2)\n", + "out3 = FileOutput(name=\"query12.json\", output_type=OutputType.FILE_JSON, statement_indexes=[1,2])\n", + "\n", + "outputs = Outputs(out1, out2, out3) \n", + "# or\n", + "outputs = Outputs([out1, out2, out3])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "results_base_path = '/home/idies/workspace/Temporary/{}/scratch/sciqueryjobs/'.format(SciQuery.get_user().userName)\n", + "# or\n", + "sq.results_base_path = results_base_path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "jobid = sq.submit_query_job(sql_query=sql_query, outputs=outputs, results_base_path=results_base_path, job_alias=job_alias)\n", + "job = sq.wait_for_job(jobid)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "outputs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.outputs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.get_json_output(0)\n", + "# or\n", + "job.get_json_output(job.outputs[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.get_json_output(1)\n", + "# or\n", + "job.get_json_output(job.outputs[1])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "job.get_json_output(2)\n", + "# or\n", + "job.get_json_output(job.outputs[2])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8 (py38)", + "language": "python", + "name": "py38" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/Examples/TestNotebook.ipynb b/Examples/TestNotebook.ipynb new file mode 100644 index 0000000..b5ef126 --- /dev/null +++ b/Examples/TestNotebook.ipynb @@ -0,0 +1,132 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import time;\n", + "import random;\n", + "import os;\n", + "from datetime import datetime" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(str(datetime.now()))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "now = str(datetime.now());\n", + "print(now + \" Starting Notebook execution...\")\n", + "os.system(\"echo '\" + now + \" Starting Notebook execution...'\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# sleep time.\n", + "time.sleep(5);#5sec" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "now = str(datetime.now());\n", + "print(now + \" Now writing HelloWorld.txt file into job directory.\");\n", + "os.system(\"echo '\" + now + \" Now writing HelloWorld.txt file into job directory.'\");\n", + "\n", + "\n", + "jobsPath = None\n", + "with open('/home/idies/keystone.token', 'r') as f:\n", + " jobsPath = f.read().rstrip('\\n')\n", + "\n", + "if jobsPath == \"\" or jobsPath == None: \n", + " raise Exception(now + ' Could not read /home/idies/keystone.token') \n", + " \n", + "print(now + \" Was able to read /home/idies/keystone.token : \" + jobsPath);\n", + "os.system(\"echo '\" + now + \" Was able to read /home/idies/keystone.token : \" + jobsPath +\"'\");\n", + "\n", + "\n", + "target = open(\"HelloWorld.txt\", 'w')\n", + "target.truncate()\n", + "target.write(jobsPath)\n", + "target.write(\"\\n\")\n", + "target.close()\n", + "\n", + "print(now + \" Finished writing HelloWorld.txt file into job directory.\");\n", + "os.system(\"echo '\" + now + \" Finished writing HelloWorld.txt file into job directory.'\");" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "now = str(datetime.now());\n", + "parameters = None;\n", + "with open('parameters.txt') as f:\n", + " parameters = f.read().rstrip('\\n')\n", + "\n", + "if parameters is None:\n", + " raise Exception('Could not read parameters.txt')\n", + " \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "now = str(datetime.now());\n", + "print(now + \" Ending TestNotebook execution.\");\n", + "os.system(\"echo '\" + now + \" Ending TestNotebook execution.\");" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/README.md b/README.md index bc1b760..aefe442 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # SciScript-Python -## Python libraries for Jupyter Notebooks +## SciServer Python libraries for Jupyter Notebooks -This python package provides functions for quick access of [SciServer](http://www.sciserver.org) APIs (web services) and tools. +This python package provides functions for quick access to [SciServer](http://www.sciserver.org) APIs (web services) and tools. [SciServer](http://www.sciserver.org) provides a new online framework for data-intensive scientifc computing in the cloud, where the motto is to bring the computation close where the data is stored, and allow seamless access and sharing of big data sets within the scientific community. @@ -12,11 +12,19 @@ Some SciServer tools you can access with this package: * [CasJobs](http://skyserver.sdss.org/CasJobs): Database storage and querying. +* SciQuery: Advanced Database storage and querying. + * [SciDrive](http://www.scidrive.org/): Drag-and-drop file storage and sharing. * [SkyServer](http://skyserver.sdss.org/): Access to the SDSS astronomical survey. * [SkyQuery](http://www.voservices.net/skyquery): Cross-match of astronomical source catalogs. + + * [Compute Jobs](http://apps.sciserver.org/compute/jobs) and [SciQuery Jobs](http://apps.sciserver.org/sciquery-ui/): Synch and asynch submission of Jupyter notebooks, shell commands and SQL queries. + + * [Files](http://apps.sciserver.org/dashboard/files): Interact with the SciServer file system. + +[API Documentation](https://www.sciserver.org/docs/sciscript-python/) Maintainer: Manuchehr Taghizadeh-Popp. @@ -33,6 +41,8 @@ Authors: Gerard Lemson, Manuchehr Taghizadeh-Popp. ## 3) Installation: +First, add required packages with `pip install -r requirements.txt` + There are 2 possibilities: automatic or manual installation. ### a) Automatic Installation and Update: diff --git a/Tests/TestNotebook.ipynb b/Tests/TestNotebook.ipynb new file mode 100644 index 0000000..51dce8c --- /dev/null +++ b/Tests/TestNotebook.ipynb @@ -0,0 +1,154 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "import time;\n", + "import random;\n", + "import os;\n", + "from datetime import datetime" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "print(str(datetime.now()))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "now = str(datetime.now());\n", + "print(now + \" Starting Notebook execution...\")\n", + "os.system(\"echo '\" + now + \" Starting Notebook execution...'\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "# sleep time.\n", + "time.sleep(5);#5sec" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "now = str(datetime.now());\n", + "print(now + \" Now writing HelloWorld.txt file into job directory.\");\n", + "os.system(\"echo '\" + now + \" Now writing HelloWorld.txt file into job directory.'\");\n", + "\n", + "\n", + "jobsPath = None\n", + "with open('/home/idies/job.path', 'r') as f:\n", + " jobsPath = f.read().rstrip('\\n')\n", + "\n", + "if jobsPath == \"\" or jobsPath == None: \n", + " raise Exception(now + ' Could not read /home/idies/job.path') \n", + " \n", + "print(now + \" Was able to read /home/idies/job.path : \" + jobsPath);\n", + "os.system(\"echo '\" + now + \" Was able to read /home/idies/job.path : \" + jobsPath +\"'\");\n", + "\n", + "\n", + "target = open(jobsPath + \"HelloWorld.txt\", 'w')\n", + "target.truncate()\n", + "target.write(jobsPath)\n", + "target.write(\"\\n\")\n", + "target.close()\n", + "\n", + "print(now + \" Finished writing HelloWorld.txt file into job directory.\");\n", + "os.system(\"echo '\" + now + \" Finished writing HelloWorld.txt file into job directory.'\");" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "now = str(datetime.now());\n", + "parameters = None;\n", + "with open('parameters.txt') as f:\n", + " parameters = f.read().rstrip('\\n')\n", + "\n", + "if parameters is None:\n", + " raise Exception('Could not read parameters.txt')\n", + " \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "now = str(datetime.now());\n", + "print(now + \" Ending TestNotebook execution.\");\n", + "os.system(\"echo '\" + now + \" Ending TestNotebook execution.\");" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/Tests/UnitTests.py b/Tests/UnitTests.py index 76aa138..860ba41 100644 --- a/Tests/UnitTests.py +++ b/Tests/UnitTests.py @@ -1,5 +1,5 @@ #!/usr/bin/python -from SciServer import Authentication, LoginPortal, Config, CasJobs, SkyServer, SkyQuery, SciDrive +from SciServer import Authentication, LoginPortal, Config, CasJobs, SkyServer, SkyQuery, SciDrive, Files, Jobs import unittest2 as unittest import os; import pandas; @@ -13,6 +13,8 @@ Authentication_loginName = '***'; Authentication_loginPassword = '***' +Authentication_login_sharedWithName = '***' +Authentication_login_sharedWithPassword = '***' #skyserver SkyServer_TestQuery = "select top 1 specobjid, ra, dec from specobj order by specobjid" @@ -39,11 +41,39 @@ SciDrive_FileName = "TestFile.csv" SciDrive_FileContent = "Column1,Column2\n4.5,5.5\n" - -#class TestVariables: -# SkyQuery_jobId = None; -# def __init__(self): -# pass +Files_FileServiceName = "FileServiceJHU" +Files_RootVolumeName1 = "Storage" +Files_UserVolumeName1 = "UserVolume555" +Files_RootVolumeName2 = "Storage" +Files_UserVolumeName2 = "UserVolume999" +Files_NewDirectoryName1 = "myNewDirectory555" +Files_NewDirectoryName2 = "myNewDirectory999" +Files_LocalFileName = "MyNewFile.txt" +Files_LocalFileContent = "#ID,Column1,Column2\n1,4.5,5.5" + + +Jobs_DockerComputeDomainName = 'Small Jobs Domain' +Jobs_FileServiceName = "FileServiceJHU" +Jobs_RootVolumeName = "Storage" +Jobs_UserVolumeName = "JobsTestVolume" +Jobs_DirectoryName = "JobsTestDirectory" +Jobs_NotebookName = 'TestNotebook.ipynb' +Jobs_NoteBookOutPutFile = 'HelloWorld.txt' +Jobs_ShellCommand = 'pwd' +Jobs_DockerImageName = 'Python + R' + +Jobs_UserVolumes = [{'name':Jobs_UserVolumeName, 'rootVolumeName':Jobs_RootVolumeName, 'owner':Authentication_loginName, 'needsWriteAccess':True}, + {'name':'scratch', 'rootVolumeName':'Temporary', 'owner':Authentication_loginName, 'needsWriteAccess':True}] +Jobs_DataVolumes = [{'name':'SDSS DAS'}] +Jobs_Parameters = 'param1=1\nparam2=2\nparam3=3' +Jobs_Alias = 'MyNewJob' + +Jobs_SqlQuery = 'select 1;' +Jobs_SqlQueryResult = 'column1\n1\n' +Jobs_RDBComputeDomainName = 'Manga (long)' +Jobs_DatabaseContextName = "manga" +Jobs_RemoteNotebookPath='/home/idies/workspace/' + Jobs_UserVolumeName + '/' + Jobs_DirectoryName + '/' + Jobs_NotebookName +Jobs_QueryResultsFile = 'myQueryResults' class TestAuthentication(unittest.TestCase): @@ -239,6 +269,9 @@ def test_SkyServer_radialSearch(self): self.maxDiff = None; self.assertEqual(SkyServer_RadialSearchResultCSV, df.to_csv(index=False, float_format="%.6f")) + + + def test_SkyServer_rectangularSearch(self): #rectangular search df = SkyServer.rectangularSearch(min_ra=258.3, max_ra=258.31, min_dec=64,max_dec=64.01, dataRelease=SkyServer_DataRelease) @@ -398,6 +431,298 @@ def test_SkyQuery_getDatasetInfo(self): def test_SkyQuery_listDatasetTables(self): tables = SkyQuery.listDatasetTables("MyDB") + + + + +class TestFileService(unittest.TestCase): + + token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword); + + def setUp(self): + pass + + # ******************************************************************************************************* + # Files section + + def test_Files_getFileServices(self): + fileServices = Files.getFileServices(); + self.assertTrue(fileServices.__len__() > 0) + + def test_Files_getFileServicesNames(self): + fileServiceNames = Files.getFileServicesNames(); + self.assertTrue(fileServiceNames.__len__() > 0) + + def test_Files_getFileServicesNames(self): + fileServiceNames = Files.getFileServicesNames(); + found = False; + for fileService in fileServiceNames: + if fileService.get('name') == Files_FileServiceName: + found=True; + + self.assertTrue(found) + + def test_Files_getFileServiceFromName(self): + fileService = Files.getFileServiceFromName(Files_FileServiceName); + self.assertTrue(fileService.get('name') == Files_FileServiceName); + + def test_Files_getRootVolumesInfo(self): + fileService = Files.getFileServiceFromName(Files_FileServiceName); + rootVolumes = Files.getRootVolumesInfo(fileService) + self.assertTrue(rootVolumes.__len__() > 0) + found = False + for rootVolume in rootVolumes: + if rootVolume.get('rootVolumeName') == Files_RootVolumeName1: + found = True + self.assertTrue(found) + + found = False + for rootVolume in rootVolumes: + if rootVolume.get('rootVolumeName') == Files_RootVolumeName2: + found = True + self.assertTrue(found) + + def test_Files_getUserVolumesInfo(self): + fileService = Files.getFileServiceFromName(Files_FileServiceName); + userVolumesInfo = Files.getUserVolumesInfo(fileService) + self.assertTrue(userVolumesInfo.__len__() > 0) + + def test_Files_createUserVolume_deleteUserVolume(self): + fileService = Files.getFileServiceFromName(Files_FileServiceName); + Files.createUserVolume(fileService,"/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1]),quiet=False) + Files.deleteUserVolume(fileService,"/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1]),quiet=False) + + def test_Files_createDir_upload_dirList_download_download_shareUserVolume(self): + + try: + fileService = Files.getFileServiceFromName(Files_FileServiceName); + os.remove(Files_LocalFileName); + Files.deleteUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName1, quiet=True) + Files.deleteUserVolume(fileService, Files_RootVolumeName1, Files_UserVolumeName2, quiet=True) + except: + pass; + + try: + fileService = Files.getFileServiceFromName(Files_FileServiceName); + + Files.createUserVolume(fileService,"/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1]),quiet=False) + Files.createUserVolume(fileService,"/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName2]),quiet=False) + + Files.createDir(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1, Files_NewDirectoryName1])); + Files.createDir(fileService, "/".join([Files_RootVolumeName2, Authentication_loginName, Files_UserVolumeName2, Files_NewDirectoryName2])); + + dirList = Files.dirList(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1, Files_NewDirectoryName1]),level=2) + self.assertTrue(dirList.get('root').get('name') == Files_NewDirectoryName1) + + Files.upload(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1, Files_NewDirectoryName1, Files_LocalFileName]),data=Files_LocalFileContent); + + dirList = Files.dirList(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1, Files_NewDirectoryName1]) ,level=2) + self.assertTrue(dirList.get('root').get('files')[0].get('name') == Files_LocalFileName) + + Files.download(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1,Files_NewDirectoryName1, Files_LocalFileName]),localFilePath=Files_LocalFileName); + + with open(Files_LocalFileName, 'r') as myfile: + downloadedFileContent = myfile.read() + assert(downloadedFileContent == Files_LocalFileContent) + + Files.delete(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1, Files_NewDirectoryName1, Files_LocalFileName])) + + dirList = Files.dirList(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1, Files_NewDirectoryName1]),level=2) + self.assertIsNone(dirList.get('root').get('files')) + + Files.upload(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1, Files_NewDirectoryName1, Files_LocalFileName]), localFilePath=Files_LocalFileName,quiet=False); + + Files.move(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1, Files_NewDirectoryName1, Files_LocalFileName]), fileService, "/".join([Files_RootVolumeName2, Authentication_loginName, Files_UserVolumeName2, Files_NewDirectoryName2, Files_LocalFileName])); + + Files.shareUserVolume(fileService, "/".join([Files_RootVolumeName2, Authentication_loginName, Files_UserVolumeName2]), sharedWith=Authentication_login_sharedWithName, type="USER",allowedActions=["read"]) + + token1 = Authentication.login(Authentication_login_sharedWithName, Authentication_login_sharedWithPassword); + + string = Files.download(fileService, "/".join([Files_RootVolumeName2, Authentication_loginName, Files_UserVolumeName2, Files_NewDirectoryName2, Files_LocalFileName]), format="txt"); + + self.assertTrue(string, Files_LocalFileContent) + + token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword); + + finally: + try: + os.remove(Files_LocalFileName); + Files.deleteUserVolume(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1]),quiet=True) + Files.deleteUserVolume(fileService, "/".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName2]),quiet=True) + except: + pass; + + +class TestJobs(unittest.TestCase): + + token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword); + + + def setUp(self): + pass + + # ******************************************************************************************************* + # Jobs section + + # Docker Jobs ################################################################################################ + + def test_Jobs_getDockerComputeDomains(self): + dockerComputeDomains = Jobs.getDockerComputeDomains() + self.assertTrue(dockerComputeDomains.__len__() > 0) + found = False + for dockerComputeDomain in dockerComputeDomains: + if dockerComputeDomain.get('name') == Jobs_DockerComputeDomainName: + found = True + self.assertTrue(found) + + def test_Jobs_getDockerComputeDomainsNames(self): + dockerComputeDomainsNames = Jobs.getDockerComputeDomainsNames() + self.assertTrue(Jobs_DockerComputeDomainName in dockerComputeDomainsNames) + + def test_Jobs_getDockerComputeDomainFromName(self): + dockerComputeDomain = Jobs.getDockerComputeDomainFromName(Jobs_DockerComputeDomainName) + self.assertTrue(dockerComputeDomain.get('name') in Jobs_DockerComputeDomainName) + + def test_Jobs_submitNotebookJob_cancel_waitForJob_getJobStatus_getJobDescription_submitShellCommandJob(self): + + fileService = Files.getFileServiceFromName(Jobs_FileServiceName); + try: + Files.deleteUserVolume(fileService, Jobs_RootVolumeName, Jobs_UserVolumeName) + except: + pass + + Files.createUserVolume(fileService, Jobs_RootVolumeName, Jobs_UserVolumeName) + Files.upload(fileService, Jobs_RootVolumeName, Jobs_UserVolumeName, + Jobs_DirectoryName + "/" + Jobs_NotebookName, + localFilePath=Jobs_NotebookName); + + dockerComputeDomain = Jobs.getDockerComputeDomainFromName(Jobs_DockerComputeDomainName) + + jobId_1 = Jobs.submitNotebookJob( + '/home/idies/workspace/' + Jobs_UserVolumeName + '/' + Jobs_DirectoryName + '/' + Jobs_NotebookName, + dockerComputeDomain, + Jobs_DockerImageName, + Jobs_UserVolumes, Jobs_DataVolumes, + Jobs_Parameters, Jobs_Alias) + Jobs.cancelJob(jobId_1) + jobStatus = Jobs.getJobStatus(jobId_1) + self.assertTrue(jobStatus.get('status') == 128) + + jobId_2 = Jobs.submitNotebookJob( + Jobs_RemoteNotebookPath, + dockerComputeDomain, + Jobs_DockerImageName, + Jobs_UserVolumes, Jobs_DataVolumes, + Jobs_Parameters, Jobs_Alias) + + jobStatus = Jobs.waitForJob(jobId_2) + self.assertTrue(jobStatus == Jobs.getJobStatus(jobId_2)) + self.assertTrue(jobStatus.get('status') == 32) + + job = Jobs.getJobDescription(jobId_2) + self.assertTrue(job.get('username') == Authentication_loginName) + self.assertTrue(job.get('dockerImageName') == Jobs_DockerImageName) + self.assertTrue(job.get('scriptURI') == Jobs_RemoteNotebookPath) + self.assertTrue(job.get('submitterDID') == Jobs_Alias) + + jobDirectory = job.get('resultsFolderURI') + relativePath = jobDirectory.split('scratch/')[1] + Jobs_NoteBookOutPutFile; + string = Files.download(fileService, 'scratch', '',relativePath,format="txt", userVolumeOwner=Authentication_loginName); + string.rstrip("\n") + self.assertTrue(string, job.get('resultsFolderURI') ) + + jobs = Jobs.getJobsList(top=2) + found = False + for job in jobs: + if jobId_1 == job.get("id"): + found = True; + self.assertTrue(found) + + found = False + for job in jobs: + if jobId_2 == job.get("id"): + found = True; + self.assertTrue(found) + + + jobId = Jobs.submitShellCommandJob(Jobs_ShellCommand, + dockerComputeDomain, + Jobs_DockerImageName, + Jobs_UserVolumes, Jobs_DataVolumes, + Jobs_Alias) + + jobStatus = Jobs.waitForJob(jobId) + self.assertTrue(jobStatus == Jobs.getJobStatus(jobId)) + self.assertTrue(jobStatus.get('status') == 32) + + job = Jobs.getJobDescription(jobId) + self.assertTrue(job.get('username') == Authentication_loginName) + self.assertTrue(job.get('dockerImageName') == Jobs_DockerImageName) + self.assertTrue(job.get('command') == Jobs_ShellCommand) + self.assertTrue(job.get('submitterDID') == Jobs_Alias) + + jobDirectory = job.get('resultsFolderURI') + relativePath = jobDirectory.split('scratch/')[1] + "command.txt"; + string = Files.download(fileService, 'scratch', '',relativePath,format="txt", userVolumeOwner=Authentication_loginName); + string.rstrip("\n") + self.assertTrue(string, job.get('resultsFolderURI') ) + + Files.deleteUserVolume(fileService, Jobs_RootVolumeName, Jobs_UserVolumeName) + + + + # RDB Jobs ################################################################################################ + + def test_Jobs_getRDBComputeDomains(self): + rdbComputeDomains = Jobs.getRDBComputeDomains() + self.assertTrue(rdbComputeDomains.__len__() > 0) + found = False + for rdbComputeDomain in rdbComputeDomains: + if rdbComputeDomain.get('name') == Jobs_RDBComputeDomainName: + found = True + self.assertTrue(found) + + def test_Jobs_getRDBComputeDomainsNames(self): + rdbComputeDomainsNames = Jobs.getRDBComputeDomainsNames() + self.assertTrue(Jobs_RDBComputeDomainName in rdbComputeDomainsNames) + + def test_Jobs_getRDBComputeDomainFromName(self): + rdbComputeDomain = Jobs.getRDBComputeDomainFromName(Jobs_RDBComputeDomainName) + self.assertTrue(rdbComputeDomain.get('name') in Jobs_RDBComputeDomainName) + + + def test_Jobs_submitRDBJob(self): + + rdbComputeDomain = Jobs.getRDBComputeDomainFromName(Jobs_RDBComputeDomainName) + + jobId = Jobs.submitRDBQueryJob(Jobs_SqlQuery, rdbComputeDomain, Jobs_DatabaseContextName, Jobs_QueryResultsFile,Jobs_Alias) + + jobStatus = Jobs.waitForJob(jobId) + self.assertTrue(jobStatus == Jobs.getJobStatus(jobId)) + self.assertTrue(jobStatus.get('status') == 32) + + job = Jobs.getJobDescription(jobId) + self.assertTrue(job.get('username') == Authentication_loginName) + self.assertTrue(job.get('rdbDomainName') == Jobs_RDBComputeDomainName) + self.assertTrue(job.get('databaseContextName') == Jobs_DatabaseContextName) + self.assertTrue(job.get('inputSql') == Jobs_SqlQuery) + self.assertTrue(job.get('submitterDID') == Jobs_Alias) + + fileService = Files.getFileServiceFromName(Jobs_FileServiceName); + jobDirectory = job.get('resultsFolderURI') + relativePath = jobDirectory.split('scratch/')[1] + Jobs_QueryResultsFile + '.csv'; + string = Files.download(fileService, 'scratch', '',relativePath,format="txt", userVolumeOwner=Authentication_loginName); + string.rstrip("\n") + self.assertTrue(string, Jobs_SqlQueryResult) + + + def test_Jobs_getJobDirectory(self): + #TBD + pass; + + + + if __name__ == '__main__': #unittest.main() unittest.TestLoader.sortTestMethodsUsing = lambda x, y: cmp(x,y); @@ -409,4 +734,6 @@ def test_SkyQuery_listDatasetTables(self): suite = testLoader.loadTestsFromTestCase(TestSkyServer); unittest.TextTestRunner(verbosity=2).run(suite) suite = testLoader.loadTestsFromTestCase(TestSciDrive); unittest.TextTestRunner(verbosity=2).run(suite) suite = testLoader.loadTestsFromTestCase(TestSkyQuery); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestFileService); unittest.TextTestRunner(verbosity=2).run(suite) + suite = testLoader.loadTestsFromTestCase(TestJobs); unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/docs_sphinx/SciServer.rst b/docs_sphinx/SciServer.rst index 27d717d..e42ad9e 100644 --- a/docs_sphinx/SciServer.rst +++ b/docs_sphinx/SciServer.rst @@ -14,8 +14,16 @@ SciServer.CasJobs module :undoc-members: :show-inheritance: +SciServer.SciQuery module +------------------------- + +.. automodule:: SciServer.SciQuery + :members: + :undoc-members: + :show-inheritance: + SciServer.Config module ------------------------- +----------------------- .. automodule:: SciServer.Config :members: @@ -23,7 +31,7 @@ SciServer.Config module :show-inheritance: SciServer.Authentication module ------------------------- +------------------------------- .. automodule:: SciServer.Authentication :members: @@ -31,7 +39,7 @@ SciServer.Authentication module :show-inheritance: SciServer.LoginPortal module ------------------------- +---------------------------- .. automodule:: SciServer.LoginPortal :members: @@ -39,7 +47,7 @@ SciServer.LoginPortal module :show-inheritance: SciServer.SciDrive module ------------------------- +------------------------- .. automodule:: SciServer.SciDrive :members: @@ -47,7 +55,7 @@ SciServer.SciDrive module :show-inheritance: SciServer.SkyServer module ------------------------- +-------------------------- .. automodule:: SciServer.SkyServer :members: @@ -55,9 +63,33 @@ SciServer.SkyServer module :show-inheritance: SciServer.SkyQuery module ------------------------- +------------------------- .. automodule:: SciServer.SkyQuery :members: :undoc-members: :show-inheritance: + +SciServer.Files module +---------------------- + +.. automodule:: SciServer.Files + :members: + :undoc-members: + :show-inheritance: + +SciServer.Jobs module +--------------------- + +.. automodule:: SciServer.Jobs + :members: + :undoc-members: + :show-inheritance: + +SciServer.Dask module +--------------------- + +.. automodule:: SciServer.Dask + :members: + :undoc-members: + :show-inheritance: diff --git a/docs_sphinx/conf.py b/docs_sphinx/conf.py index 0d13936..381fc37 100644 --- a/docs_sphinx/conf.py +++ b/docs_sphinx/conf.py @@ -48,16 +48,16 @@ # General information about the project. project = 'SciScript-Python' -copyright = '2017, SciServer' +copyright = '2024, SciServer' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '1.10.0' +version = '2.2.0' # The full version, including alpha/beta/rc tags. -release = '1.10.0' +release = '2.2.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -182,6 +182,7 @@ # Output file base name for HTML help builder. htmlhelp_basename = 'sphinxdoc' +autoclass_content = 'both' # -- Options for LaTeX output --------------------------------------------- @@ -268,7 +269,7 @@ epub_title = '.' epub_author = 'Gerard Lemson, Manuchehr Taghizadeh-Popp' epub_publisher = 'Gerard Lemson, Manuchehr Taghizadeh-Popp' -epub_copyright = '2016, SciServer' +epub_copyright = '2024, SciServer' # The basename for the epub file. It defaults to the project name. #epub_basename = '.' diff --git a/docs_sphinx/index.rst b/docs_sphinx/index.rst index 8ff9b57..e1ec1f7 100644 --- a/docs_sphinx/index.rst +++ b/docs_sphinx/index.rst @@ -4,7 +4,7 @@ contain the root `toctree` directive. Welcome to SciScript Python documentation! -============================= +========================================== Contents: diff --git a/py2/SciServer/Authentication.py b/py2/SciServer/Authentication.py index 4610b49..a02f6c7 100644 --- a/py2/SciServer/Authentication.py +++ b/py2/SciServer/Authentication.py @@ -6,7 +6,7 @@ import requests import os.path import warnings - +import getpass from SciServer import Config @@ -17,6 +17,7 @@ class KeystoneUser: """ id = None userName = None + token = None class Token: """ @@ -26,6 +27,7 @@ class Token: token = Token(); +keystoneUser = KeystoneUser(); def getKeystoneUserWithToken(token): """ @@ -38,25 +40,40 @@ def getKeystoneUserWithToken(token): .. seealso:: Authentication.getToken, Authentication.login, Authentication.setToken. """ - loginURL = Config.AuthenticationURL - if ~loginURL.endswith("/"): - loginURL = loginURL + "/" - loginURL = loginURL + token - getResponse = requests.get(loginURL) - if getResponse.status_code != 200: - raise Exception("Error when getting the keystone user with token " + str(token) +".\nHttp Response from the Authentication API returned status code " + str(getResponse.status_code) + ":\n" + getResponse.content.decode()); + if keystoneUser.token == token and keystoneUser.token is not None: + return keystoneUser; + + else: + taskName = "" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Authentication.getKeystoneUserWithToken" + else: + taskName = "SciScript-Python.Authentication.getKeystoneUserWithToken" + + loginURL = Config.AuthenticationURL + if ~loginURL.endswith("/"): + loginURL = loginURL + "/" + + loginURL = loginURL + token + "?TaskName=" + taskName; - responseJson = json.loads((getResponse.content.decode())) + getResponse = requests.get(loginURL) + if getResponse.status_code != 200: + raise Exception("Error when getting the keystone user with token " + str(token) +".\nHttp Response from the Authentication API returned status code " + str(getResponse.status_code) + ":\n" + getResponse.content.decode()); - ksu = KeystoneUser() - ksu.userName = responseJson["token"]["user"]["name"] - ksu.id = responseJson["token"]["user"]["id"] + responseJson = json.loads((getResponse.content.decode())) - return ksu + ksu = KeystoneUser() + ksu.userName = responseJson["token"]["user"]["name"] + ksu.id = responseJson["token"]["user"]["id"] + keystoneUser.token = token; + keystoneUser.userName = ksu.userName + keystoneUser.id = ksu.id + return ksu -def login(UserName, Password): + +def login(UserName=None, Password=None): """ Logs the user into SciServer and returns the authentication token. This function is useful when SciScript-Python library methods are executed outside the SciServer-Compute environment. @@ -64,17 +81,30 @@ def login(UserName, Password): so the user has to use Authentication.login in order to log into SciServer manually and get the authentication token. Authentication.login also sets the token value in the python instance argument variable "--ident", and as the local object Authentication.token (of class Authentication.Token). - :param UserName: name of the user (string) - :param Password: password of the user (string) + :param UserName: name of the user (string). If not set, then a user name prompt will show. + :param Password: password of the user (string). If not set, then a password prompt will show. :return: authentication token (string) :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. - :example: token = Authentication.login('loginName','loginPassword') + :example: token1 = Authentication.login(); token2 = Authentication.login('loginName','loginPassword'); .. seealso:: Authentication.getKeystoneUserWithToken, Authentication.getToken, Authentication.setToken, Authentication.token. """ - loginURL = Config.AuthenticationURL + taskName = "" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Authentication.Login" + else: + taskName = "SciScript-Python.Authentication.Login" + + loginURL = Config.AuthenticationURL + "?TaskName=" + taskName + + userName = UserName; + password = Password; + if userName is None: + userName = getpass.getpass(prompt="Enter SciServer user name: ") + if password is None: + password = getpass.getpass(prompt="Enter SciServer password: ") - authJson = {"auth":{"identity":{"password":{"user":{"name":UserName,"password":Password}}}}} + authJson = {"auth":{"identity":{"password":{"user":{"name":userName,"password":password}}}}} data = json.dumps(authJson).encode() @@ -101,29 +131,49 @@ def getToken(): """ try: - if token.value is not None: - return token.value + + if Config.isSciServerComputeEnvironment(): + tokenFile = Config.KeystoneTokenPath; # '/home/idies/keystone.token' + if os.path.isfile(tokenFile): + with open(tokenFile, 'r') as f: + _token = f.read().rstrip('\n') + if _token is not None and _token != "": + token.value = _token; + + found = False + ident = identArgIdentifier() + for arg in sys.argv: + if (arg.startswith(ident)): + sys.argv.remove(arg) + sys.argv.append(ident + _token) + found = True + if not found: + sys.argv.append(ident + _token) + + return _token + else: + warnings.warn("In Authentication.getToken: Cannot find token in system token file " + str(Config.KeystoneTokenPath) + ".", Warning, stacklevel=2) + return None; + else: + warnings.warn("In Authentication.getToken: Cannot find system token file " + str(Config.KeystoneTokenPath) + ".", Warning, stacklevel=2) + return None; else: - _token = None - ident = identArgIdentifier() - for arg in sys.argv: - if (arg.startswith(ident)): - _token = arg[len(ident):] - - if _token is not None and _token != "": - setToken(_token) - return _token + if token.value is not None: + return token.value else: - tokenFile = Config.KeystoneTokenPath; # '/home/idies/keystone.token' - if os.path.isfile(tokenFile) : - with open(tokenFile, 'r') as f: - _token = f.read().rstrip('\n') - if _token is not None and _token != "": - setToken(_token) - return _token - else: - warnings.warn("In Authentication.getToken: Keystone token is not defined. Returning a None token value. Keystone token is not stored in the command line argument --ident, nor in the system token file " + str(Config.KeystoneTokenPath) + ".", Warning, stacklevel=2) - return None; + _token = None + ident = identArgIdentifier() + for arg in sys.argv: + if (arg.startswith(ident)): + _token = arg[len(ident):] + + if _token is not None and _token != "": + token.value = _token + return _token + else: + warnings.warn("In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.", Warning, stacklevel=2) + return None; + except Exception as e: raise e; @@ -142,18 +192,20 @@ def setToken(_token): if _token == "": warnings.warn("Authentication token is being set as an empty string.", Warning, stacklevel=2) - token.value = _token; - - found = False - ident = identArgIdentifier() - for arg in sys.argv: - if (arg.startswith(ident)): - sys.argv.remove(arg) + if Config.isSciServerComputeEnvironment(): + warnings.warn("Authentication token cannot be set to arbitary value when inside SciServer-Compute environment.", Warning, stacklevel=2) + else: + token.value = _token; + + found = False + ident = identArgIdentifier() + for arg in sys.argv: + if (arg.startswith(ident)): + sys.argv.remove(arg) + sys.argv.append(ident + _token) + found = True + if not found: sys.argv.append(ident + _token) - found = True - if not found: - sys.argv.append(ident + _token) - def identArgIdentifier(): diff --git a/py2/SciServer/CasJobs.py b/py2/SciServer/CasJobs.py index 6c4b66c..6937684 100644 --- a/py2/SciServer/CasJobs.py +++ b/py2/SciServer/CasJobs.py @@ -10,6 +10,16 @@ from SciServer import Authentication, Config +class Task: + """ + The class TaskName stores the name of the task that executes the API call. + """ + name = None + + +task = Task(); + + def getSchemaName(): """ Returns the WebServiceID that identifies the schema for a user in MyScratch database with CasJobs. @@ -24,7 +34,14 @@ def getSchemaName(): if token is not None and token != "": keystoneUserId = Authentication.getKeystoneUserWithToken(token).id - usersUrl = Config.CasJobsRESTUri + "/users/" + keystoneUserId + + taskName = "" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.getSchemaName" + else: + taskName = "SciScript-Python.CasJobs.getSchemaName" + + usersUrl = Config.CasJobsRESTUri + "/users/" + keystoneUserId + "?TaskName=" + taskName headers={'X-Auth-Token': token,'Content-Type': 'application/json'} getResponse = requests.get(usersUrl,headers=headers) if getResponse.status_code != 200: @@ -51,7 +68,13 @@ def getTables(context="MyDB"): token = Authentication.getToken() if token is not None and token != "": - TablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables" + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.getTables" + else: + taskName = "SciScript-Python.CasJobs.getTables" + + TablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables" + "?TaskName=" + taskName headers={'X-Auth-Token': token,'Content-Type': 'application/json'} @@ -100,15 +123,19 @@ def executeQuery(sql, context="MyDB", format="pandas"): else: raise Exception("Error when executing query. Illegal format parameter specification: " + str(format)); - QueryUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/query" - - TaskName = ""; - if Config.isSciServerComputeEnvironment(): - TaskName = "Compute.SciScript-Python.CasJobs.executeQuery" + taskName = ""; + if task.name is not None: + taskName = task.name; + task.name = None; else: - TaskName = "SciScript-Python.CasJobs.executeQuery" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.executeQuery" + else: + taskName = "SciScript-Python.CasJobs.executeQuery" - query = {"Query": sql, "TaskName": TaskName} + QueryUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/query" + "?TaskName=" + taskName + + query = {"Query": sql, "TaskName": taskName} data = json.dumps(query).encode() @@ -124,8 +151,16 @@ def executeQuery(sql, context="MyDB", format="pandas"): if (format == "readable") or (format == "StringIO"): return StringIO(postResponse.content.decode()) elif format == "pandas": - r=json.loads(postResponse.content.decode()) - return pandas.DataFrame(r['Result'][0]['Data'],columns=r['Result'][0]['Columns']) + r = json.loads(postResponse.content.decode()) + if len(r['Result']) > 1: + res = [] + for result in r['Result']: + res.append(pandas.DataFrame(result['Data'], columns=result['Columns'])) + + return res + else: + return pandas.DataFrame(r['Result'][0]['Data'], columns=r['Result'][0]['Columns']) + elif format == "csv": return postResponse.content.decode() elif format == "dict": @@ -154,15 +189,15 @@ def submitJob(sql, context="MyDB"): token = Authentication.getToken() if token is not None and token != "": - QueryUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/jobs" - - TaskName = ""; + taskName = ""; if Config.isSciServerComputeEnvironment(): - TaskName = "Compute.SciScript-Python.CasJobs.submitJob" + taskName = "Compute.SciScript-Python.CasJobs.submitJob" else: - TaskName = "SciScript-Python.CasJobs.submitJob" + taskName = "SciScript-Python.CasJobs.submitJob" + + QueryUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/jobs" + "?TaskName=" + taskName - query = {"Query": sql, "TaskName": TaskName} + query = {"Query": sql, "TaskName": taskName} data = json.dumps(query).encode() @@ -193,7 +228,13 @@ def getJobStatus(jobId): token = Authentication.getToken() if token is not None and token != "": - QueryUrl = Config.CasJobsRESTUri + "/jobs/" + str(jobId) + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.getJobStatus" + else: + taskName = "SciScript-Python.CasJobs.getJobStatus" + + QueryUrl = Config.CasJobsRESTUri + "/jobs/" + str(jobId) + "?TaskName=" + taskName headers={'X-Auth-Token': token,'Content-Type': 'application/json'} @@ -220,7 +261,13 @@ def cancelJob(jobId): token = Authentication.getToken() if token is not None and token != "": - QueryUrl = Config.CasJobsRESTUri + "/jobs/" + str(jobId) + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.cancelJob" + else: + taskName = "SciScript-Python.CasJobs.cancelJob" + + QueryUrl = Config.CasJobsRESTUri + "/jobs/" + str(jobId) + "?TaskName=" + taskName headers={'X-Auth-Token': token,'Content-Type': 'application/json'} @@ -233,12 +280,13 @@ def cancelJob(jobId): raise Exception("User token is not defined. First log into SciServer.") -def waitForJob(jobId, verbose=True): +def waitForJob(jobId, verbose=False, pollTime = 5): """ - Queries the job status from casjobs every 2 seconds and waits for the casjobs job to return a status of 3, 4, or 5 (Cancelled, Failed or Finished, respectively). + Queries regularly the job status and waits until the job is completed. :param jobId: id of job (integer) - :param verbose: if True, will print "wait" messages on the screen while the job is not done. If False, will suppress printing messages on the screen. + :param verbose: if True, will print "wait" messages on the screen while the job is still running. If False, will suppress the printing of messages on the screen. + :param pollTime: idle time interval (integer, in seconds) before querying again for the job status. Minimum value allowed is 5 seconds. :return: After the job is finished, returns a dictionary object containing the job status and related metadata. The "Status" field can be equal to 0 (Ready), 1 (Started), 2 (Canceling), 3(Canceled), 4 (Failed) or 5 (Finished). :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the CasJobs API returns an error. :example: CasJobs.waitForJob(CasJobs.submitJob("select 1")) @@ -247,6 +295,7 @@ def waitForJob(jobId, verbose=True): """ try: + minPollTime = 5 # in seconds complete = False waitingStr = "Waiting..." @@ -266,14 +315,14 @@ def waitForJob(jobId, verbose=True): #print back, print "Done!" else: - time.sleep(2) + time.sleep(max(minPollTime,pollTime)); return jobDesc except Exception as e: raise e; -def writeFitsFileFromQuery(fileName, queryString, context="MyDB"): +def writeFitsFileFromQuery(fileName, queryString, context="MyDB"): """ Executes a quick CasJobs query and writes the result to a local Fits file (http://www.stsci.edu/institute/software_hardware/pyfits). @@ -287,6 +336,13 @@ def writeFitsFileFromQuery(fileName, queryString, context="MyDB"): .. seealso:: CasJobs.submitJob, CasJobs.getJobStatus, CasJobs.executeQuery, CasJobs.getPandasDataFrameFromQuery, CasJobs.getNumpyArrayFromQuery """ try: + + + if Config.isSciServerComputeEnvironment(): + task.name = "Compute.SciScript-Python.CasJobs.writeFitsFileFromQuery" + else: + task.name = "SciScript-Python.CasJobs.writeFitsFileFromQuery" + bytesio = executeQuery(queryString, context=context, format="fits") theFile = open(fileName, "w+b") @@ -312,6 +368,12 @@ def getPandasDataFrameFromQuery(queryString, context="MyDB"): .. seealso:: CasJobs.submitJob, CasJobs.getJobStatus, CasJobs.executeQuery, CasJobs.writeFitsFileFromQuery, CasJobs.getNumpyArrayFromQuery """ try: + + if Config.isSciServerComputeEnvironment(): + task.name = "Compute.SciScript-Python.CasJobs.getPandasDataFrameFromQuery" + else: + task.name = "SciScript-Python.CasJobs.getPandasDataFrameFromQuery" + cvsResponse = executeQuery(queryString, context=context,format="readable") #if the index column is not specified then it will add it's own column which causes problems when uploading the transformed data @@ -337,7 +399,13 @@ def getNumpyArrayFromQuery(queryString, context="MyDB"): """ try: + if Config.isSciServerComputeEnvironment(): + task.name = "Compute.SciScript-Python.CasJobs.getNumpyArrayFromQuery" + else: + task.name = "SciScript-Python.CasJobs.getNumpyArrayFromQuery" + dataFrame = getPandasDataFrameFromQuery(queryString, context) + return dataFrame.as_matrix() except Exception as e: @@ -359,6 +427,12 @@ def uploadPandasDataFrameToTable(dataFrame, tableName, context="MyDB"): .. seealso:: CasJobs.uploadCSVDataToTable """ try: + + if Config.isSciServerComputeEnvironment(): + task.name = "Compute.SciScript-Python.CasJobs.uploadPandasDataFrameToTable" + else: + task.name = "SciScript-Python.CasJobs.uploadPandasDataFrameToTable" + if dataFrame.index.name is not None and dataFrame.index.name != "": sio = dataFrame.to_csv().encode("utf8") else: @@ -387,7 +461,18 @@ def uploadCSVDataToTable(csvData, tableName, context="MyDB"): #if (Config.executeMode == "debug"): # print "Uploading ", sys.getsizeof(CVSdata), "bytes..." - tablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables/" + tableName + + taskName = ""; + if task.name is not None: + taskName = task.name; + task.name = None; + else: + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.uploadCSVDataToTable" + else: + taskName = "SciScript-Python.CasJobs.uploadCSVDataToTable" + + tablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables/" + tableName + "?TaskName=" + taskName headers={} headers['X-Auth-Token']= token diff --git a/py2/SciServer/Config.py b/py2/SciServer/Config.py index 36981ec..c2b0c70 100644 --- a/py2/SciServer/Config.py +++ b/py2/SciServer/Config.py @@ -1,3 +1,5 @@ +import json +import os """ The SciServer.Config module contains important parameters for the correct functioning of the SciServer package.\n Although these parameters must be set/defined by the admin or user before the installation of the package, they can also be accessed and changed on-the-fly while on the python session.\n @@ -12,6 +14,8 @@ - **Config.SkyServerWSurl**: defines the base URL of the SkyServer web service API (string). E.g., "https://skyserver.sdss.org" +- **Config.RacmApiURL**: defines the base URL of the RACM API (string). E.g., "https://www.sciserver.org/racm" + - **Config.DataRelease**: defines the SDSS data release (string), to be used to build the full SkyServer API url along with Config.SkyServerWSurl. E.g., "DR13" - **Config.KeystoneTokenPath**: defines the local path (string) to the file containing the user's authentication token in the SciServer-Compute environment. E.g., "/home/idies/keystone.token". Unlikely to change since it is hardcoded in SciServer-Compute. @@ -20,18 +24,39 @@ """ # URLs for accessing SciServer web services (API endpoints) CasJobsRESTUri = "https://skyserver.sdss.org/CasJobs/RestApi" -AuthenticationURL = "https://portal.sciserver.org/login-portal/keystone/v3/tokens" +AuthenticationURL = "https://apps.sciserver.org/login-portal/keystone/v3/tokens" SciDriveHost = "https://www.scidrive.org" SkyQueryUrl = "http://voservices.net/skyquery/Api/V1" SkyServerWSurl = "https://skyserver.sdss.org" -DataRelease = "DR13" -KeystoneTokenPath = "/home/idies/keystone.token" #this path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment -version = "sciserver-v1.10.0" #sciserver release version +RacmApiURL = "https://apps.sciserver.org/racm" +DataRelease = "DR14" +KeystoneTokenPath = "/home/idies/keystone.token" #the path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment +version = "sciserver-v2.2.0" #sciserver release version +ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. +def _load_config(filename): + if os.path.exists(filename): + with open(filename) as f: + _config_data = json.load(f) + global CasJobsRESTUri, AuthenticationURL, SciDriveHost, SkyQueryUrl, SkyServerWSurl + global RacmApiURL, DataRelease, KeystoneTokenPath, version, ComputeJobDirectoryFile + CasJobsRESTUri = _config_data.get('CasJobsRESTUri', CasJobsRESTUri) + AuthenticationURL = _config_data.get('AuthenticationURL', AuthenticationURL) + SciDriveHost = _config_data.get('SciDriveHost', SciDriveHost) + SkyQueryUrl = _config_data.get('SkyQueryUrl', SkyQueryUrl) + SkyServerWSurl = _config_data.get('SkyServerWSurl', SkyServerWSurl) + RacmApiURL = _config_data.get('RacmApiURL', RacmApiURL) + DataRelease = _config_data.get('DataRelease', DataRelease) + KeystoneTokenPath = _config_data.get('KeystoneTokenPath', KeystoneTokenPath) + version = _config_data.get('version', version) + ComputeJobDirectoryFile = _config_data.get('ComputeJobDirectoryFile', ComputeJobDirectoryFile) +_CONFIG_DIR = os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')) +_SCISERVER_SYSTEM_CONFIG_DIR = '/etc/' # will not likely exist on non *nix systems +for config_dir in [_SCISERVER_SYSTEM_CONFIG_DIR, _CONFIG_DIR]: + _load_config(os.path.join(config_dir, 'sciserver', 'sciscript.json')) # returns TRUE if the library is running inside the SciServer-Compute, and FALSE if not -import os; def isSciServerComputeEnvironment(): """ Checks whether the library is being run within the SciServer-Compute environment. diff --git a/py2/SciServer/Files.py b/py2/SciServer/Files.py new file mode 100644 index 0000000..1c4d39e --- /dev/null +++ b/py2/SciServer/Files.py @@ -0,0 +1,703 @@ +__author__ = 'mtaghiza' + +from SciServer import Authentication, Config +import requests +import json +from io import StringIO +from io import BytesIO +import warnings +import os + + +def getFileServices(verbose=True): + """ + Gets the definitions of file services that a user is able to access. A FileService represents a file system that contains root volumes accessible to the user for public/private data storage. Within each rootVolume, users can create sharable userVolumes for storing files. + + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: list of dictionaries, where each dictionary represents the description of a FileService that the user is able to access. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServices = Files.getFileServices(); + + .. seealso:: Files.getFileServiceFromName + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.getFileServices" + else: + taskName = "SciScript-Python.Files.getFileServices" + + url = Config.RacmApiURL + "/storem/fileservices?TaskName="+taskName; + + headers = {'X-Auth-Token': token} + res = requests.get(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + fileServices = []; + fileServicesAPIs = json.loads(res.content.decode()) + for fileServicesAPI in fileServicesAPIs: + url = fileServicesAPI.get("apiEndpoint") + name = fileServicesAPI.get("name") + url = url + "api/volumes/?TaskName="+taskName; + try: + res = requests.get(url, headers=headers) + except: + if verbose: + warnings.warn("Error when getting definition of FileService named '" + name + "' with API URL '" + fileServicesAPI.get("apiEndpoint") + "'. This FileService might be not available", Warning, stacklevel=2) + + if res.status_code >= 200 and res.status_code < 300: + fileServices.append(json.loads(res.content.decode())); + else: + if verbose: + warnings.warn("Error when getting definition of FileService named '" + name + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode(),Warning, stacklevel=2) + return fileServices; + else: + raise Exception("Error when getting the list of FileServices.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + + +def getFileServicesNames(fileServices=None, verbose=True): + """ + Returns the names and description of the fileServices available to the user. + + :param fileServices: a list of FileService objects (dictionaries), as returned by Files.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: an array of dicts, where each dict has the name and description of a file service available to the user. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServiceNames = Files.getFileServicesNames(); + + .. seealso:: Files.getFileServices + """ + + if fileServices is None: + fileServices = getFileServices(verbose); + + fileServiceNames = []; + for fileService in fileServices: + fileServiceNames.append({"name":fileService.get('name'),"description":fileService.get('description')}) + + return fileServiceNames + + + + + +def getFileServiceFromName(fileServiceName, fileServices=None, verbose=True): + """ + Returns a FileService object, given its registered name. + + :param fileServiceName: name of the FileService, as shown within the results of Files.getFileServices() + :param fileServices: a list of FileService objects (dictionaries), as returned by Files.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: a FileService object (dictionary) that defines a FileService. A list of these kind of objects available to the user is returned by the function Jobs.getFileServices(). If no fileService can be found, then returns None. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileService = Files.getFileServiceFromName('FileServiceAtJHU'); + + .. seealso:: Files.getFileServices + """ + + if fileServiceName is None: + raise Exception("fileServiceName is not defined.") + else: + if fileServices is None: + fileServices = getFileServices(verbose); + + if fileServices.__len__() > 0: + for fileService in fileServices: + if fileServiceName == fileService.get('name'): + return fileService; + + if verbose: + warnings.warn("FileService of name '" + fileServiceName + "' is not available to the user or does not exist.", Warning, stacklevel=2) + return None + else: + if verbose: + warnings.warn("There are no FileServices available for the user.", Warning, stacklevel=2) + return None + + + + +def __getFileServiceAPIUrl(fileService): + """ + Gets the API endpoint URL of a FileService. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :return: API endpoint URL of the FileService (string). + :example: fileServiceAPIUrl = Files.__getFileServiceAPIUrl(); + + .. seealso:: Files.getFileServiceFromName + """ + + url = None; + if type(fileService) == type(""): + fileServices = getFileServices(False); + _fileService = getFileServiceFromName(fileService, fileServices, verbose=False); + url = _fileService.get("apiEndpoint"); + + else: + url = fileService.get("apiEndpoint"); + + if not url.endswith("/"): + url = url + "/" + + return url + + +def getRootVolumesInfo(fileService, verbose=True): + """ + Gets the names and descriptions of root volumes available to the user in a particular FileService. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: list of dictionaries, where each dictionary contains the name and description of a root volume. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServices = Files.getFileServices(); rootVolumesInfo = getRootVolumesInfo(fileServices[0]) + + .. seealso:: Files.getUserVolumesInfo + """ + if type(fileService) == str: + fileService = getFileServiceFromName(fileService, verbose=verbose) + + rootVolumes = [] + for rootVolume in fileService.get("rootVolumes"): + rootVolumes.append({"rootVolumeName":rootVolume.get("name"), "rootVolumeDescription":rootVolume.get("description")}) + + return rootVolumes + +def getDataVolumesInfo(fileService, verbose=True): + """ + Gets the names and descriptions of data volumes available to the user in a particular FileService. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: list of dictionaries, where each dictionary contains the name and description of a data volume. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServices = Files.getFileServices(); dataVolumesInfo = getDataVolumesInfo(fileServices[0]) + + .. seealso:: Files.getRootVolumesInfo + """ + if type(fileService) == str: + fileService = getFileServiceFromName(fileService, verbose=verbose) + + dataVolumes = [] + for dataVolume in fileService.get("dataVolumes"): + dataVolumes.append({"dataVolumeName":dataVolume.get("name"), "dataVolumeDescription":dataVolume.get("description")}) + + return dataVolumes + + +def getUserVolumesInfo(fileService, rootVolumeName = None, verbose=True): + """ + Gets the names definitions the RootVolumes available in a particular FileService, and of the file services that a user is able to access. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolumeName: name of root Volume (string) for which the user volumes are fetched. If set to None, then user volumes in all root folders are fetched. + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: list of dictionaries, where each dictionary contains the name and description of a root volumes that a user is able to access. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServices = Files.getFileServices(); rootVolumeNames = Files.getUserVolumesInfo(fileServices[0]) + + .. seealso:: Files.getRootVolumes, + """ + if type(fileService) == str: + fileService = getFileServiceFromName(fileService, verbose=verbose) + + userVolumes = []; + for rootVolume in fileService.get("rootVolumes"): + for userVolume in rootVolume.get('userVolumes'): + path=os.path.join(rootVolume.get('name'),userVolume.get('owner'),userVolume.get('name')) + if rootVolumeName is not None: + if rootVolume.get('name') == rootVolumeName: + userVolumes.append({"userVolumeName": userVolume.get('name'), "path":path, "userVolumeDescription": userVolume.get('description'), "rootVolumeName":rootVolume.get("name"), "rootVolumeDescription":rootVolume.get("description")}) + + else: + userVolumes.append({"userVolumeName": userVolume.get('name'), "path":path,"userVolumeDescription": userVolume.get('description'),"rootVolumeName": rootVolume.get("name"), "rootVolumeDescription": rootVolume.get("description")}) + + + return userVolumes + + + +def splitPath(path, fileService): + """ + If the topVolume in the 'path' is a rootVolume, then this function splits the path of the form rootVolume/userVolumeOwner/userVolume/relativePath/... and returns the tuple (rootVolume, userVolumeOwner, userVolume, relativePath, True). On the other hand, + if the topVolume in the 'path' is a dataVolume, then this function splits the path of the form dataVolume/userVolumeOwner/userVolume/relativePath/... and returns the tuple (dataVolume, None, None, relativePath, False). + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: file system path (string), starting from the root volume level or data volume level. Example: rootVolume/userVolumeOwner/userVolume/relativePath... or dataVolume/relativePath... + :return: a tuple containing the components: (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeADataVolume) + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.splitPath("Storage/myUserName/persistent/my/Relative/Path", fileServices[0]); + + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName + """ + if path.startswith("/"): + path = path[1:] + + + path = path.split("/") + if len(path) == 0: + raise Exception("path variable does not conform with the format 'rootVolume/userVolumeOwner/userVolume/relativePath...' or format 'dataVolume/relativePath...'") + + topVolume = path[0] + isTopVolumeARootVolume = False + + for rootVolume in fileService.get("rootVolumes"): + if rootVolume.get("name") == topVolume: + isTopVolumeARootVolume = True + + if isTopVolumeARootVolume: + if len(path) < 3: + raise Exception("path variable does not conform with the format 'rootVolume/userVolumeOwner/userVolume/relativePath...'") + + rootVolume = path[0] + userVolumeOwner = path[1] + userVolume = path[2] + relativePath = "/".join(path[3:]) + return (rootVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) + else: + if len(path) < 2: + raise Exception("path variable does not conform with the format 'dataVolume/relativePath...'") + + dataVolume = path[0] + relativePath = "/".join(path[1:]) + return (dataVolume, None, None, relativePath, isTopVolumeARootVolume) + + + + +def createUserVolume(fileService, path, quiet=True): + """ + Create a user volume. + + :param fileService: name of fileService (string), or object (dictionary) that defines the file service that contains the user volume. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: path (in the remote file service) to the user volume (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume + :param quiet: if set to False, will throw an error if the User Volume already exists. If True, won't throw an error. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.createUserVolume(fileServices[0], "volumes","newUserVolume"); + + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.createUserVolume" + else: + taskName = "SciScript-Python.Files.createUserVolume" + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (rootVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path); + + url = __getFileServiceAPIUrl(fileService) + "api/volume/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?quiet="+str(quiet) + "&TaskName="+taskName; + + headers = {'X-Auth-Token': token} + res = requests.put(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + pass; + else: + raise Exception("Error when creating user volume '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def deleteUserVolume(fileService, path, quiet=True): + """ + Delete a user volume. + + :param fileService: name of fileService (string), or object (dictionary) that defines the file service that contains the root volume. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: path (in the remote file service) to the user volume (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume + :param quiet: If set to False, it will throw an error if user volume does not exists. If set to True. it will not throw an error. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.deleteUserVolume("volumes","newUserVolume",fileServices[0]); + + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.createUserVolume" + else: + taskName = "SciScript-Python.Files.createUserVolume" + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (rootVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + + url = __getFileServiceAPIUrl(fileService) + "api/volume/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?quiet="+str(quiet)+"&TaskName="+taskName; + + headers = {'X-Auth-Token': token} + res = requests.delete(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + pass + else: + raise Exception("Error when deleting user volume '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def createDir(fileService, path, quiet=True): + """ + Create a directory. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: path (in the remote file service) to the directory (string), starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/directory or dataVolume/directory + :param quiet: If set to False, it will throw an error if the directory already exists. If set to True. it will not throw an error. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.createDir(fileServices[0], "Storage/myUserName/persistent/myNewDir"); + + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.createDir" + else: + taskName = "SciScript-Python.Files.createDir" + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/folder/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName; + else: + url = __getFileServiceAPIUrl(fileService) + "api/folder/" + topVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName; + + headers = {'X-Auth-Token': token} + res = requests.put(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + pass; + else: + raise Exception("Error when creating directory '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def upload(fileService, path, data="", localFilePath=None, quiet=True): + """ + Uploads data or a local file into a path defined in the file system. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: path (in the remote file service) to the destination file (string), starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/destinationFile.txt or dataVolume/destinationFile.txt + :param data: string containing data to be uploaded, in case localFilePath is not set. + :param localFilePath: path to a local file to be uploaded (string), + :param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume. + :param quiet: If set to False, it will throw an error if the file already exists. If set to True. it will not throw an error. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.upload(fileServices[0], "myRootVolume/myUserName/myUserVolume/myUploadedFile.txt", None, localFilePath="/myFile.txt"); + + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.UploadFile" + else: + taskName = "SciScript-Python.Files.UploadFile" + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/file/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName + else: + url = __getFileServiceAPIUrl(fileService) + "api/file/" + topVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName + + headers = {'X-Auth-Token': token} + + if localFilePath is not None and localFilePath != "": + with open(localFilePath, "rb") as file: + res = requests.put(url, data=file, headers=headers, stream=True) + else: + if data != None: + res = requests.put(url, data=data, headers=headers, stream=True) + else: + raise Exception("Error: No local file or data specified for uploading."); + + if res.status_code >= 200 and res.status_code < 300: + pass; + else: + raise Exception("Error when uploading file to '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def download(fileService, path, localFilePath=None, format="txt", quiet=True): + """ + Downloads a file from the remote file system into the local file system, or returns the file content as an object in several formats. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: String defining the path (in the remote file service) of the file to be downloaded, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/fileToBeDownloaded.txt or dataVolume/fileToBeDownloaded.txt + :param localFilePath: local destination path of the file to be downloaded. If set to None, then an object of format 'format' will be returned. + :param format: name (string) of the returned object's type (if localFilePath is not defined). This parameter can be "StringIO" (io.StringIO object containing readable text), "BytesIO" (io.BytesIO object containing readable binary data), "response" ( the HTTP response as an object of class requests.Response) or "txt" (a text string). If the parameter 'localFilePath' is defined, then the 'format' parameter is not used and the file is downloaded to the local file system instead. + :param userVolumeOwner: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. + :param quiet: If set to False, it will throw an error if the file already exists. If set to True. it will not throw an error. + :return: If the 'localFilePath' parameter is defined, then it will return True when the file is downloaded successfully in the local file system. If the 'localFilePath' is not defined, then the type of the returned object depends on the value of the 'format' parameter (either io.StringIO, io.BytesIO, requests.Response or string). + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.upload(fileServices[0], "Storage/myUserName/persistent/fileToBeDownloaded.txt", localFilePath="/fileToBeDownloaded.txt"); + + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.upload, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + + if localFilePath is not None: + if os.path.isfile(localFilePath) and not quiet: + raise Exception("Error when downloading '" + str(path) + "' from file service '" + str(fileService.get("name")) + "'. Local file '" + localFilePath + "' already exists."); + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.DownloadFile" + else: + taskName = "SciScript-Python.Files.DownloadFile" + + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/file/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?TaskName=" + taskName; + else: + url = __getFileServiceAPIUrl(fileService) + "api/file/" + topVolume + "/" + relativePath + "?TaskName=" + taskName; + + headers = {'X-Auth-Token': token} + + res = requests.get(url, stream=True, headers=headers) + + if res.status_code < 200 or res.status_code >= 300: + raise Exception("Error when downloading '" + str(path) + "' from file service '" + str(fileService.get("name")) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + + if localFilePath is not None and localFilePath != "": + + bytesio = BytesIO(res.content) + theFile = open(localFilePath, "w+b") + theFile.write(bytesio.read()) + theFile.close() + return True + + else: + + if format is not None and format != "": + if format == "StringIO": + return StringIO(res.content.decode()) + if format == "txt": + return res.content.decode() + elif format == "BytesIO": + return BytesIO(res.content) + elif format == "response": + return res; + else: + raise Exception("Unknown format '" + format + "' when trying to download from remote File System the file " + str(relativePath) + ".\n"); + else: + raise Exception("Wrong format parameter value\n"); + + else: + raise Exception("User token is not defined. First log into SciServer.") + + + +def dirList(fileService, path, level=1, options=''): + """ + Lists the contents of a directory. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: String defining the path (in the remote file service) of the directory to be listed, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/directoryToBeListed or dataVolume/directoryToBeListed + :param level: amount (int) of listed directory levels that are below or at the same level to that of the relativePath. + :param options: string of file filtering options. + :return: dictionary containing the directory listing. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); dirs = Files.dirList(fileServices[0], "Storage/myUserName/persistent/", level=2); + + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.createDir + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.dirList" + else: + taskName = "SciScript-Python.Files.dirList" + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/jsontree/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?options=" + options + "&level=" + str(level) + "&TaskName=" + taskName; + else: + url = __getFileServiceAPIUrl(fileService) + "api/jsontree/" + topVolume + "/" + relativePath + "?options=" + options + "&level=" + str(level) + "&TaskName=" + taskName; + + headers = {'X-Auth-Token': token} + res = requests.get(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + return json.loads(res.content.decode()); + else: + raise Exception("Error when listing contents of '" + str(path) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def move(fileService, path, destinationFileService, destinationPath, replaceExisting=True, doCopy=True): + """ + Moves or copies a file or folder. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: String defining the origin path (in the remote fileService) of the file or directory to be copied/moved, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/fileToBeMoved.txt or dataVolume/fileToBeMoved.txt + :param destinationFileService: name of fileService (string), or object (dictionary) that defines a destination file service (where the file is moved/copied into). A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param destinationRelativePath: String defining the destination path (in the remote destinationFileService) of the file or directory to be copied/moved, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume/recentlyMovedFile.txt. + :param replaceExisting: If set to False, it will throw an error if the file already exists, If set to True, it will not throw and eeror in that case. + :param doCopy: if set to True, then it will copy the file or folder. If set to False, then the file or folder will be moved. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.move(fileServices[0], "Storage/myUserName/persistent/myFile.txt", fileServices[0], "Storage/myUserName/persistent/myFile2.txt") + + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.upload, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.Move" + else: + taskName = "SciScript-Python.Files.Move" + + if type(fileService) == str: + destinationFileServiceName = destinationFileService + fileService = getFileServiceFromName(fileService) + else: + destinationFileServiceName = destinationFileService['name'] + + + if type(destinationFileService) == str: + destinationFileService = getFileServiceFromName(destinationFileService) + + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + (destinationTopVolume, destinationUserVolumeOwner, destinationUserVolume, destinationRelativePath, isDestinationTopVolumeARootVolume) = splitPath(destinationPath, destinationFileService); + + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/data/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName; + jsonDict = {'destinationPath': destinationRelativePath, + 'destinationRootVolume': destinationTopVolume, + 'destinationUserVolume': destinationUserVolume, + 'destinationOwnerName': destinationUserVolumeOwner, + 'destinationFileService': destinationFileServiceName}; + else: + url = __getFileServiceAPIUrl(fileService) + "api/data/" + topVolume + "/" + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName; + jsonDict = {'destinationPath': destinationRelativePath, + 'destinationDataVolume': destinationTopVolume, + 'destinationFileService': destinationFileServiceName}; + + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.put(url, stream=True, headers=headers, json=jsonDict) + + if res.status_code < 200 or res.status_code >= 300: + raise Exception("Error when moving '" + str(path) + "' in file service '" + str(fileService.get("name")) + "' to '" + str(destinationPath) + "' in file service '" + str(destinationFileService.get("name")) + "'. \nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + + else: + raise Exception("User token is not defined. First log into SciServer.") + + + +def delete(fileService, path, quiet=True): + """ + Deletes a directory or file in the File System. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: String defining the path (in the remote fileService) of the file or directory to be deleted, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/fileToBeDeleted.txt or dataVolume/fileToBeDeleted.txt + :param quiet: If set to False, it will throw an error if the file does not exist. If set to True. it will not throw an error. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.delete(fileServices[0], "Storage/myUserName/persistent/myUselessFile.txt"); + + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.delete" + else: + taskName = "SciScript-Python.Files.delete" + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/data/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName + else: + url = __getFileServiceAPIUrl(fileService) + "api/data/" + topVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName + + headers = {'X-Auth-Token': token} + res = requests.delete(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + pass; + else: + raise Exception("Error when deleting '" + str(path) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + + +def shareUserVolume(fileService, path, sharedWith, allowedActions, type="USER"): + """ + Shares a user volume with another user or group + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: String defining the path (in the remote fileService) of the user volume to be shared, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume. + :param sharedWith: name (string) of user or group that the user volume is shared with. + :param allowedActions: array of strings defining actions the user or group is allowed to do with respect to the shared user volume. E.g.: ["read","write","grant","delete"]. The "grant" action means that the user or group can also share the user volume with another user or group. The "delete" action meand ability to delete the user volume (use with care). + :param type: type (string) of the entity defined by the "sharedWith" parameter. Can be set to "USER" or "GROUP". + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.shareUserVolume(getFileServices[0], "Storage/myUserName/myUSerVolume", "userName", ["read","write"], type="USER"); + + .. seealso:: Files.getFileServices(), Files.getFilrmsieServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.ShareUserVolume" + else: + taskName = "SciScript-Python.Files.ShareUserVolume" + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (rootVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + + + data = [{'name': sharedWith, 'type':type, 'allowedActions': allowedActions }] + body = json.dumps(data).encode() + + url = __getFileServiceAPIUrl(fileService) + "api/share/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?TaskName="+taskName + + headers = {'X-Auth-Token': token,'Content-Type':'application/json'} + res = requests.patch(url, headers=headers, data=body) + + if res.status_code >= 200 and res.status_code < 300: + pass; + else: + raise Exception("Error when sharing userVolume '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py new file mode 100644 index 0000000..76f2e70 --- /dev/null +++ b/py2/SciServer/Jobs.py @@ -0,0 +1,663 @@ +__author__ = 'mtaghiza' + +import sys +import os; +import os.path; +from SciServer import Authentication, Config +import requests +import json +import time; + + + +def getDockerComputeDomains(): + """ + Gets a list of all registered Docker compute domains that the user has access to. + + :return: a list of dictionaries, each one containing the definition of a Docker compute domain. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: dockerComputeDomains = Jobs.getDockerComputeDomains(); + + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getRDBComputeDomains, Jobs.cancelJob + """ + token = Authentication.getToken() + if token is not None and token != "": + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.getDockerComputeDomains" + else: + taskName = "SciScript-Python.Jobs.getDockerComputeDomains" + + url = Config.RacmApiURL + "/jobm/rest/computedomains?batch=true&interactive=false&TaskName=" + taskName + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting Docker Compute Domains from JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def getDockerComputeDomainsNames(dockerComputeDomains=None): + """ + Returns the names of the docker compute domains available to the user. + + :param dockerComputeDomains: a list of dockerComputeDomain objects (dictionaries), as returned by Jobs.getDockerComputeDomains(). If not set, then an extra internal call to Jobs.getDockerComputeDomains() is made. + :return: an array of strings, each being the name of a docker compute domain available to the user. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: dockerComputeDomainsNames = Files.getDockerComputeDomainsNames(); + + .. seealso:: Files.getDockerComputeDomains + """ + if dockerComputeDomains is None: + dockerComputeDomains = getDockerComputeDomains(); + + dockerComputeDomainsNames = []; + for dockerComputeDomain in dockerComputeDomains: + dockerComputeDomainsNames.append(dockerComputeDomain.get('name')) + + return dockerComputeDomainsNames; + + +def getDockerComputeDomainFromName(dockerComputeDomainName, dockerComputeDomains = None): + """ + Returns a DockerComputeDomain object, given its registered name. + + :param dockerComputeDomainName: name of the DockerComputeDomainName, as shown within the results of Jobs.getDockerComputeDomains() + :param dockerComputeDomains: a list of dockerComputeDomain objects (dictionaries), as returned by Jobs.getDockerComputeDomains(). If not set, then an internal call to Jobs.getDockerComputeDomains() is made. + :return: a DockerComputeDomain object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: dockerComputeDomain = Jobs.getDockerComputeDomainFromName('dockerComputeDomainAtJHU'); + + .. seealso:: Jobs.getDockerComputeDomains, Jobs.getRDBComputeDomains, Jobs.getRDBComputeDomainFromName + """ + if dockerComputeDomainName is None: + raise Exception("dockerComputeDomainName is not defined.") + else: + if dockerComputeDomains is None: + dockerComputeDomains = getDockerComputeDomains(); + + if dockerComputeDomains.__len__() > 0: + for dockerComputeDomain in dockerComputeDomains: + if dockerComputeDomainName == dockerComputeDomain.get('name'): + return dockerComputeDomain; + else: + raise Exception("There are no DockerComputeDomains available for the user."); + + raise Exception("DockerComputeDomain of name '" + dockerComputeDomainName + "' is not available or does not exist."); + + +def getRDBComputeDomains(): + """ + Gets a list of all registered Relational Database (RDB) compute domains that the user has access to. + + :return: a list of dictionaries, each one containing the definition of an RDB compute domain. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: rdbComputeDomains = Jobs.getRDBComputeDomains(); + + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.getRDBComputeDomains" + else: + taskName = "SciScript-Python.Jobs.getRDBComputeDomains" + + url = Config.RacmApiURL + "/jobm/rest/computedomains/rdb?TaskName=" + taskName + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + if res.status_code != 200: + raise Exception("Error when getting RDB Compute Domains from JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def getRDBComputeDomainsNames(rdbComputeDomains=None): + """ + Returns the names of the RDB compute domains available to the user. + + :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. + :return: an array of strings, each being the name of a rdb compute domain available to the user. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: dockerComputeDomainsNames = Files.getDockerComputeDomainsNames(); + + .. seealso:: Files.getRDBComputeDomains + """ + if rdbComputeDomains is None: + rdbComputeDomains = getRDBComputeDomains(); + + rdbComputeDomainsNames = []; + for rdbComputeDomain in rdbComputeDomains: + rdbComputeDomainsNames.append(rdbComputeDomain.get('name')) + + return rdbComputeDomainsNames; + + +def getRDBComputeDomainFromName(rdbComputeDomainName, rdbComputeDomains = None): + """ + Returns an RDBComputeDomain object, given its registered name. + + :param rdbComputeDomainName: name of the RDBComputeDomainName, as shown within the results of Jobs.getRDBComputeDomains() + :param rdbComputeDomains: a list of rdbComputeDomain objects (dictionaries), as returned by Jobs.getRDBComputeDomains(). If not set, then an extra internal call to Jobs.getRDBComputeDomains() is made. + :return: an RDBComputeDomain object (dictionary) that defines an RDB compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: rdbComputeDomain = Jobs.getRDBComputeDomainFromName('rdbComputeDomainAtJHU'); + + .. seealso:: Jobs.getDockerComputeDomains, Jobs.getRDBComputeDomains, Jobs.getDockerComputeDomainFromName + """ + if rdbComputeDomainName is None: + raise Exception("rdbComputeDomainName is not defined.") + else: + if rdbComputeDomains is None: + rdbComputeDomains = getRDBComputeDomains(); + + if rdbComputeDomains.__len__() > 0: + for rdbComputeDomain in rdbComputeDomains: + if rdbComputeDomainName == rdbComputeDomain.get('name'): + return rdbComputeDomain; + else: + raise Exception("There are no RDBComputeDomains available for the user."); + + raise Exception("RDBComputeDomain of name '" + rdbComputeDomainName + "' is not available or does not exist."); + + +def getJobsList(top=10, open=None, start=None, end=None, type='all'): + """ + Gets the list of Jobs submitted by the user. + + :param top: top number of jobs (integer) returned. If top=None, then all jobs are returned. + :param open: If set to 'True', then only returns jobs that have not finished executing and wrapped up (status <= FINISHED). If set to 'False' then only returnes jobs that are still running. If set to 'None', then returns both finished and unfinished jobs. + :param start: The earliest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no lower bound on date. + :param end: The latest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no upper bound on date. + :param type: type (string) of jobs returned. Can take values of 'rdb' (for returning only relational database jobs), 'docker' (for returning only Docker jobs) and 'all' (all job types are returned). + :return: a list of dictionaries, each one containing the definition of a submitted job. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. + :example: jobs = Jobs.getJobsList(top=2); + + .. seealso:: Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.getJobsList" + else: + taskName = "SciScript-Python.Jobs.getJobsList" + + + topString = ("top=" + str(top) + "&" if top != None else "") + startString = ("start=" + str(start) + "&" if start != None else "") + endString = ("end=" + str(end) + "&" if end != None else "") + if open is None: + openString = ""; + else: + if open is True: + openString = "open=true&"; + else: + openString = "open=false&"; + + url = Config.RacmApiURL + "/jobm/rest/jobs?" + if(type=='rdb'): + url = Config.RacmApiURL + "/jobm/rest/rdbjobs?" + if(type=='docker'): + url = Config.RacmApiURL + "/jobm/rest/dockerjobs?" + + url = url + topString + startString + endString + "TaskName=" + taskName; + + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting list of jobs from JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + +def getJobDescription(jobId): + """ + Gets the definition of the job, + + :param jobId: Id of job + :return: dictionary containing the description or definition of the job. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. + :example: job1 = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); job2 = Jobs.getJobDescription(job1.get('id')); + + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.getJobDescription" + else: + taskName = "SciScript-Python.Jobs.getJobDescription" + + url = Config.RacmApiURL + "/jobm/rest/jobs/" + str(jobId) + "?TaskName="+taskName + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting from JOBM API the job status of jobId=" + str(jobId) + ".\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def getJobStatus(jobId): + """ + Gets a dictionary with the job status as an integer value, together with its semantic meaning. The integer value is a power of 2, that is, 1:PENDING, 2:QUEUED, 4:ACCEPTED, 8:STARTED, 16:FINISHED, 32:SUCCESS, 64:ERROR and 128:CANCELED + + :param jobId: Id of job (integer). + :return: dictionary with the integer value of the job status, as well as its semantic meaning. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. + :example: jobId = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); status = Jobs.getJobStatus(jobId); + + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + intStatus = getJobDescription(jobId)["status"] + + if intStatus == 1: + return {'status':intStatus, 'statusMeaning':"PENDING", 'jobId':jobId} + elif intStatus == 2: + return {'status': intStatus, 'statusMeaning': "QUEUED", 'jobId':jobId} + elif intStatus == 4: + return {'status': intStatus, 'statusMeaning': "ACCEPTED", 'jobId':jobId} + elif intStatus == 8: + return {'status': intStatus, 'statusMeaning': "STARTED", 'jobId':jobId} + elif intStatus == 16: + return {'status': intStatus, 'statusMeaning': "FINISHED", 'jobId':jobId} + elif intStatus == 32: + return {'status': intStatus, 'statusMeaning': "SUCCESS", 'jobId':jobId} + elif intStatus == 64: + return {'status': intStatus, 'statusMeaning': "ERROR", 'jobId':jobId} + elif intStatus == 128: + return {'status': intStatus, 'statusMeaning': "CANCELED", 'jobId':jobId} + else: + raise Exception("Invalid integer value given to job status.") + + +def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=None, userVolumes=None, dataVolumes=None, resultsFolderPath="", parameters="", jobAlias= ""): + """ + Submits a Jupyter Notebook for execution (as an asynchronous job) inside a Docker compute domain. + + :param notebookPath: path of the notebook within the filesystem mounted in SciServer-Compute (string). Example: notebookPath = '/home/idies/worskpace/persistent/JupyterNotebook.ipynb' + :param dockerComputeDomain: object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). + :param dockerImageName: name (string) of the Docker image for executing the notebook. E.g., dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object. + :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', , 'needsWriteAccess':True}] . A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=None, then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume. + :param dataVolumes: a list with the names of data volumes that will be mounted to the docker Image. E.g.: dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}]. A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted. + :param resultsFolderPath: full path to results folder (string) where the original notebook is copied to and executed. E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder. If not set, then a default folder will be set automatically. + :param parameters: string containing parameters that the notebook might need during its execution. This string is written in the 'parameters.txt' file in the same directory level where the notebook is being executed. + :param jobAlias: alias (string) of job, defined by the user. + :return: the job ID (int) + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. + :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitNotebookJob('/home/idies/workspace/persistent/Notebook.ipynb', dockerComputeDomain, 'Python (astro)', [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}], [{'name':'SDSS_DAS'}], 'param1=1\\nparam2=2\\nparam3=3','myNewJob') + + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.submitNotebookJob" + else: + taskName = "SciScript-Python.Jobs.submitNotebookJob" + + if dockerComputeDomain is None: + dockerComputeDomains = getDockerComputeDomains(); + if dockerComputeDomains .__len__() > 0: + dockerComputeDomain = dockerComputeDomains[0]; + else: + raise Exception("There are no dockerComputeDomains available for the user."); + + if dockerImageName is None: + images = dockerComputeDomain.get('images'); + if images.__len__() > 0: + dockerImageName = images[0].get('name') + else: + raise Exception("dockerComputeDomain has no docker images available for the user."); + + uVols = []; + if userVolumes is None: + for vol in dockerComputeDomain.get('userVolumes'): + if 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + + + else: + for uVol in userVolumes: + found = False; + for vol in dockerComputeDomain.get('userVolumes'): + if vol.get('name') == uVol.get('name') and vol.get('rootVolumeName') == uVol.get('rootVolumeName') and vol.get('owner') == uVol.get('owner'): + found = True; + if (uVol.has_key('needsWriteAccess')): + if uVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + else: + if 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + + if not found: + raise Exception("User volume '" + uVol.get('name') + "' not found within Compute domain") + + datVols = []; + if dataVolumes is None: + for vol in dockerComputeDomain.get('volumes'): + datVols.append({'name': vol.get('name')}); + + else: + for dVol in dataVolumes: + found = False; + for vol in dockerComputeDomain.get('volumes'): + if vol.get('name') == dVol.get('name'): + found = True; + datVols.append({'name': vol.get('name')}); + + if not found: + raise Exception("Data volume '" + dVol.get('name') + "' not found within Compute domain") + + + dockerComputeEndpoint = dockerComputeDomain.get('apiEndpoint'); + + dockerJobModel = { + "command": parameters, + "scriptURI": notebookPath, + "submitterDID": jobAlias, + "dockerComputeEndpoint": dockerComputeEndpoint, + "dockerImageName": dockerImageName, + "resultsFolderURI": resultsFolderPath, + "volumeContainers": datVols, + "userVolumes": uVols + } + data = json.dumps(dockerJobModel).encode() + url = Config.RacmApiURL + "/jobm/rest/jobs/docker?TaskName="+taskName; + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.post(url, data=data, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when submitting a notebook job to the JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return (json.loads(res.content.decode())).get('id') + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageName = None, userVolumes = None, dataVolumes = None, resultsFolderPath = "", jobAlias = ""): + """ + Submits a shell command for execution (as an asynchronous job) inside a Docker compute domain. + + :param shellCommand: shell command (string) defined by the user. + :param dockerComputeDomain: object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). + :param dockerImageName: name (string) of the Docker image for executing the notebook. E.g., dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object. + :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image. + E.g., userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', , 'needsWriteAccess':True}] + A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=None, then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume. + :param dataVolumes: a list with the names of data volumes that will be mounted to the docker Image. + E.g., dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}]. + A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted. + :param resultsFolderPath: full path to results folder (string) where the shell command is executed. E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder. If not set, then a default folder will be set automatically. + :param jobAlias: alias (string) of job, defined by the user. + :return: the job ID (int) + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. + :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitShellCommandJob('pwd', dockerComputeDomain, 'Python (astro)', [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}], [{'name':'SDSS_DAS'}], 'myNewJob') + + .. seealso:: Jobs.submitNotebookJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.submitShellCommandJob" + else: + taskName = "SciScript-Python.Jobs.submitShellCommandJob" + + + if dockerComputeDomain is None: + dockerComputeDomains = getDockerComputeDomains(); + if dockerComputeDomains .__len__() > 0: + dockerComputeDomain = dockerComputeDomains[0]; + else: + raise Exception("There are no dockerComputeDomains available for the user."); + + if dockerImageName is None: + images = dockerComputeDomain.get('images'); + if images.__len__() > 0: + dockerImageName = images[0].get('name') + else: + raise Exception("dockerComputeDomain has no docker images available for the user."); + + uVols = []; + if userVolumes is None: + for vol in dockerComputeDomain.get('userVolumes'): + if 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + + + else: + for uVol in userVolumes: + found = False; + for vol in dockerComputeDomain.get('userVolumes'): + if vol.get('name') == uVol.get('name'): + found = True; + if (uVol.has_key('needsWriteAccess')): + if uVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + else: + if 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + + if not found: + raise Exception("User volume '" + uVol.get('name') + "' not found within Compute domain") + + datVols = []; + if dataVolumes is None: + for vol in dockerComputeDomain.get('volumes'): + datVols.append({'name': vol.get('name')}); + + else: + for dVol in dataVolumes: + found = False; + for vol in dockerComputeDomain.get('volumes'): + if vol.get('name') == dVol.get('name'): + found = True; + datVols.append({'name': vol.get('name')}); + + if not found: + raise Exception("Data volume '" + dVol.get('name') + "' not found within Compute domain") + + + dockerComputeEndpoint = dockerComputeDomain.get('apiEndpoint'); + + dockerJobModel = { + "command": shellCommand, + "submitterDID": jobAlias, + "dockerComputeEndpoint": dockerComputeEndpoint, + "dockerImageName": dockerImageName, + "volumeContainers": datVols, + "userVolumes": uVols, + "resultsFolderURI": resultsFolderPath + } + data = json.dumps(dockerJobModel).encode() + url = Config.RacmApiURL + "/jobm/rest/jobs/docker?TaskName="+taskName; + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.post(url, data=data, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when submitting a job to the JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return (json.loads(res.content.decode())).get('id') + else: + raise Exception("User token is not defined. First log into SciServer.") + +def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = None, resultsName='queryResults', resultsFolderPath="", jobAlias = ""): + """ + Submits a sql query for execution (as an asynchronous job) inside a relational database (RDB) compute domain. + + :param sqlQuery: sql query (string) + :param rdbComputeDomain: object (dictionary) that defines a relational database (RDB) compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains(). + :param databaseContextName: database context name (string) on which the sql query is executed. + :param resultsName: name (string) of the table or file (without file type ending) that contains the query result. In case the sql query has multiple statements, should be set to a list of names (e.g., ['result1','result2']). + :param resultsFolderPath: full path to results folder (string) where query output tables are written into. E.g.: /home/idies/workspace/rootVOlume/username/userVolume/jobsFolder . If not set, then a default folder will be set automatically. + :param jobAlias: alias (string) of job, defined by the user. + :return: a dictionary containing the definition of the submitted job. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. + :example: job = Jobs.submitRDBQueryJob('select 1';,None, None, 'myQueryResults', 'myNewJob') + + .. seealso:: Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.submitRDBQueryJob" + else: + taskName = "SciScript-Python.Jobs.submitRDBQueryJob" + + if rdbComputeDomain is None: + rdbComputeDomains = getRDBComputeDomains(); + if rdbComputeDomains .__len__() > 0: + rdbComputeDomain = rdbComputeDomains[0]; + else: + raise Exception("There are no rdbComputeDomains available for the user."); + + if databaseContextName is None: + databaseContexts = rdbComputeDomain.get('databaseContexts'); + if databaseContexts.__len__() > 0: + databaseContextName = databaseContexts[0].get('name') + else: + raise Exception("rbdComputeDomain has no database contexts available for the user."); + + targets = []; + if type(resultsName) == str: + targets.append({'location': resultsName, 'type': 'FILE_CSV', 'resultNumber': 1}); + elif type(resultsName) == list: + if len(set(resultsName)) != len(resultsName): + raise Exception("Elements of parameter 'resultsName' must be unique"); + + for i in range(len(resultsName)): + if type(resultsName[i]) == str: + targets.append({'location': resultsName[i], 'type': 'FILE_CSV', 'resultNumber': i+1}); + else: + raise Exception("Elements of array 'resultsName' are not strings"); + + else: + raise Exception("Type of parameter 'resultsName' is not supported"); + + + rdbDomainId = rdbComputeDomain.get('id'); + + dockerJobModel = { + "inputSql": sqlQuery, + "submitterDID": jobAlias, + "databaseContextName": databaseContextName, + "rdbDomainId": rdbDomainId, + "targets": targets, + "resultsFolderURI":resultsFolderPath + } + + data = json.dumps(dockerJobModel).encode() + url = Config.RacmApiURL + "/jobm/rest/jobs/rdb?TaskName="+taskName; + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.post(url, data=data, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when submitting a job to the JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return (json.loads(res.content.decode())).get('id') + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def cancelJob(jobId): + """ + Cancels the execution of a job. + + :param jobId: Id of the job (integer) + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error. + :example: job = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); isCanceled = Jobs.cancelJob(job.get('id')); + + .. seealso:: Jobs.submitNotebookJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains. + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.cancelJob" + else: + taskName = "SciScript-Python.Jobs.cancelJob" + + + url = Config.RacmApiURL + "/jobm/rest/jobs/" + str(jobId) + "/cancel?TaskName="+taskName + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.post(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting from JOBM API the job status of jobId=" + str(jobId) + ".\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + pass; + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def waitForJob(jobId, verbose=False, pollTime = 5): + """ + Queries regularly the job status and waits until the job is completed. + + :param jobId: id of job (integer) + :param verbose: if True, will print "wait" messages on the screen while the job is still running. If False, will suppress the printing of messages on the screen. + :param pollTime: idle time interval (integer, in seconds) before querying again for the job status. Minimum value allowed is 5 seconds. + :return: After the job is finished, returns a dictionary object containing the job definition. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitShellCommandJob(dockerComputeDomain,'pwd', 'Python (astro)');Jobs.waitForJob(job.get('id')) + + .. seealso:: Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.submitNotebookJob, Jobs.submitShellCommandJob + """ + try: + minPollTime = 5 # in seconds + complete = False + waitingStr = "Waiting..." + back = "\b" * len(waitingStr) + if verbose: + print waitingStr, + + while not complete: + if verbose: + #print back, + print waitingStr, + jobDesc = getJobStatus(jobId) + jobStatus = jobDesc["status"] + if jobStatus >= 32: + complete = True + if verbose: + #print back, + print "Done!" + else: + time.sleep(max(minPollTime,pollTime)); + + return jobDesc + except Exception as e: + raise e; + diff --git a/py2/SciServer/SciDrive.py b/py2/SciServer/SciDrive.py index b3df2db..523e02d 100644 --- a/py2/SciServer/SciDrive.py +++ b/py2/SciServer/SciDrive.py @@ -7,6 +7,15 @@ from SciServer import Config, Authentication +class Task: + """ + The class TaskName stores the name of the task that executes the API call. + """ + name = None + + +task = Task(); + def createContainer(path): """ Creates a container (directory) in SciDrive @@ -25,7 +34,13 @@ def createContainer(path): 'uri="vos://' + Config.SciDriveHost + '!vospace/' + path + '">' '' '') - url = Config.SciDriveHost + '/vospace-2.0/nodes/' + path + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.createContainer" + else: + taskName = "SciScript-Python.SciDrive.createContainer" + + url = Config.SciDriveHost + '/vospace-2.0/nodes/' + path + "?TaskName=" + taskName; data = str.encode(containerBody) headers = {'X-Auth-Token': token, 'Content-Type': 'application/xml'} res = requests.put(url, data=data, headers=headers) @@ -52,7 +67,14 @@ def upload(path, data="", localFilePath=""): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SciDriveHost + '/vospace-2.0/1/files_put/dropbox/' + path + + taskName = "" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.upload" + else: + taskName = "SciScript-Python.SciDrive.upload" + + url = Config.SciDriveHost + '/vospace-2.0/1/files_put/dropbox/' + path + "?TaskName=" + taskName; headers = {'X-Auth-Token': token} if(localFilePath != ""): with open(localFilePath, "rb") as file: @@ -85,7 +107,17 @@ def publicUrl(path): token = Authentication.getToken() if token is not None and token != "": - url = Config.SciDriveHost + '/vospace-2.0/1/media/sandbox/' + str(path) + taskName = ""; + if task.name is not None: + taskName = task.name; + task.name = None; + else: + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.publicUrl" + else: + taskName = "SciScript-Python.SciDrive.publicUrl" + + url = Config.SciDriveHost + '/vospace-2.0/1/media/sandbox/' + str(path) + "?TaskName=" + taskName headers = {'X-Auth-Token': token} res = requests.get(url, headers=headers) if res.status_code != 200: @@ -113,7 +145,13 @@ def directoryList(path=""): token = Authentication.getToken() if token is not None and token != "": - url = Config.SciDriveHost + "/vospace-2.0/1/metadata/sandbox/" + str(path) + "?list=True&path=" + str(path) + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.directoryList" + else: + taskName = "SciScript-Python.SciDrive.directoryList" + + url = Config.SciDriveHost + "/vospace-2.0/1/metadata/sandbox/" + str(path) + "?list=True&path=" + str(path) + "&TaskName=" + taskName; headers = {'X-Auth-Token': token} res = requests.get(url, headers=headers) if res.status_code != 200: @@ -143,6 +181,12 @@ def download(path, format="text", localFilePath=""): token = Authentication.getToken() if token is not None and token != "": + taskName = ""; + if Config.isSciServerComputeEnvironment(): + task.name = "Compute.SciScript-Python.SciDrive.download" + else: + task.name = "SciScript-Python.SciDrive.download" + fileUrl = publicUrl(path) res = requests.get(fileUrl, stream=True) if res.status_code != 200: @@ -194,7 +238,13 @@ def delete(path): 'uri="vos://' + Config.SciDriveHost + '!vospace/' + path + '">' '' '') - url = Config.SciDriveHost + '/vospace-2.0/nodes/' + path + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.delete" + else: + taskName = "SciScript-Python.SciDrive.delete" + + url = Config.SciDriveHost + '/vospace-2.0/nodes/' + path + "?TaskName=" + taskName; data = str.encode(containerBody) headers = {'X-Auth-Token': token, 'Content-Type': 'application/xml'} res = requests.delete(url, data=data, headers=headers) diff --git a/py2/SciServer/SkyQuery.py b/py2/SciServer/SkyQuery.py index e2991ae..65c2ef9 100644 --- a/py2/SciServer/SkyQuery.py +++ b/py2/SciServer/SkyQuery.py @@ -27,7 +27,14 @@ def getJobStatus(jobId): """ token = Authentication.getToken() if token is not None and token != "": - statusURL = Config.SkyQueryUrl + '/Jobs.svc/jobs/' + str(jobId) + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.getJobStatus" + else: + taskName = "SciScript-Python.SkyQuery.getJobStatus" + + statusURL = Config.SkyQueryUrl + '/Jobs.svc/jobs/' + str(jobId) + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -57,7 +64,13 @@ def cancelJob(jobId): token = Authentication.getToken() if token is not None and token != "": - statusURL = Config.SkyQueryUrl + '/Jobs.svc/jobs/' + jobId + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.cancelJob" + else: + taskName = "SciScript-Python.SkyQuery.cancelJob" + + statusURL = Config.SkyQueryUrl + '/Jobs.svc/jobs/' + jobId + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -93,7 +106,14 @@ def listQueues(): """ token = Authentication.getToken() if token is not None and token != "": - jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues' + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.listQueues" + else: + taskName = "SciScript-Python.SkyQuery.listQueues" + + jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -113,7 +133,7 @@ def getQueueInfo(queue): """ Returns information about a particular job queue (more info in http://www.voservices.net/skyquery). - :param queue: queue name (string) + :param queue: queue name (string) :return: a dictionary containing information associated to the queue. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error. :example: queueInfo = SkyQuery.getQueueInfo('quick') @@ -123,7 +143,13 @@ def getQueueInfo(queue): token = Authentication.getToken() if token is not None and token != "": - jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.getQueueInfo" + else: + taskName = "SciScript-Python.SkyQuery.getQueueInfo" + + jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -154,7 +180,13 @@ def submitJob(query, queue='quick'): token = Authentication.getToken() if token is not None and token != "": - jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + '/jobs' + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.submitJob" + else: + taskName = "SciScript-Python.SkyQuery.submitJob" + + jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + '/jobs' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -174,12 +206,13 @@ def submitJob(query, queue='quick'): raise Exception("User token is not defined. First log into SciServer.") -def waitForJob(jobId, verbose=True): +def waitForJob(jobId, verbose=False, pollTime = 5): """ - Queries the job status from SkyQuery every 2 seconds and waits for the SkyQuery job to be completed. + Queries regularly the job status and waits until the job is completed. :param jobId: id of job (integer) - :param verbose: if True, will print "wait" messages on the screen while the job is not done. If False, will suppress printing messages on the screen. + :param verbose: if True, will print "wait" messages on the screen while the job is still running. If False, will suppress the printing of messages on the screen. + :param pollTime: idle time interval (integer, in seconds) before querying again for the job status. Minimum value allowed is 5 seconds. :return: After the job is finished, returns a dictionary object containing the job status and related metadata. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error. :example: SkyQuery.waitForJob(SkyQuery.submitJob("select 1")) @@ -187,8 +220,8 @@ def waitForJob(jobId, verbose=True): .. seealso:: SkyQuery.submitJob, SkyQuery.getJobStatus. """ try: + minPollTime = 5 # in seconds complete = False - waitingStr = "Waiting..." back = "\b" * len(waitingStr) if verbose: @@ -199,13 +232,13 @@ def waitForJob(jobId, verbose=True): #print back, print waitingStr, jobDesc = getJobStatus(jobId) - if jobDesc['status'] == 'completed': + if jobDesc.get('dateFinished') is not None: complete = True if verbose: #print back, print "Done!" else: - time.sleep(2) + time.sleep(max(minPollTime,pollTime)); return jobDesc except Exception as e: @@ -226,7 +259,14 @@ def listJobs(queue="quick"): """ token = Authentication.getToken() if token is not None and token != "": - jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + '/jobs?' + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.listJobs" + else: + taskName = "SciScript-Python.SkyQuery.listJobs" + + jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + '/jobs' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -261,7 +301,13 @@ def listAllDatasets(): token = Authentication.getToken() if token is not None and token != "": - schemaURL = Config.SkyQueryUrl + '/Schema.svc/datasets' + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.listAllDatasets" + else: + taskName = "SciScript-Python.SkyQuery.listAllDatasets" + + schemaURL = Config.SkyQueryUrl + '/Schema.svc/datasets' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -290,7 +336,14 @@ def getDatasetInfo(datasetName="MyDB"): """ token = Authentication.getToken() if token is not None and token != "": - schemaURL = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.getDatasetInfo" + else: + taskName = "SciScript-Python.SkyQuery.getDatasetInfo" + + schemaURL = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -318,7 +371,14 @@ def listDatasetTables(datasetName="MyDB"): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables' + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.listDatasetTables" + else: + taskName = "SciScript-Python.SkyQuery.listDatasetTables" + + url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -348,7 +408,14 @@ def getTableInfo(tableName, datasetName="MyDB"): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables/' + tableName + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.getTableInfo" + else: + taskName = "SciScript-Python.SkyQuery.getTableInfo" + + url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables/' + tableName + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -377,7 +444,14 @@ def listTableColumns(tableName, datasetName="MyDB"): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables/' + tableName + '/columns' + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.listTableColumns" + else: + taskName = "SciScript-Python.SkyQuery.listTableColumns" + + url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables/' + tableName + '/columns' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -412,7 +486,13 @@ def getTable(tableName, datasetName="MyDB", top = None): token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.getTable" + else: + taskName = "SciScript-Python.SkyQuery.getTable" + + url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + "?TaskName=" + taskName if top != None and top != "": url = url + '?top=' + str(top) @@ -443,7 +523,14 @@ def dropTable(tableName, datasetName="MyDB"): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.dropTable" + else: + taskName = "SciScript-Python.SkyQuery.dropTable" + + url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -468,13 +555,20 @@ def uploadTable(uploadData, tableName, datasetName="MyDB", format="csv"): :param format: format of the 'data' parameter. Set to 'csv' for now. :return: returns True if the table was uploaded successfully. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error. - :example: result = SkyQuery.uploadTable("Column1,Column2\n4.5,5.5\n", tableName="myTable", datasetName="MyDB", format="csv") + :example: result = SkyQuery.uploadTable("Column1,Column2\\n4.5,5.5\\n", tableName="myTable", datasetName="MyDB", format="csv") .. seealso:: SkyQuery.listQueues, SkyQuery.listAllDatasets, SkyQuery.getDatasetInfo, SkyQuery.listDatasetTables, SkyQuery.getTableInfo, SkyQuery.getTable, SkyQuery.submitJob """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.uploadTable" + else: + taskName = "SciScript-Python.SkyQuery.uploadTable" + + url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + "?TaskName=" + taskName ctype = "" if format == "csv": ctype = 'text/csv' diff --git a/py2/SciServer/SkyServer.py b/py2/SciServer/SkyServer.py index a664723..ca20934 100644 --- a/py2/SciServer/SkyServer.py +++ b/py2/SciServer/SkyServer.py @@ -60,12 +60,11 @@ def getJpegImgCutout(ra, dec, scale=0.7, width=512, height=512, opt="", query="" """ Gets a rectangular image cutout from a region of the sky in SDSS, centered at (ra,dec). Return type is numpy.ndarray.\n - :param ra: Right Ascension of the image's center. - :param dec: Declination of the image's center. - :param scale: scale of the image, measured in [arcsec/pix] - :param width: Right Ascension of the image's center. - :param ra: Right Ascension of the image's center. - :param height: Height of the image, measured in [pix]. + :param ra: Right Ascension of the image's center in floating point ICRS J2000 coordinates. + :param dec: Declination of the image's center in floating point ICRS J2000 coordinates. + :param scale: Scale of the image, measured in [arcsec/pix] + :param width: Width of image, measured in pixels (width in arcsec = width * scale). If larger than 2048, silently truncated to 2048. + :param height: Height of image, measured in pixels (width in arcsec = width * scale). If larger than 2048, silently truncated to 2048. :param opt: Optional drawing options, expressed as concatenation of letters (string). The letters options are \n \t"G": Grid. Draw a N-S E-W grid through the center\n \t"L": Label. Draw the name, scale, ra, and dec on image.\n diff --git a/py2/SciServer/__init__.py b/py2/SciServer/__init__.py index a7f4476..79179af 100644 --- a/py2/SciServer/__init__.py +++ b/py2/SciServer/__init__.py @@ -5,21 +5,31 @@ Some SciServer tools you can access with this package:\n -* `Login Portal `_: Single sign-on portal to all SciServer applications. +* `Login Portal `_: Single sign-on portal to all SciServer applications. \t\tAlthough some tools accept anonymous access, you can use Authentication.login to login and access the tools and your own data and environment (after registering in the Login Portal). If you are running this package in a Jupyter Notebook in the SciServer-Compute environment, the use of Authentication.login is not necessary since it's done automatically. -* `CasJobs `_: Database storage and querying. +* `CasJobs `_: Database storage and querying. \t\tYou can have access big databases, as well as save your data tables in your own database called 'MyDB'. The user can run synchronous or asynchronous SQL queries and get the result back as an R data-frame (using CasJobs.executeQuery or CasJobs.submitJob, respectively). Uploading of CSV files or R data-frames into a database table can be done using CasJobs.uploadCSVToTable and CasJobs.uploadDataFrameToTable, respectively. -* `SciDrive `_: Drag-and-drop file storage and sharing. +* `SciDrive `_: Drag-and-drop file storage and sharing. \t\tYou can create directories in SciDrive using SciDrive.createContainer, upload a file to SciDrive using SciDrive.upload, and share its URL with your collaborators by using SciDrive.publicUrl. -* `SkyServer `_: Access to the SDSS astronomical survey. +* `SkyServer `_: Access to the SDSS astronomical survey. \t\tYou can query the SDSS database using SkyServer.sqlSearch, run cone searches using SkyServer.radialSearch, or get cutout images from the sky using SkyServer.getJpegImgCutout, between other tasks. -* `SkyQuery `_: Cross-match of astronomical source catalogs. +* `SkyQuery `_: Cross-match of astronomical source catalogs. \t\tYou can use this scalable database system for uploading your own catalogs and cross-matching them against huge astronomical source catalogs, or even cross-matching huge catalogs against each other!. Use SkyQuery.submitJob to run the cross-match, and use SkyQuery.listAllDatasets, SkyQuery.listDatasetTables and SkyQuery.listTableColumns to browse the catalogs and database schema. +* `Compute Jobs `_: Submission of Jupyter notebooks or shell commands as jobs +\t\tYou can execute whole Jupyter notebooks and shell commands as asynchronous batch jobs, as well as synchronous jobs. + +* `SciQuery Jobs `_: Submission of SQL queries. +\t\tYou can execute synchronous or s synchronous SQL queries against Postgres and other database backends. + +* `Files `_: Interaction with the SciServer file system. +\t\tYou can upload, and download data into your own file space in SciServer, as well as share data with your collaborators, between other things. + + **References** * SciServer-Python repository: http://www.github.com/sciserver/SciScript-Python @@ -31,7 +41,7 @@ **Authors**: Gerard Lemson , Manuchehr Taghizadeh-Popp -**Version**: sciserver-v1.10.0 +**Version**: sciserver-v2.2.0 """ diff --git a/py2/setup.py b/py2/setup.py index 33d3959..26e8abf 100644 --- a/py2/setup.py +++ b/py2/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages setup( name = "SciServer", - version = "1.10.0", + version = "2.2.0", packages = find_packages(), ) \ No newline at end of file diff --git a/py3/SciServer/Authentication.py b/py3/SciServer/Authentication.py index 4610b49..e4af767 100644 --- a/py3/SciServer/Authentication.py +++ b/py3/SciServer/Authentication.py @@ -6,7 +6,7 @@ import requests import os.path import warnings - +import getpass from SciServer import Config @@ -17,6 +17,7 @@ class KeystoneUser: """ id = None userName = None + token = None class Token: """ @@ -26,6 +27,7 @@ class Token: token = Token(); +keystoneUser = KeystoneUser(); def getKeystoneUserWithToken(token): """ @@ -38,25 +40,41 @@ def getKeystoneUserWithToken(token): .. seealso:: Authentication.getToken, Authentication.login, Authentication.setToken. """ - loginURL = Config.AuthenticationURL - if ~loginURL.endswith("/"): - loginURL = loginURL + "/" - loginURL = loginURL + token - getResponse = requests.get(loginURL) - if getResponse.status_code != 200: - raise Exception("Error when getting the keystone user with token " + str(token) +".\nHttp Response from the Authentication API returned status code " + str(getResponse.status_code) + ":\n" + getResponse.content.decode()); + if keystoneUser.token == token and keystoneUser.token is not None: + return keystoneUser; + + else: + taskName = "" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Authentication.getKeystoneUserWithToken" + else: + taskName = "SciScript-Python.Authentication.getKeystoneUserWithToken" + + loginURL = Config.AuthenticationURL + if ~loginURL.endswith("/"): + loginURL = loginURL + "/" + + loginURL = loginURL + token + "?TaskName=" + taskName; - responseJson = json.loads((getResponse.content.decode())) + getResponse = requests.get(loginURL) + if getResponse.status_code != 200: + raise Exception("Error when getting the keystone user with token " + str(token) +".\nHttp Response from the Authentication API returned status code " + str(getResponse.status_code) + ":\n" + getResponse.content.decode()); - ksu = KeystoneUser() - ksu.userName = responseJson["token"]["user"]["name"] - ksu.id = responseJson["token"]["user"]["id"] + responseJson = json.loads((getResponse.content.decode())) - return ksu + ksu = KeystoneUser() + ksu.userName = responseJson["token"]["user"]["name"] + ksu.id = responseJson["token"]["user"]["id"] + ksu.token = token + keystoneUser.token = token; + keystoneUser.userName = ksu.userName + keystoneUser.id = ksu.id + return ksu -def login(UserName, Password): + +def login(UserName=None, Password=None): """ Logs the user into SciServer and returns the authentication token. This function is useful when SciScript-Python library methods are executed outside the SciServer-Compute environment. @@ -64,17 +82,30 @@ def login(UserName, Password): so the user has to use Authentication.login in order to log into SciServer manually and get the authentication token. Authentication.login also sets the token value in the python instance argument variable "--ident", and as the local object Authentication.token (of class Authentication.Token). - :param UserName: name of the user (string) - :param Password: password of the user (string) + :param UserName: name of the user (string). If not set, then a user name prompt will show. + :param Password: password of the user (string). If not set, then a password prompt will show. :return: authentication token (string) :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. - :example: token = Authentication.login('loginName','loginPassword') + :example: token1 = Authentication.login(); token2 = Authentication.login('loginName','loginPassword'); .. seealso:: Authentication.getKeystoneUserWithToken, Authentication.getToken, Authentication.setToken, Authentication.token. """ - loginURL = Config.AuthenticationURL + taskName = "" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Authentication.Login" + else: + taskName = "SciScript-Python.Authentication.Login" + + loginURL = Config.AuthenticationURL + "?TaskName=" + taskName + + userName = UserName; + password = Password; + if userName is None: + userName = getpass.getpass(prompt="Enter SciServer user name: ") + if password is None: + password = getpass.getpass(prompt="Enter SciServer password: ") - authJson = {"auth":{"identity":{"password":{"user":{"name":UserName,"password":Password}}}}} + authJson = {"auth":{"identity":{"password":{"user":{"name":userName,"password":password}}}}} data = json.dumps(authJson).encode() @@ -101,29 +132,49 @@ def getToken(): """ try: - if token.value is not None: - return token.value + + if Config.isSciServerComputeEnvironment(): + tokenFile = Config.KeystoneTokenPath; # '/home/idies/keystone.token' + if os.path.isfile(tokenFile): + with open(tokenFile, 'r') as f: + _token = f.read().rstrip('\n') + if _token is not None and _token != "": + token.value = _token; + + found = False + ident = identArgIdentifier() + for arg in sys.argv: + if (arg.startswith(ident)): + sys.argv.remove(arg) + sys.argv.append(ident + _token) + found = True + if not found: + sys.argv.append(ident + _token) + + return _token + else: + warnings.warn("In Authentication.getToken: Cannot find token in system token file " + str(Config.KeystoneTokenPath) + ".", Warning, stacklevel=2) + return None; + else: + warnings.warn("In Authentication.getToken: Cannot find system token file " + str(Config.KeystoneTokenPath) + ".", Warning, stacklevel=2) + return None; else: - _token = None - ident = identArgIdentifier() - for arg in sys.argv: - if (arg.startswith(ident)): - _token = arg[len(ident):] - - if _token is not None and _token != "": - setToken(_token) - return _token + if token.value is not None: + return token.value else: - tokenFile = Config.KeystoneTokenPath; # '/home/idies/keystone.token' - if os.path.isfile(tokenFile) : - with open(tokenFile, 'r') as f: - _token = f.read().rstrip('\n') - if _token is not None and _token != "": - setToken(_token) - return _token - else: - warnings.warn("In Authentication.getToken: Keystone token is not defined. Returning a None token value. Keystone token is not stored in the command line argument --ident, nor in the system token file " + str(Config.KeystoneTokenPath) + ".", Warning, stacklevel=2) - return None; + _token = None + ident = identArgIdentifier() + for arg in sys.argv: + if (arg.startswith(ident)): + _token = arg[len(ident):] + + if _token is not None and _token != "": + token.value = _token + return _token + else: + warnings.warn("In Authentication.getToken: Authentication token is not defined: the user did not log in with the Authentication.login function, or the token has not been stored in the command line argument --ident.", Warning, stacklevel=2) + return None; + except Exception as e: raise e; @@ -142,18 +193,20 @@ def setToken(_token): if _token == "": warnings.warn("Authentication token is being set as an empty string.", Warning, stacklevel=2) - token.value = _token; - - found = False - ident = identArgIdentifier() - for arg in sys.argv: - if (arg.startswith(ident)): - sys.argv.remove(arg) + if Config.isSciServerComputeEnvironment(): + warnings.warn("Authentication token cannot be set to arbitary value when inside SciServer-Compute environment.", Warning, stacklevel=2) + else: + token.value = _token; + + found = False + ident = identArgIdentifier() + for arg in sys.argv: + if (arg.startswith(ident)): + sys.argv.remove(arg) + sys.argv.append(ident + _token) + found = True + if not found: sys.argv.append(ident + _token) - found = True - if not found: - sys.argv.append(ident + _token) - def identArgIdentifier(): diff --git a/py3/SciServer/CasJobs.py b/py3/SciServer/CasJobs.py index 6d28ff3..896455c 100644 --- a/py3/SciServer/CasJobs.py +++ b/py3/SciServer/CasJobs.py @@ -10,6 +10,16 @@ from SciServer import Authentication, Config +class Task: + """ + The class TaskName stores the name of the task that executes the API call. + """ + name = None + + +task = Task(); + + def getSchemaName(): """ Returns the WebServiceID that identifies the schema for a user in MyScratch database with CasJobs. @@ -24,7 +34,14 @@ def getSchemaName(): if token is not None and token != "": keystoneUserId = Authentication.getKeystoneUserWithToken(token).id - usersUrl = Config.CasJobsRESTUri + "/users/" + keystoneUserId + + taskName = "" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.getSchemaName" + else: + taskName = "SciScript-Python.CasJobs.getSchemaName" + + usersUrl = Config.CasJobsRESTUri + "/users/" + keystoneUserId + "?TaskName=" + taskName headers={'X-Auth-Token': token,'Content-Type': 'application/json'} getResponse = requests.get(usersUrl,headers=headers) if getResponse.status_code != 200: @@ -51,7 +68,13 @@ def getTables(context="MyDB"): token = Authentication.getToken() if token is not None and token != "": - TablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables" + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.getTables" + else: + taskName = "SciScript-Python.CasJobs.getTables" + + TablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables" + "?TaskName=" + taskName headers={'X-Auth-Token': token,'Content-Type': 'application/json'} @@ -100,15 +123,19 @@ def executeQuery(sql, context="MyDB", format="pandas"): else: raise Exception("Error when executing query. Illegal format parameter specification: " + str(format)); - QueryUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/query" - - TaskName = ""; - if Config.isSciServerComputeEnvironment(): - TaskName = "Compute.SciScript-Python.CasJobs.executeQuery" + taskName = ""; + if task.name is not None: + taskName = task.name; + task.name = None; else: - TaskName = "SciScript-Python.CasJobs.executeQuery" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.executeQuery" + else: + taskName = "SciScript-Python.CasJobs.executeQuery" - query = {"Query": sql, "TaskName": TaskName} + QueryUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/query" + "?TaskName=" + taskName + + query = {"Query": sql, "TaskName": taskName} data = json.dumps(query).encode() @@ -125,7 +152,15 @@ def executeQuery(sql, context="MyDB", format="pandas"): return StringIO(postResponse.content.decode()) elif format == "pandas": r=json.loads(postResponse.content.decode()) - return pandas.DataFrame(r['Result'][0]['Data'],columns=r['Result'][0]['Columns']) + if len(r['Result']) > 1: + res = [] + for result in r['Result']: + res.append(pandas.DataFrame(result['Data'],columns=result['Columns'])) + + return res + else: + return pandas.DataFrame(r['Result'][0]['Data'],columns=r['Result'][0]['Columns']) + elif format == "csv": return postResponse.content.decode() elif format == "dict": @@ -154,15 +189,15 @@ def submitJob(sql, context="MyDB"): token = Authentication.getToken() if token is not None and token != "": - QueryUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/jobs" - - TaskName = ""; + taskName = ""; if Config.isSciServerComputeEnvironment(): - TaskName = "Compute.SciScript-Python.CasJobs.submitJob" + taskName = "Compute.SciScript-Python.CasJobs.submitJob" else: - TaskName = "SciScript-Python.CasJobs.submitJob" + taskName = "SciScript-Python.CasJobs.submitJob" + + QueryUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/jobs" + "?TaskName=" + taskName - query = {"Query": sql, "TaskName": TaskName} + query = {"Query": sql, "TaskName": taskName} data = json.dumps(query).encode() @@ -193,7 +228,13 @@ def getJobStatus(jobId): token = Authentication.getToken() if token is not None and token != "": - QueryUrl = Config.CasJobsRESTUri + "/jobs/" + str(jobId) + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.getJobStatus" + else: + taskName = "SciScript-Python.CasJobs.getJobStatus" + + QueryUrl = Config.CasJobsRESTUri + "/jobs/" + str(jobId) + "?TaskName=" + taskName headers={'X-Auth-Token': token,'Content-Type': 'application/json'} @@ -220,7 +261,13 @@ def cancelJob(jobId): token = Authentication.getToken() if token is not None and token != "": - QueryUrl = Config.CasJobsRESTUri + "/jobs/" + str(jobId) + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.cancelJob" + else: + taskName = "SciScript-Python.CasJobs.cancelJob" + + QueryUrl = Config.CasJobsRESTUri + "/jobs/" + str(jobId) + "?TaskName=" + taskName headers={'X-Auth-Token': token,'Content-Type': 'application/json'} @@ -233,12 +280,13 @@ def cancelJob(jobId): raise Exception("User token is not defined. First log into SciServer.") -def waitForJob(jobId, verbose=True): +def waitForJob(jobId, verbose=False, pollTime = 5): """ - Queries the job status from casjobs every 2 seconds and waits for the casjobs job to return a status of 3, 4, or 5 (Cancelled, Failed or Finished, respectively). + Queries regularly the job status and waits until the job is completed. :param jobId: id of job (integer) - :param verbose: if True, will print "wait" messages on the screen while the job is not done. If False, will suppress printing messages on the screen. + :param verbose: if True, will print "wait" messages on the screen while the job is still running. If False, will suppress the printing of messages on the screen. + :param pollTime: idle time interval (integer, in seconds) before querying again for the job status. Minimum value allowed is 5 seconds. :return: After the job is finished, returns a dictionary object containing the job status and related metadata. The "Status" field can be equal to 0 (Ready), 1 (Started), 2 (Canceling), 3(Canceled), 4 (Failed) or 5 (Finished). :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the CasJobs API returns an error. :example: CasJobs.waitForJob(CasJobs.submitJob("select 1")) @@ -247,6 +295,7 @@ def waitForJob(jobId, verbose=True): """ try: + minPollTime = 5 # in seconds complete = False waitingStr = "Waiting..." @@ -266,7 +315,7 @@ def waitForJob(jobId, verbose=True): #print(back, end="") print("Done!") else: - time.sleep(2) + time.sleep(max(minPollTime,pollTime)); return jobDesc except Exception as e: @@ -287,6 +336,13 @@ def writeFitsFileFromQuery(fileName, queryString, context="MyDB"): .. seealso:: CasJobs.submitJob, CasJobs.getJobStatus, CasJobs.executeQuery, CasJobs.getPandasDataFrameFromQuery, CasJobs.getNumpyArrayFromQuery """ try: + + + if Config.isSciServerComputeEnvironment(): + task.name = "Compute.SciScript-Python.CasJobs.writeFitsFileFromQuery" + else: + task.name = "SciScript-Python.CasJobs.writeFitsFileFromQuery" + bytesio = executeQuery(queryString, context=context, format="fits") theFile = open(fileName, "w+b") @@ -312,6 +368,12 @@ def getPandasDataFrameFromQuery(queryString, context="MyDB"): .. seealso:: CasJobs.submitJob, CasJobs.getJobStatus, CasJobs.executeQuery, CasJobs.writeFitsFileFromQuery, CasJobs.getNumpyArrayFromQuery """ try: + + if Config.isSciServerComputeEnvironment(): + task.name = "Compute.SciScript-Python.CasJobs.getPandasDataFrameFromQuery" + else: + task.name = "SciScript-Python.CasJobs.getPandasDataFrameFromQuery" + cvsResponse = executeQuery(queryString, context=context,format="readable") #if the index column is not specified then it will add it's own column which causes problems when uploading the transformed data @@ -337,7 +399,13 @@ def getNumpyArrayFromQuery(queryString, context="MyDB"): """ try: + if Config.isSciServerComputeEnvironment(): + task.name = "Compute.SciScript-Python.CasJobs.getNumpyArrayFromQuery" + else: + task.name = "SciScript-Python.CasJobs.getNumpyArrayFromQuery" + dataFrame = getPandasDataFrameFromQuery(queryString, context) + return dataFrame.as_matrix() except Exception as e: @@ -359,6 +427,12 @@ def uploadPandasDataFrameToTable(dataFrame, tableName, context="MyDB"): .. seealso:: CasJobs.uploadCSVDataToTable """ try: + + if Config.isSciServerComputeEnvironment(): + task.name = "Compute.SciScript-Python.CasJobs.uploadPandasDataFrameToTable" + else: + task.name = "SciScript-Python.CasJobs.uploadPandasDataFrameToTable" + if dataFrame.index.name is not None and dataFrame.index.name != "": sio = dataFrame.to_csv().encode("utf8") else: @@ -387,7 +461,18 @@ def uploadCSVDataToTable(csvData, tableName, context="MyDB"): #if (Config.executeMode == "debug"): # print "Uploading ", sys.getsizeof(CVSdata), "bytes..." - tablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables/" + tableName + + taskName = ""; + if task.name is not None: + taskName = task.name; + task.name = None; + else: + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.CasJobs.uploadCSVDataToTable" + else: + taskName = "SciScript-Python.CasJobs.uploadCSVDataToTable" + + tablesUrl = Config.CasJobsRESTUri + "/contexts/" + context + "/Tables/" + tableName + "?TaskName=" + taskName headers={} headers['X-Auth-Token']= token diff --git a/py3/SciServer/Config.py b/py3/SciServer/Config.py index 36981ec..557c195 100644 --- a/py3/SciServer/Config.py +++ b/py3/SciServer/Config.py @@ -1,3 +1,5 @@ +import json +import os """ The SciServer.Config module contains important parameters for the correct functioning of the SciServer package.\n Although these parameters must be set/defined by the admin or user before the installation of the package, they can also be accessed and changed on-the-fly while on the python session.\n @@ -12,26 +14,61 @@ - **Config.SkyServerWSurl**: defines the base URL of the SkyServer web service API (string). E.g., "https://skyserver.sdss.org" +- **Config.RacmApiURL**: defines the base URL of the RACM API (string). E.g., "https://www.sciserver.org/racm" + - **Config.DataRelease**: defines the SDSS data release (string), to be used to build the full SkyServer API url along with Config.SkyServerWSurl. E.g., "DR13" - **Config.KeystoneTokenPath**: defines the local path (string) to the file containing the user's authentication token in the SciServer-Compute environment. E.g., "/home/idies/keystone.token". Unlikely to change since it is hardcoded in SciServer-Compute. +- **Config.ComputeUrl**: defines the base URL of the Compute webapp (string). E.g., "https://apps.sciserver.org/compute". + +- **Config.SciqueryURL**: defines the base URL of the SciQuery web API (string). E.g., "https://apps.sciserver.org/sciquery-api". + - **Config.version**: defines the SciServer release version tag (string), to which this package belongs. E.g., "sciserver-v1.9.3" """ # URLs for accessing SciServer web services (API endpoints) CasJobsRESTUri = "https://skyserver.sdss.org/CasJobs/RestApi" -AuthenticationURL = "https://portal.sciserver.org/login-portal/keystone/v3/tokens" +AuthenticationURL = "https://apps.sciserver.org/login-portal/keystone/v3/tokens" SciDriveHost = "https://www.scidrive.org" SkyQueryUrl = "http://voservices.net/skyquery/Api/V1" SkyServerWSurl = "https://skyserver.sdss.org" -DataRelease = "DR13" -KeystoneTokenPath = "/home/idies/keystone.token" #this path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment -version = "sciserver-v1.10.0" #sciserver release version +RacmApiURL = "https://apps.sciserver.org/racm" +DataRelease = "DR15" +KeystoneTokenPath = "/home/idies/keystone.token" #the path to the file containing the user's keystone token is hardcoded in the sciserver-compute environment +version = "sciserver-v2.2.0" #sciserver release version +ComputeJobDirectoryFile = "/home/idies/jobs.path" #the path to the file in the "Docker job container" that shows the directory path where the asynchronous compute job is being executed. +ComputeUrl = "https://apps.sciserver.org/compute" +SciqueryURL = "https://apps.sciserver.org/sciquery-api" +ComputeWorkDir = "/home/idies/workspace/" + +def _load_config(filename): + if os.path.exists(filename): + with open(filename) as f: + _config_data = json.load(f) + global CasJobsRESTUri, AuthenticationURL, SciDriveHost, SkyQueryUrl, SkyServerWSurl + global RacmApiURL, DataRelease, KeystoneTokenPath, version, ComputeJobDirectoryFile + global ComputeUrl, SciqueryURL, ComputeWorkDir + CasJobsRESTUri = _config_data.get('CasJobsRESTUri', CasJobsRESTUri) + AuthenticationURL = _config_data.get('AuthenticationURL', AuthenticationURL) + SciDriveHost = _config_data.get('SciDriveHost', SciDriveHost) + SkyQueryUrl = _config_data.get('SkyQueryUrl', SkyQueryUrl) + SkyServerWSurl = _config_data.get('SkyServerWSurl', SkyServerWSurl) + RacmApiURL = _config_data.get('RacmApiURL', RacmApiURL) + DataRelease = _config_data.get('DataRelease', DataRelease) + KeystoneTokenPath = _config_data.get('KeystoneTokenPath', KeystoneTokenPath) + version = _config_data.get('version', version) + ComputeJobDirectoryFile = _config_data.get('ComputeJobDirectoryFile', ComputeJobDirectoryFile) + ComputeUrl = _config_data.get('ComputeUrl', ComputeUrl) + SciqueryURL = _config_data.get('SciqueryURL', SciqueryURL) + ComputeWorkDir = _config_data.get('ComputeWorkDir', ComputeWorkDir) +_CONFIG_DIR = os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')) +_SCISERVER_SYSTEM_CONFIG_DIR = '/etc/' # will not likely exist on non *nix systems +for config_dir in [_SCISERVER_SYSTEM_CONFIG_DIR, _CONFIG_DIR]: + _load_config(os.path.join(config_dir, 'sciserver', 'sciscript.json')) # returns TRUE if the library is running inside the SciServer-Compute, and FALSE if not -import os; def isSciServerComputeEnvironment(): """ Checks whether the library is being run within the SciServer-Compute environment. diff --git a/py3/SciServer/Dask.py b/py3/SciServer/Dask.py new file mode 100644 index 0000000..e7251c3 --- /dev/null +++ b/py3/SciServer/Dask.py @@ -0,0 +1,66 @@ +import requests +from requests.exceptions import HTTPError +from base64 import b64decode +from pathlib import Path +from dask.distributed import Client +from distributed.security import Security +from os.path import expanduser +import SciServer.Authentication +import SciServer.Config +import json + +def getClient(ref_id=None): + """ + Creates a new client for the specified Dask cluster. + + If `ref_id` is not set, cluster connection + properties will be read from the ``~/dask-cluster.json`` file + injected into new SciServer Compute containers automatically + when they are created with an attached Dask cluster. + + :param ref_id: Dask cluster reference ID in SciServer Compute, + defaults to `None` + + :return: Dask client object + :rtype: :class:`dask.distributed.Client` + """ + token = SciServer.Authentication.getToken() + data = None + if (ref_id is None): + with open(expanduser('~/dask-cluster.json')) as f: + data = json.load(f) + else: + try: + response = requests.get(''.join([SciServer.Config.ComputeUrl.rstrip('/'), '/api/dask/clusters/', ref_id]), + params = {'connectionInfo': 'true'}, + headers = {'X-Auth-Token': token}) + response.raise_for_status() + data = response.json()['connection'] + except HTTPError as err: + try: + msg = str(err) + '\n' + response.json()['error']['stackTrace'] + except: + raise err + raise HTTPError(msg) + + base_dir = expanduser('~/.sciserver/dask') + Path(base_dir).mkdir(parents=True, exist_ok=True) + + with open(base_dir + '/ca.pem', 'w+b') as f: + f.write(b64decode(data['ca'])) + + with open(base_dir + '/client-cert.pem', 'w+b') as f: + f.write(b64decode(data['clientCert'])) + + with open(base_dir + '/client-key.pem', 'w+b') as f: + f.write(b64decode(data['clientKey'])) + + scheduler_url = data['schedulerUrl'] + client = Client( + scheduler_url, + security = Security(tls_ca_file = base_dir + '/ca.pem', + tls_client_cert = base_dir + '/client-cert.pem', + tls_client_key = base_dir + '/client-key.pem', + require_encryption=True)) + + return client diff --git a/py3/SciServer/Files.py b/py3/SciServer/Files.py new file mode 100644 index 0000000..0e306fd --- /dev/null +++ b/py3/SciServer/Files.py @@ -0,0 +1,707 @@ +__author__ = 'mtaghiza' + +from SciServer import Authentication, Config +import requests +import json +from io import StringIO +from io import BytesIO +import warnings +import os + + +def getFileServices(verbose=True): + """ + Gets the definitions of file services that a user is able to access. A FileService represents a file system that contains root volumes accessible to the user for public/private data storage. Within each rootVolume, users can create sharable userVolumes for storing files. + + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: list of dictionaries, where each dictionary represents the description of a FileService that the user is able to access. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServices = Files.getFileServices(); + + .. seealso:: Files.getFileServiceFromName + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.getFileServices" + else: + taskName = "SciScript-Python.Files.getFileServices" + + url = Config.RacmApiURL + "/storem/fileservices?TaskName="+taskName; + + headers = {'X-Auth-Token': token} + res = requests.get(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + fileServices = []; + fileServicesAPIs = json.loads(res.content.decode()) + for fileServicesAPI in fileServicesAPIs: + url = fileServicesAPI.get("apiEndpoint") + name = fileServicesAPI.get("name") + url = url + "api/volumes/?TaskName="+taskName; + try: + res = requests.get(url, headers=headers) + except: + if verbose: + warnings.warn("Error when getting definition of FileService named '" + name + "' with API URL '" + fileServicesAPI.get("apiEndpoint") + "'. This FileService might be not available", Warning, stacklevel=2) + + if res.status_code >= 200 and res.status_code < 300: + fileServices.append(json.loads(res.content.decode())); + else: + if verbose: + warnings.warn("Error when getting definition of FileService named '" + name + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode(),Warning, stacklevel=2) + return fileServices; + else: + raise Exception("Error when getting the list of FileServices.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + + +def getFileServicesNames(fileServices=None, verbose=True): + """ + Returns the names and description of the fileServices available to the user. + + :param fileServices: a list of FileService objects (dictionaries), as returned by Files.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: an array of dicts, where each dict has the name and description of a file service available to the user. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServiceNames = Files.getFileServicesNames(); + + .. seealso:: Files.getFileServices + """ + + if fileServices is None: + fileServices = getFileServices(verbose); + + fileServiceNames = []; + for fileService in fileServices: + fileServiceNames.append({"name":fileService.get('name'),"description":fileService.get('description')}) + + return fileServiceNames + + + + + +def getFileServiceFromName(fileServiceName, fileServices=None, verbose=True): + """ + Returns a FileService object, given its registered name. + + :param fileServiceName: name of the FileService, as shown within the results of Files.getFileServices() + :param fileServices: a list of FileService objects (dictionaries), as returned by Files.getFileServices(). If not set, then an extra internal call to Jobs.getFileServices() is made. + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: a FileService object (dictionary) that defines a FileService. A list of these kind of objects available to the user is returned by the function Jobs.getFileServices(). If no fileService can be found, then returns None. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileService = Files.getFileServiceFromName('FileServiceAtJHU'); + + .. seealso:: Files.getFileServices + """ + + if fileServiceName is None: + raise Exception("fileServiceName is not defined.") + else: + if fileServices is None: + fileServices = getFileServices(verbose); + + if fileServices.__len__() > 0: + for fileService in fileServices: + if fileServiceName == fileService.get('name'): + return fileService; + + if verbose: + warnings.warn("FileService of name '" + fileServiceName + "' is not available to the user or does not exist.", Warning, stacklevel=2) + return None + else: + if verbose: + warnings.warn("There are no FileServices available for the user.", Warning, stacklevel=2) + return None + + + + +def __getFileServiceAPIUrl(fileService): + """ + Gets the API endpoint URL of a FileService. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :return: API endpoint URL of the FileService (string). + :example: fileServiceAPIUrl = Files.__getFileServiceAPIUrl(); + + .. seealso:: Files.getFileServiceFromName + """ + + url = None; + if type(fileService) == type(""): + fileServices = getFileServices(False); + _fileService = getFileServiceFromName(fileService, fileServices, verbose=False); + url = _fileService.get("apiEndpoint"); + + else: + url = fileService.get("apiEndpoint"); + + if not url.endswith("/"): + url = url + "/" + + return url + + +def getRootVolumesInfo(fileService, verbose=True): + """ + Gets the names and descriptions of root volumes available to the user in a particular FileService. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: list of dictionaries, where each dictionary contains the name and description of a root volume. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServices = Files.getFileServices(); rootVolumesInfo = getRootVolumesInfo(fileServices[0]) + + .. seealso:: Files.getUserVolumesInfo + """ + if type(fileService) == str: + fileService = getFileServiceFromName(fileService, verbose=verbose) + + rootVolumes = [] + for rootVolume in fileService.get("rootVolumes"): + rootVolumes.append({"rootVolumeName":rootVolume.get("name"), "rootVolumeDescription":rootVolume.get("description")}) + + return rootVolumes + +def getDataVolumesInfo(fileService, verbose=True): + """ + Gets the names and descriptions of data volumes available to the user in a particular FileService. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: list of dictionaries, where each dictionary contains the name and description of a data volume. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServices = Files.getFileServices(); dataVolumesInfo = getDataVolumesInfo(fileServices[0]) + + .. seealso:: Files.getRootVolumesInfo + """ + if type(fileService) == str: + fileService = getFileServiceFromName(fileService, verbose=verbose) + + dataVolumes = [] + for dataVolume in fileService.get("dataVolumes"): + dataVolumes.append({"dataVolumeName":dataVolume.get("name"), "dataVolumeDescription":dataVolume.get("description")}) + + return dataVolumes + + +def getUserVolumesInfo(fileService, rootVolumeName = None, verbose=True): + """ + Gets the names definitions the RootVolumes available in a particular FileService, and of the file services that a user is able to access. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param rootVolumeName: name of root Volume (string) for which the user volumes are fetched. If set to None, then user volumes in all root folders are fetched. + :param verbose: boolean parameter defining whether warnings will be printed (set to True) or not (set to False). + :return: list of dictionaries, where each dictionary contains the name and description of a root volumes that a user is able to access. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: fileServices = Files.getFileServices(); rootVolumeNames = Files.getUserVolumesInfo(fileServices[0]) + + .. seealso:: Files.getRootVolumes, + """ + if type(fileService) == str: + fileService = getFileServiceFromName(fileService, verbose=verbose) + + userVolumes = []; + for rootVolume in fileService.get("rootVolumes"): + for userVolume in rootVolume.get('userVolumes'): + path=os.path.join(rootVolume.get('name'),userVolume.get('owner'),userVolume.get('name')) + if rootVolumeName is not None: + if rootVolume.get('name') == rootVolumeName: + userVolumes.append({"userVolumeName": userVolume.get('name'), "path":path, "userVolumeDescription": userVolume.get('description'), "rootVolumeName":rootVolume.get("name"), "rootVolumeDescription":rootVolume.get("description")}) + + else: + userVolumes.append({"userVolumeName": userVolume.get('name'), "path":path,"userVolumeDescription": userVolume.get('description'),"rootVolumeName": rootVolume.get("name"), "rootVolumeDescription": rootVolume.get("description")}) + + + return userVolumes + + + +def splitPath(path, fileService): + """ + If the topVolume in the 'path' is a rootVolume, then this function splits the path of the form rootVolume/userVolumeOwner/userVolume/relativePath and returns the tuple (rootVolume, userVolumeOwner, userVolume, relativePath, True). On the other hand, + if the topVolume in the 'path' is a dataVolume, then this function splits the path of the form dataVolume/relativePath and returns the tuple (dataVolume, None, None, relativePath, False). + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: file system path (string), starting from the root volume level or data volume level. Example: rootVolume/userVolumeOwner/userVolume/relativePath... or dataVolume/relativePath... + :return: a tuple containing the components: (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeADataVolume) + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.splitPath("Storage/myUserName/persistent/my/Relative/Path", fileServices[0]); + + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName + """ + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + if path.startswith("/"): + path = path[1:] + + + path = path.split("/") + if len(path) == 0: + raise Exception("path variable does not conform with the format 'rootVolume/userVolumeOwner/userVolume/relativePath...' or format 'dataVolume/relativePath...'") + + topVolume = path[0] + isTopVolumeARootVolume = False + + for rootVolume in fileService.get("rootVolumes"): + if rootVolume.get("name") == topVolume: + isTopVolumeARootVolume = True + + if isTopVolumeARootVolume: + if len(path) < 3: + raise Exception("path variable does not conform with the format 'rootVolume/userVolumeOwner/userVolume/relativePath...'") + + rootVolume = path[0] + userVolumeOwner = path[1] + userVolume = path[2] + relativePath = "/".join(path[3:]) + return (rootVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) + else: + if len(path) < 1: + raise Exception("path variable does not conform with the format 'dataVolume/relativePath...'") + + dataVolume = path[0] + relativePath = "/".join(path[1:]) + return (dataVolume, None, None, relativePath, isTopVolumeARootVolume) + + + + +def createUserVolume(fileService, path, quiet=True): + """ + Create a user volume. + + :param fileService: name of fileService (string), or object (dictionary) that defines the file service that contains the user volume. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: path (in the remote file service) to the user volume (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume + :param quiet: if set to False, will throw an error if the User Volume already exists. If True, won't throw an error. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.createUserVolume(fileServices[0], "volumes","newUserVolume"); + + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.createUserVolume" + else: + taskName = "SciScript-Python.Files.createUserVolume" + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (rootVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + + url = __getFileServiceAPIUrl(fileService) + "api/volume/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?quiet="+str(quiet) + "&TaskName="+taskName; + + headers = {'X-Auth-Token': token} + res = requests.put(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + pass; + else: + raise Exception("Error when creating user volume '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def deleteUserVolume(fileService, path, quiet=True): + """ + Delete a user volume. + + :param fileService: name of fileService (string), or object (dictionary) that defines the file service that contains the root volume. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: path (in the remote file service) to the user volume (string), starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume + :param quiet: If set to False, it will throw an error if user volume does not exists. If set to True. it will not throw an error. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.deleteUserVolume("volumes","newUserVolume",fileServices[0]); + + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.createUserVolume" + else: + taskName = "SciScript-Python.Files.createUserVolume" + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (rootVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + + url = __getFileServiceAPIUrl(fileService) + "api/volume/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?quiet="+str(quiet)+"&TaskName="+taskName; + + headers = {'X-Auth-Token': token} + res = requests.delete(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + pass + else: + raise Exception("Error when deleting user volume '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def createDir(fileService, path, quiet=True): + """ + Create a directory. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: path (in the remote file service) to the directory (string), starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/directory or dataVolume/directory + :param quiet: If set to False, it will throw an error if the directory already exists. If set to True. it will not throw an error. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.createDir(fileServices[0], "Storage/myUserName/persistent/myNewDir"); + + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.createDir" + else: + taskName = "SciScript-Python.Files.createDir" + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/folder/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName; + else: + url = __getFileServiceAPIUrl(fileService) + "api/folder/" + topVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName; + + headers = {'X-Auth-Token': token} + res = requests.put(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + pass; + else: + raise Exception("Error when creating directory '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def upload(fileService, path, data="", localFilePath=None, quiet=True): + """ + Uploads data or a local file into a path defined in the file system. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: path (in the remote file service) to the destination file (string), starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/destinationFile.txt or dataVolume/destinationFile.txt + :param data: string containing data to be uploaded, in case localFilePath is not set. + :param localFilePath: path to a local file to be uploaded (string), + :param userVolumeOwner: name (string) of owner of the userVolume. Can be left undefined if requester is the owner of the user volume. + :param quiet: If set to False, it will throw an error if the file already exists. If set to True. it will not throw an error. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.upload(fileServices[0], "myRootVolume/myUserName/myUserVolume/myUploadedFile.txt", None, localFilePath="/myFile.txt"); + + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.UploadFile" + else: + taskName = "SciScript-Python.Files.UploadFile" + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/file/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName + else: + url = __getFileServiceAPIUrl(fileService) + "api/file/" + topVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName + + headers = {'X-Auth-Token': token} + + if localFilePath is not None and localFilePath != "": + with open(localFilePath, "rb") as file: + res = requests.put(url, data=file, headers=headers, stream=True) + else: + if data != None: + res = requests.put(url, data=data, headers=headers, stream=True) + else: + raise Exception("Error: No local file or data specified for uploading."); + + if res.status_code >= 200 and res.status_code < 300: + pass; + else: + raise Exception("Error when uploading file to '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def download(fileService, path, localFilePath=None, format="txt", quiet=True): + """ + Downloads a file from the remote file system into the local file system, or returns the file content as an object in several formats. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: String defining the path (in the remote file service) of the file to be downloaded, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/fileToBeDownloaded.txt or dataVolume/fileToBeDownloaded.txt + :param localFilePath: local destination path of the file to be downloaded. If set to None, then an object of format 'format' will be returned. + :param format: name (string) of the returned object's type (if localFilePath is not defined). This parameter can be "StringIO" (io.StringIO object containing readable text), "BytesIO" (io.BytesIO object containing readable binary data), "response" ( the HTTP response as an object of class requests.Response) or "txt" (a text string). If the parameter 'localFilePath' is defined, then the 'format' parameter is not used and the file is downloaded to the local file system instead. + :param userVolumeOwner: name (string) of owner of the volume. Can be left undefined if requester is the owner of the volume. + :param quiet: If set to False, it will throw an error if the file already exists. If set to True. it will not throw an error. + :return: If the 'localFilePath' parameter is defined, then it will return True when the file is downloaded successfully in the local file system. If the 'localFilePath' is not defined, then the type of the returned object depends on the value of the 'format' parameter (either io.StringIO, io.BytesIO, requests.Response or string). + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.upload(fileServices[0], "Storage/myUserName/persistent/fileToBeDownloaded.txt", localFilePath="/fileToBeDownloaded.txt"); + + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.upload, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + + if localFilePath is not None: + if os.path.isfile(localFilePath) and not quiet: + raise Exception("Error when downloading '" + str(path) + "' from file service '" + str(fileService.get("name")) + "'. Local file '" + localFilePath + "' already exists."); + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.DownloadFile" + else: + taskName = "SciScript-Python.Files.DownloadFile" + + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/file/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?TaskName=" + taskName; + else: + url = __getFileServiceAPIUrl(fileService) + "api/file/" + topVolume + "/" + relativePath + "?TaskName=" + taskName; + + headers = {'X-Auth-Token': token} + + res = requests.get(url, stream=True, headers=headers) + + if res.status_code < 200 or res.status_code >= 300: + raise Exception("Error when downloading '" + str(path) + "' from file service '" + str(fileService.get("name")) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + + if localFilePath is not None and localFilePath != "": + + bytesio = BytesIO(res.content) + theFile = open(localFilePath, "w+b") + theFile.write(bytesio.read()) + theFile.close() + return True + + else: + + if format is not None and format != "": + if format == "StringIO": + return StringIO(res.content.decode()) + if format == "txt": + return res.content.decode() + elif format == "BytesIO": + return BytesIO(res.content) + elif format == "response": + return res; + else: + raise Exception("Unknown format '" + format + "' when trying to download from remote File System the file " + str(relativePath) + ".\n"); + else: + raise Exception("Wrong format parameter value\n"); + + else: + raise Exception("User token is not defined. First log into SciServer.") + + + +def dirList(fileService, path, level=1, options=''): + """ + Lists the contents of a directory. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: String defining the path (in the remote file service) of the directory to be listed, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/directoryToBeListed or dataVolume/directoryToBeListed + :param level: amount (int) of listed directory levels that are below or at the same level to that of the relativePath. + :param options: string of file filtering options. + :return: dictionary containing the directory listing. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); dirs = Files.dirList(fileServices[0], "Storage/myUserName/persistent/", level=2); + + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.delete, Files.upload, Files.download, Files.createDir + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.dirList" + else: + taskName = "SciScript-Python.Files.dirList" + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/jsontree/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?options=" + options + "&level=" + str(level) + "&TaskName=" + taskName; + else: + url = __getFileServiceAPIUrl(fileService) + "api/jsontree/" + topVolume + "/" + relativePath + "?options=" + options + "&level=" + str(level) + "&TaskName=" + taskName; + + headers = {'X-Auth-Token': token} + res = requests.get(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + return json.loads(res.content.decode()); + else: + raise Exception("Error when listing contents of '" + str(path) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def move(fileService, path, destinationFileService, destinationPath, replaceExisting=True, doCopy=True): + """ + Moves or copies a file or folder. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: String defining the origin path (in the remote fileService) of the file or directory to be copied/moved, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/fileToBeMoved.txt or dataVolume/fileToBeMoved.txt + :param destinationFileService: name of fileService (string), or object (dictionary) that defines a destination file service (where the file is moved/copied into). A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param destinationRelativePath: String defining the destination path (in the remote destinationFileService) of the file or directory to be copied/moved, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/recentlyMovedFile.txt or dataVolume/recentlyMovedFile.txt} + :param replaceExisting: If set to False, it will throw an error if the file already exists, If set to True, it will not throw and eeror in that case. + :param doCopy: if set to True, then it will copy the file or folder. If set to False, then the file or folder will be moved. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.move(fileServices[0], "Storage/myUserName/persistent/myFile.txt", fileServices[0], "Storage/myUserName/persistent/myFile2.txt") + + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.delete, Files.upload, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.Move" + else: + taskName = "SciScript-Python.Files.Move" + + if type(fileService) == str: + destinationFileServiceName = destinationFileService + fileService = getFileServiceFromName(fileService) + else: + destinationFileServiceName = destinationFileService['name'] + + + if type(destinationFileService) == str: + destinationFileService = getFileServiceFromName(destinationFileService) + + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + (destinationTopVolume, destinationUserVolumeOwner, destinationUserVolume, destinationRelativePath, isDestinationTopVolumeARootVolume) = splitPath(destinationPath, destinationFileService); + + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/data/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName; + jsonDict = {'destinationPath': destinationRelativePath, + 'destinationRootVolume': destinationTopVolume, + 'destinationUserVolume': destinationUserVolume, + 'destinationOwnerName': destinationUserVolumeOwner, + 'destinationFileService': destinationFileServiceName}; + else: + url = __getFileServiceAPIUrl(fileService) + "api/data/" + topVolume + "/" + relativePath + "?replaceExisting=" + str(replaceExisting) + "&doCopy=" + str(doCopy) + "&TaskName=" + taskName; + jsonDict = {'destinationPath': destinationRelativePath, + 'destinationDataVolume': destinationTopVolume, + 'destinationFileService': destinationFileServiceName}; + + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.put(url, stream=True, headers=headers, json=jsonDict) + + if res.status_code < 200 or res.status_code >= 300: + raise Exception("Error when moving '" + str(path) + "' in file service '" + str(fileService.get("name")) + "' to '" + str(destinationPath) + "' in file service '" + str(destinationFileService.get("name")) + "'. \nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + + else: + raise Exception("User token is not defined. First log into SciServer.") + + + +def delete(fileService, path, quiet=True): + """ + Deletes a directory or file in the File System. + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: String defining the path (in the remote fileService) of the file or directory to be deleted, starting from the root volume level or data volume level. Examples: rootVolume/userVolumeOwner/userVolume/fileToBeDeleted.txt or dataVolume/fileToBeDeleted.txt + :param quiet: If set to False, it will throw an error if the file does not exist. If set to True. it will not throw an error. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.delete(fileServices[0], "Storage/myUserName/persistent/myUselessFile.txt"); + + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.delete" + else: + taskName = "SciScript-Python.Files.delete" + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (topVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + + if isTopVolumeARootVolume: + url = __getFileServiceAPIUrl(fileService) + "api/data/" + topVolume + "/" + userVolumeOwner + "/" + userVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName="+taskName + else: + url = __getFileServiceAPIUrl(fileService) + "api/data/" + topVolume + "/" + relativePath + "?quiet=" + str(quiet) + "&TaskName=" + taskName + + headers = {'X-Auth-Token': token} + res = requests.delete(url, headers=headers) + + if res.status_code >= 200 and res.status_code < 300: + pass; + else: + raise Exception("Error when deleting '" + str(path) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + raise Exception("User token is not defined. First log into SciServer.") + + + +def shareUserVolume(fileService, path, sharedWith, allowedActions, sharee_type="USER"): + """ + Shares a user volume with another user or group + + :param fileService: name of fileService (string), or object (dictionary) that defines a file service. A list of these kind of objects available to the user is returned by the function Files.getFileServices(). + :param path: String defining the path (in the remote fileService) of the user volume to be shared, starting from the root volume level. Example: rootVolume/userVolumeOwner/userVolume. + :param sharedWith: name (string) of user or group that the user volume is shared with. + :param allowedActions: array of strings defining actions the user or group is allowed to do with respect to the shared user volume. E.g.: ["read","write","grant","delete"]. The "grant" action means that the user or group can also share the user volume with another user or group. The "delete" action meand ability to delete the user volume (use with care). + :param sharee_type: type (string) of the entity defined by the "sharedWith" parameter. Can be set to "USER" or "GROUP". + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the FileService API returns an error. + :example: fileServices = Files.getFileServices(); Files.shareUserVolume(getFileServices[0], "Storage/myUserName/myUSerVolume", "userName", ["read","write"], type="USER"); + + .. seealso:: Files.getFileServices(), Files.getFileServiceFromName, Files.createDir, Files.upload, Files.download, Files.dirList + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Files.ShareUserVolume" + else: + taskName = "SciScript-Python.Files.ShareUserVolume" + + if type(fileService) == str: + fileService = getFileServiceFromName(fileService) + + (rootVolume, userVolumeOwner, userVolume, relativePath, isTopVolumeARootVolume) = splitPath(path, fileService); + + + data = [{'name': sharedWith, 'type':sharee_type, 'allowedActions': allowedActions }] + body = json.dumps(data).encode() + + url = __getFileServiceAPIUrl(fileService) + "api/share/" + rootVolume + "/" + userVolumeOwner + "/" + userVolume + "?TaskName="+taskName + + headers = {'X-Auth-Token': token,'Content-Type':'application/json'} + res = requests.patch(url, headers=headers, data=body) + + if res.status_code >= 200 and res.status_code < 300: + pass; + else: + raise Exception("Error when sharing userVolume '" + str(path) + "' in file service '" + str(fileService.get('name')) + "'.\nHttp Response from FileService API returned status code " + str(res.status_code) + ":\n" + res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py new file mode 100644 index 0000000..525598e --- /dev/null +++ b/py3/SciServer/Jobs.py @@ -0,0 +1,506 @@ +__author__ = 'mtaghiza' + +import sys +import os; +import os.path; +from SciServer import Authentication, Config +import requests +import json +import time; + + + +def getDockerComputeDomains(): + """ + Gets a list of all registered Docker compute domains that the user has access to. + + :return: a list of dictionaries, each one containing the definition of a Docker compute domain. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: dockerComputeDomains = Jobs.getDockerComputeDomains(); + + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.cancelJob + """ + token = Authentication.getToken() + if token is not None and token != "": + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.getDockerComputeDomains" + else: + taskName = "SciScript-Python.Jobs.getDockerComputeDomains" + + url = Config.RacmApiURL + "/jobm/rest/computedomains?batch=true&interactive=false&TaskName=" + taskName + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting Docker Compute Domains from JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def getDockerComputeDomainsNames(dockerComputeDomains=None): + """ + Returns the names of the docker compute domains available to the user. + + :param dockerComputeDomains: a list of dockerComputeDomain objects (dictionaries), as returned by Jobs.getDockerComputeDomains(). If not set, then an extra internal call to Jobs.getDockerComputeDomains() is made. + :return: an array of strings, each being the name of a docker compute domain available to the user. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the RACM API returns an error. + :example: dockerComputeDomainsNames = Files.getDockerComputeDomainsNames(); + + .. seealso:: Files.getDockerComputeDomains + """ + if dockerComputeDomains is None: + dockerComputeDomains = getDockerComputeDomains(); + + dockerComputeDomainsNames = []; + for dockerComputeDomain in dockerComputeDomains: + dockerComputeDomainsNames.append(dockerComputeDomain.get('name')) + + return dockerComputeDomainsNames; + + +def getDockerComputeDomainFromName(dockerComputeDomainName, dockerComputeDomains = None): + """ + Returns a DockerComputeDomain object, given its registered name. + + :param dockerComputeDomainName: name of the DockerComputeDomainName, as shown within the results of Jobs.getDockerComputeDomains() + :param dockerComputeDomains: a list of dockerComputeDomain objects (dictionaries), as returned by Jobs.getDockerComputeDomains(). If not set, then an internal call to Jobs.getDockerComputeDomains() is made. + :return: a DockerComputeDomain object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: dockerComputeDomain = Jobs.getDockerComputeDomainFromName('dockerComputeDomainAtJHU'); + + .. seealso:: Jobs.getDockerComputeDomains + """ + if dockerComputeDomainName is None: + raise Exception("dockerComputeDomainName is not defined.") + else: + if dockerComputeDomains is None: + dockerComputeDomains = getDockerComputeDomains(); + + if dockerComputeDomains.__len__() > 0: + for dockerComputeDomain in dockerComputeDomains: + if dockerComputeDomainName == dockerComputeDomain.get('name'): + return dockerComputeDomain; + else: + raise Exception("There are no DockerComputeDomains available for the user."); + + raise Exception("DockerComputeDomain of name '" + dockerComputeDomainName + "' is not available or does not exist."); + + + +def getJobsList(top=10, open=None, start=None, end=None, type='all'): + """ + Gets the list of Jobs submitted by the user. + + :param top: top number of jobs (integer) returned. If top=None, then all jobs are returned. + :param open: If set to 'True', then only returns jobs that have not finished executing and wrapped up (status <= FINISHED). If set to 'False' then only returnes jobs that are still running. If set to 'None', then returns both finished and unfinished jobs. + :param start: The earliest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no lower bound on date. + :param end: The latest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. If set to 'None', then there is no upper bound on date. + :param type: type (string) of jobs returned. Can take values of 'rdb' (for returning only relational database jobs), 'docker' (for returning only Docker jobs) and 'all' (all job types are returned). + :return: a list of dictionaries, each one containing the definition of a submitted job. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. + :example: jobs = Jobs.getJobsList(top=2); + + .. seealso:: Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.getJobsList" + else: + taskName = "SciScript-Python.Jobs.getJobsList" + + + topString = ("top=" + str(top) + "&" if top != None else "") + startString = ("start=" + str(start) + "&" if start != None else "") + endString = ("end=" + str(end) + "&" if end != None else "") + if open is None: + openString = ""; + else: + if open is True: + openString = "open=true&"; + else: + openString = "open=false&"; + + url = Config.RacmApiURL + "/jobm/rest/jobs?" + if(type=='rdb'): + url = Config.RacmApiURL + "/jobm/rest/rdbjobs?" + if(type=='docker'): + url = Config.RacmApiURL + "/jobm/rest/dockerjobs?" + + url = url + topString + startString + endString + "TaskName=" + taskName; + + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting list of jobs from JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + +def getJobDescription(jobId): + """ + Gets the definition of the job, + + :param jobId: Id of job + :return: dictionary containing the description or definition of the job. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. + :example: job1 = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); job2 = Jobs.getJobDescription(job1.get('id')); + + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.getJobDescription" + else: + taskName = "SciScript-Python.Jobs.getJobDescription" + + url = Config.RacmApiURL + "/jobm/rest/jobs/" + str(jobId) + "?TaskName="+taskName + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting from JOBM API the job status of jobId=" + str(jobId) + ".\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return json.loads(res.content.decode()) + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def getJobStatus(jobId): + """ + Gets a dictionary with the job status as an integer value, together with its semantic meaning. The integer value is a power of 2, that is, 1:PENDING, 2:QUEUED, 4:ACCEPTED, 8:STARTED, 16:FINISHED, 32:SUCCESS, 64:ERROR and 128:CANCELED + + :param jobId: Id of job (integer). + :return: dictionary with the integer value of the job status, as well as its semantic meaning. + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error. + :example: jobId = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); status = Jobs.getJobStatus(jobId); + + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + intStatus = getJobDescription(jobId)["status"] + + if intStatus == 1: + return {'status':intStatus, 'statusMeaning':"PENDING", 'jobId':jobId} + elif intStatus == 2: + return {'status': intStatus, 'statusMeaning': "QUEUED", 'jobId':jobId} + elif intStatus == 4: + return {'status': intStatus, 'statusMeaning': "ACCEPTED", 'jobId':jobId} + elif intStatus == 8: + return {'status': intStatus, 'statusMeaning': "STARTED", 'jobId':jobId} + elif intStatus == 16: + return {'status': intStatus, 'statusMeaning': "FINISHED", 'jobId':jobId} + elif intStatus == 32: + return {'status': intStatus, 'statusMeaning': "SUCCESS", 'jobId':jobId} + elif intStatus == 64: + return {'status': intStatus, 'statusMeaning': "ERROR", 'jobId':jobId} + elif intStatus == 128: + return {'status': intStatus, 'statusMeaning': "CANCELED", 'jobId':jobId} + else: + raise Exception("Invalid integer value given to job status.") + + +def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=None, userVolumes=None, dataVolumes=None, resultsFolderPath="", parameters="", jobAlias= ""): + """ + Submits a Jupyter Notebook for execution (as an asynchronous job) inside a Docker compute domain. + + :param notebookPath: path of the notebook within the filesystem mounted in SciServer-Compute (string). Example: notebookPath = '/home/idies/worskpace/persistent/JupyterNotebook.ipynb' + :param dockerComputeDomain: object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). + :param dockerImageName: name (string) of the Docker image for executing the notebook. E.g., dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object. + :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', , 'needsWriteAccess':True}] . A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=None, then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume. + :param dataVolumes: a list with the names of data volumes that will be mounted to the docker Image. E.g.: dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}]. A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted. + :param resultsFolderPath: full path to results folder (string) where the original notebook is copied to and executed. E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder. If not set, then a default folder will be set automatically. + :param parameters: string containing parameters that the notebook might need during its execution. This string is written in the 'parameters.txt' file in the same directory level where the notebook is being executed. + :param jobAlias: alias (string) of job, defined by the user. + :return: the job ID (int) + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. + :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitNotebookJob('/home/idies/workspace/persistent/Notebook.ipynb', dockerComputeDomain, 'Python (astro)', [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}], [{'name':'SDSS_DAS'}], 'param1=1\\nparam2=2\\nparam3=3','myNewJob') + + .. seealso:: Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.submitNotebookJob" + else: + taskName = "SciScript-Python.Jobs.submitNotebookJob" + + if dockerComputeDomain is None: + dockerComputeDomains = getDockerComputeDomains(); + if dockerComputeDomains .__len__() > 0: + dockerComputeDomain = dockerComputeDomains[0]; + else: + raise Exception("There are no dockerComputeDomains available for the user."); + + if dockerImageName is None: + images = dockerComputeDomain.get('images'); + if images.__len__() > 0: + dockerImageName = images[0].get('name') + else: + raise Exception("dockerComputeDomain has no docker images available for the user."); + + uVols = []; + if userVolumes is None: + for vol in dockerComputeDomain.get('userVolumes'): + if 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + + + else: + for uVol in userVolumes: + found = False; + for vol in dockerComputeDomain.get('userVolumes'): + if vol.get('name') == uVol.get('name') and vol.get('rootVolumeName') == uVol.get('rootVolumeName') and vol.get('owner') == uVol.get('owner'): + found = True; + if (uVol.get('needsWriteAccess')): + if uVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + else: + if 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + + if not found: + raise Exception("User volume '" + uVol.get('name') + "' not found within Compute domain") + + datVols = []; + if dataVolumes is None: + for vol in dockerComputeDomain.get('volumes'): + datVols.append({'name': vol.get('name')}); + + else: + for dVol in dataVolumes: + found = False; + for vol in dockerComputeDomain.get('volumes'): + if vol.get('name') == dVol.get('name'): + found = True; + datVols.append({'name': vol.get('name')}); + + if not found: + raise Exception("Data volume '" + dVol.get('name') + "' not found within Compute domain") + + + dockerComputeEndpoint = dockerComputeDomain.get('apiEndpoint'); + + dockerJobModel = { + "command": parameters, + "scriptURI": notebookPath, + "submitterDID": jobAlias, + "dockerComputeEndpoint": dockerComputeEndpoint, + "dockerImageName": dockerImageName, + "resultsFolderURI": resultsFolderPath, + "volumeContainers": datVols, + "userVolumes": uVols + } + data = json.dumps(dockerJobModel).encode() + url = Config.RacmApiURL + "/jobm/rest/jobs/docker?TaskName="+taskName; + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.post(url, data=data, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when submitting a notebook job to the JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return (json.loads(res.content.decode())).get('id') + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageName = None, userVolumes = None, dataVolumes = None, resultsFolderPath = "", jobAlias = ""): + """ + Submits a shell command for execution (as an asynchronous job) inside a Docker compute domain. + + :param shellCommand: shell command (string) defined by the user. + :param dockerComputeDomain: object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains(). + :param dockerImageName: name (string) of the Docker image for executing the notebook. E.g., dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object. + :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image. + E.g., userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', , 'needsWriteAccess':True}] + A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=None, then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume. + :param dataVolumes: a list with the names of data volumes that will be mounted to the docker Image. + E.g., dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}]. + A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted. + :param resultsFolderPath: full path to results folder (string) where the shell command is executed. E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder. If not set, then a default folder will be set automatically. + :param jobAlias: alias (string) of job, defined by the user. + :return: the job ID (int) + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error, or if the volumes defined by the user are not available in the Docker compute domain. + :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitShellCommandJob('pwd', dockerComputeDomain, 'Python (astro)', [{'name':'persistent'},{'name':'scratch', 'needsWriteAccess':True}], [{'name':'SDSS_DAS'}], 'myNewJob') + + .. seealso:: Jobs.submitNotebookJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob + """ + + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.submitShellCommandJob" + else: + taskName = "SciScript-Python.Jobs.submitShellCommandJob" + + + if dockerComputeDomain is None: + dockerComputeDomains = getDockerComputeDomains(); + if dockerComputeDomains .__len__() > 0: + dockerComputeDomain = dockerComputeDomains[0]; + else: + raise Exception("There are no dockerComputeDomains available for the user."); + + if dockerImageName is None: + images = dockerComputeDomain.get('images'); + if images.__len__() > 0: + dockerImageName = images[0].get('name') + else: + raise Exception("dockerComputeDomain has no docker images available for the user."); + + uVols = []; + if userVolumes is None: + for vol in dockerComputeDomain.get('userVolumes'): + if 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + + + else: + for uVol in userVolumes: + found = False; + for vol in dockerComputeDomain.get('userVolumes'): + if vol.get('name') == uVol.get('name'): + found = True; + if (uVol.get('needsWriteAccess')): + if uVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + else: + if 'write' in vol.get('allowedActions'): + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True}); + else: + uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False}); + + if not found: + raise Exception("User volume '" + uVol.get('name') + "' not found within Compute domain") + + datVols = []; + if dataVolumes is None: + for vol in dockerComputeDomain.get('volumes'): + datVols.append({'name': vol.get('name')}); + + else: + for dVol in dataVolumes: + found = False; + for vol in dockerComputeDomain.get('volumes'): + if vol.get('name') == dVol.get('name'): + found = True; + datVols.append({'name': vol.get('name')}); + + if not found: + raise Exception("Data volume '" + dVol.get('name') + "' not found within Compute domain") + + + dockerComputeEndpoint = dockerComputeDomain.get('apiEndpoint'); + + dockerJobModel = { + "command": shellCommand, + "submitterDID": jobAlias, + "dockerComputeEndpoint": dockerComputeEndpoint, + "dockerImageName": dockerImageName, + "volumeContainers": datVols, + "userVolumes": uVols, + "resultsFolderURI": resultsFolderPath + } + data = json.dumps(dockerJobModel).encode() + url = Config.RacmApiURL + "/jobm/rest/jobs/docker?TaskName="+taskName; + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.post(url, data=data, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when submitting a job to the JOBM API.\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return (json.loads(res.content.decode())).get('id') + else: + raise Exception("User token is not defined. First log into SciServer.") + +def cancelJob(jobId): + """ + Cancels the execution of a job. + + :param jobId: Id of the job (integer) + :raises: Throws an exception if the HTTP request to the Authentication URL returns an error. Throws an exception if the HTTP request to the JOBM API returns an error. + :example: job = Jobs.submitShellCommandJob(Jobs.getDockerComputeDomains()[0],'pwd', 'Python (astro)'); isCanceled = Jobs.cancelJob(job.get('id')); + + .. seealso:: Jobs.submitNotebookJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains. + """ + token = Authentication.getToken() + if token is not None and token != "": + + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.Jobs.cancelJob" + else: + taskName = "SciScript-Python.Jobs.cancelJob" + + + url = Config.RacmApiURL + "/jobm/rest/jobs/" + str(jobId) + "/cancel?TaskName="+taskName + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.post(url, headers=headers, stream=True) + + if res.status_code != 200: + raise Exception("Error when getting from JOBM API the job status of jobId=" + str(jobId) + ".\nHttp Response from JOBM API returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + pass; + else: + raise Exception("User token is not defined. First log into SciServer.") + + +def waitForJob(jobId, verbose=False, pollTime = 5): + """ + Queries regularly the job status and waits until the job is completed. + + :param jobId: id of job (integer) + :param verbose: if True, will print "wait" messages on the screen while the job is still running. If False, will suppress the printing of messages on the screen. + :param pollTime: idle time interval (integer, in seconds) before querying again for the job status. Minimum value allowed is 5 seconds. + :return: After the job is finished, returns a dictionary object containing the job definition. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the JOBM API returns an error. + :example: dockerComputeDomain = Jobs.getDockerComputeDomains()[0]; job = Jobs.submitShellCommandJob(dockerComputeDomain,'pwd', 'Python (astro)');Jobs.waitForJob(job.get('id')) + + .. seealso:: Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.submitNotebookJob, Jobs.submitShellCommandJob + """ + try: + minPollTime = 5 # in seconds + complete = False + waitingStr = "Waiting..." + back = "\b" * len(waitingStr) + if verbose: + print(waitingStr) + + while not complete: + if verbose: + #print back, + print(waitingStr) + jobDesc = getJobStatus(jobId) + jobStatus = jobDesc["status"] + if jobStatus >= 32: + complete = True + if verbose: + #print back, + print("Done!") + else: + time.sleep(max(minPollTime,pollTime)); + + return jobDesc + except Exception as e: + raise e; + diff --git a/py3/SciServer/SciDrive.py b/py3/SciServer/SciDrive.py index b3df2db..523e02d 100644 --- a/py3/SciServer/SciDrive.py +++ b/py3/SciServer/SciDrive.py @@ -7,6 +7,15 @@ from SciServer import Config, Authentication +class Task: + """ + The class TaskName stores the name of the task that executes the API call. + """ + name = None + + +task = Task(); + def createContainer(path): """ Creates a container (directory) in SciDrive @@ -25,7 +34,13 @@ def createContainer(path): 'uri="vos://' + Config.SciDriveHost + '!vospace/' + path + '">' '' '') - url = Config.SciDriveHost + '/vospace-2.0/nodes/' + path + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.createContainer" + else: + taskName = "SciScript-Python.SciDrive.createContainer" + + url = Config.SciDriveHost + '/vospace-2.0/nodes/' + path + "?TaskName=" + taskName; data = str.encode(containerBody) headers = {'X-Auth-Token': token, 'Content-Type': 'application/xml'} res = requests.put(url, data=data, headers=headers) @@ -52,7 +67,14 @@ def upload(path, data="", localFilePath=""): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SciDriveHost + '/vospace-2.0/1/files_put/dropbox/' + path + + taskName = "" + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.upload" + else: + taskName = "SciScript-Python.SciDrive.upload" + + url = Config.SciDriveHost + '/vospace-2.0/1/files_put/dropbox/' + path + "?TaskName=" + taskName; headers = {'X-Auth-Token': token} if(localFilePath != ""): with open(localFilePath, "rb") as file: @@ -85,7 +107,17 @@ def publicUrl(path): token = Authentication.getToken() if token is not None and token != "": - url = Config.SciDriveHost + '/vospace-2.0/1/media/sandbox/' + str(path) + taskName = ""; + if task.name is not None: + taskName = task.name; + task.name = None; + else: + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.publicUrl" + else: + taskName = "SciScript-Python.SciDrive.publicUrl" + + url = Config.SciDriveHost + '/vospace-2.0/1/media/sandbox/' + str(path) + "?TaskName=" + taskName headers = {'X-Auth-Token': token} res = requests.get(url, headers=headers) if res.status_code != 200: @@ -113,7 +145,13 @@ def directoryList(path=""): token = Authentication.getToken() if token is not None and token != "": - url = Config.SciDriveHost + "/vospace-2.0/1/metadata/sandbox/" + str(path) + "?list=True&path=" + str(path) + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.directoryList" + else: + taskName = "SciScript-Python.SciDrive.directoryList" + + url = Config.SciDriveHost + "/vospace-2.0/1/metadata/sandbox/" + str(path) + "?list=True&path=" + str(path) + "&TaskName=" + taskName; headers = {'X-Auth-Token': token} res = requests.get(url, headers=headers) if res.status_code != 200: @@ -143,6 +181,12 @@ def download(path, format="text", localFilePath=""): token = Authentication.getToken() if token is not None and token != "": + taskName = ""; + if Config.isSciServerComputeEnvironment(): + task.name = "Compute.SciScript-Python.SciDrive.download" + else: + task.name = "SciScript-Python.SciDrive.download" + fileUrl = publicUrl(path) res = requests.get(fileUrl, stream=True) if res.status_code != 200: @@ -194,7 +238,13 @@ def delete(path): 'uri="vos://' + Config.SciDriveHost + '!vospace/' + path + '">' '' '') - url = Config.SciDriveHost + '/vospace-2.0/nodes/' + path + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SciDrive.delete" + else: + taskName = "SciScript-Python.SciDrive.delete" + + url = Config.SciDriveHost + '/vospace-2.0/nodes/' + path + "?TaskName=" + taskName; data = str.encode(containerBody) headers = {'X-Auth-Token': token, 'Content-Type': 'application/xml'} res = requests.delete(url, data=data, headers=headers) diff --git a/py3/SciServer/SciQuery.py b/py3/SciServer/SciQuery.py new file mode 100644 index 0000000..76edf39 --- /dev/null +++ b/py3/SciServer/SciQuery.py @@ -0,0 +1,1579 @@ +from SciServer import Authentication, Config, Files, Jobs +import pandas as pd +import json +import requests +import cachetools.func +from collections.abc import Iterable +from datetime import datetime +import warnings +from pathlib import PurePosixPath +from typing import Union, List +import time + + +class OutputType: + """ + Contains a set of allowed database output types. + """ + FILE_JSON = "FILE_JSON" + FILE_CSV = "FILE_CSV" + DATABASE_TABLE = "TABLE" + + +@cachetools.func.ttl_cache(maxsize=128, ttl=120) +def _get_file_service(file_service: str = None): + file_services = Files.getFileServices(verbose=False) + if file_service is None: + if len(file_services) > 0: + return file_services[0] + raise Exception("No file services available for the user.") + else: + for fs in file_services: + if file_service == fs.get("name") or file_service == fs.get("identifier"): + return fs + raise Exception("Unable to find fileService") + + +class Output: + + def __init__(self, + name: str = "output.json", + output_type: str = OutputType.FILE_JSON, + statement_indexes: Union[int, List[int]] = 1): + """ + Base class for output objects, including files or database tables. + """ + if type(name) != str: + raise TypeError("Invalid type for input parameter 'name'.") + if type(output_type) != str: + raise TypeError("Invalid type for input parameter 'output_type'.") + if type(statement_indexes) not in [list, int]: + raise TypeError("Invalid type for input parameter 'statement_indexes'.") + self.name = name + self.output_type = output_type + self.statement_indexes = None + self.set_statement_indexes(statement_indexes) + + def set_statement_indexes(self, statement_indexes: Union[int, List[int]] = 1): + """ + Sets the index(es) of the sql statement(s) within the input query, whose result-set(s) is(are) going to be + written into this Output. + + :param statement_indexes: integer or list of integers, which are the indices (starting with 1) of the sql + statements within the input query, whose resultsets are going to be written into this Output. + """ + if not isinstance(statement_indexes, Iterable): + statement_indexes = [statement_indexes] + for index in statement_indexes: + if type(index) != int or index <= 0: + raise TypeError("Invalid type for input parameter 'statement_indexes'") + self.statement_indexes = [i for i in sorted(set(statement_indexes))] + + def __str__(self): + return "Output of name = {}, type= {}, statement_indexes = {}".format(self.name, self.output_type, + self.statement_indexes) + def __repr__(self): + return "Output(name = {}, type= {}, statement_indexes = {})".format(self.name, self.output_type, + self.statement_indexes) + + +class FileOutput(Output): + + def __init__(self, + name: str = "output.json", + output_type: str = OutputType.FILE_JSON, + statement_indexes: Union[int, List[int]] = 1, + file_service: str = None): + """ + Defines the output of a database query into a file. + + :param name: name of the file (string), such as "result.json" + :param output_type: type (string) of the file containing the query result(s) (e.g., "FILE_JSON"). + As set of possible values is given by the static members of class 'SciQuery.OutputTargetType' + :param statement_indexes: list of integers or integer. Each integer value denotes the index or position (>=1) + of the sql statements within the input query, whose resultset is going to be written into this OutputTarget + :param file_service: string denoting name or identifier of file service where the output file is written into. + """ + if file_service: + file_service = FileOutput.find_file_service(file_service) + self.file_service_name = file_service['name'] + self.file_service_identifier = file_service['identifier'] + else: + self.file_service_name = None + self.file_service_identifier = None + + if not name: + raise NameError("Input parameter name cannot be empty or None") + name = name.rstrip("/") + file_path = PurePosixPath(name) + if name == file_path.name: # means no path included in 'name' input parameter + self.file_base_path = None + self.file = name + self.path = None + self.file_service_path = None + else: + if not name.startswith(Config.ComputeWorkDir): + file_path = PurePosixPath(Config.ComputeWorkDir + name) # in case it is relative path + self.file_base_path = str(file_path.parent) + self.file_base_path = self.file_base_path if self.file_base_path.endswith("/") \ + else self.file_base_path + "/" + self.file = file_path.name + self.path = self.file_base_path + self.file + self.file_service_path = self.path.replace(Config.ComputeWorkDir, "", 1) + name = file_path.name + + super().__init__(name, output_type, statement_indexes) + + def get_path(self) -> str: + """ + Get file output path on SciServer's FileSystem + """ + if self.path: + return self.path + else: + raise Exception("Attribute 'file_base_path' is not set.") + + + @classmethod + def get_default(cls): + """ + Gets a feault FileOutput object filled with default values: JSON output file where only the 1st SQL statement of + the query is written in it. + """ + return cls("result.json", OutputType.FILE_JSON, 1) + + @staticmethod + def build_file_base_path(top_volume: str = "Temporary", + user_volume: str = "scratch", + user_volume_owner_name: str = "", + relative_path: str = "sciqueryjobs", + add_date_ending: bool = False) -> str: + """ + Builds the base path for an output file located in the SciServer filesystem. + + :param top_volume: name (str) of top volume in SciServer's filesystem, such as "Temporary" or "Storage". + :param user_volume: name (str) of user volume in SciServer's filesystem. + :param user_volume_owner_name: name (str) of user volume owner in SciServer's filesystem. + :param relative_path: relative path (str) after /// + :param add_date_ending: if True, then the relative path is built from the current date. + :return: path string + """ + if not top_volume: + raise NameError("Input parameter top_volume cannot be empty or None") + + if add_date_ending: + now = datetime.now() + date1 = now.strftime("%Y-%m-%d") + date2 = now.strftime("%Hh%Mm%S.%fs") + relative_path = "{0}/{1}/{2}".format(relative_path.rstrip('/'), date1, date2) + + if user_volume: + if not user_volume_owner_name: + user_volume_owner_name = SciQuery.get_user().userName + path = str(PurePosixPath(Config.ComputeWorkDir, top_volume, user_volume_owner_name, user_volume, relative_path)) + else: + path = str(PurePosixPath(Config.ComputeWorkDir, top_volume, relative_path)) + return path if path.endswith("/") else path + "/" + + @staticmethod + def find_file_service(file_service: Union[str, dict] = None) -> dict: + """ + Gets FileService info + """ + if isinstance(file_service, dict): + file_service = file_service.get("identifier") + return _get_file_service(file_service) + + def __str__(self): + return f"File Output of name = {self.name}, type= {self.output_type}, statement_indexes = " \ + f"{self.statement_indexes}" + + def __repr__(self): + return f"FileOutput(name= {self.name}, type= {self.output_type}, statement_indexes = {self.statement_indexes})" + + +class DatabaseTableOutput(Output): + + def __init__(self, + table: str = "resultTable", + database: str = None, + statement_indexes: Union[int, List[int]] = 1, + rdb_compute_domain: str = None, + schema: str = ""): + + """ + Defines the output of a database query into a database table. + + :param table: name of the output database table (string), such as "resultTable" + :param database: name of the database (string) where the output table in created. If it is owned explicitly by + a user, then it should follow the pattern "mydb:" + :param statement_indexes: list of integers or integer. Each integer value denotes the index or position (>=1) + :param rdb_compute_domain: name (string) of the relational database (RDB) compute domain that contains the + database, or object of class RDBComputeDomain corresponding to it. + Name of such domains available to the user is returned by the function Jobs.getRDBComputeDomainNames(). + :param schema: database schema (string) + of the sql statements within the input query, whose resultset is going to be written into this OutputTarget + """ + if type(table) != str or type(schema) != str: + raise TypeError("Input parameter(s) 'table' or 'schema' should be of type string.") + + domain = RDBComputeDomains.get_default_rdb_compute_domain() if not rdb_compute_domain else rdb_compute_domain + if not database: + if type(domain) == str: + database = SciQuery.get_rdb_compute_domains().get_rdb_compute_domain(domain).get_default_database().name + else: + database = domain.get_default_database().name + domain = domain.name + + + self.table = table + self.database = database + self.rdb_compute_domain_name = domain + self.schema = schema + name = ".".join([self.rdb_compute_domain_name, self.database, self.schema, self.table]) + super().__init__(name, OutputType.DATABASE_TABLE, statement_indexes) + + @classmethod + def get_default(cls): + """ + Gets a OutputTarget object filled with default values: JSON output file where only the 1st SQL statement of + the query is written in it. + """ + return cls(table = "resultTable", + database = RDBComputeDomains.get_default_rdb_compute_domain().get_default_database().name, + rdb_compute_domain = RDBComputeDomains.get_default_rdb_compute_domain().name, + schema = "", + statement_indexes = [1]) + + def __str__(self): + return "Database Table Output of table= {}, database= {}, rdb_compute_domain_name= {}, schema= {}, " \ + "statement_indexes= {}".format(self.table, self.database, self.rdb_compute_domain_name, self.schema, + self.statement_indexes) + + def __repr__(self): + return "DatabaseTableOutput(table= {}, database= {}, rdb_compute_domain_name= {}, schema= {}, " \ + "statement_indexes= {})".format(self.table, self.database, self.rdb_compute_domain_name, self.schema, + self.statement_indexes) + + +class Outputs(list): + + def __init__(self, *outputs): + """ + Contains a list of output objects, defining database query result outputs. + """ + super().__init__() + for output in outputs: + outs = output if isinstance(output, Iterable) else [output] + for out in outs: + self.append(out) + + def append(self, obj): + """ + Appends an Output object to this list. + """ + if isinstance(obj, Output): + super().append(obj) + else: + raise NameError("Input object is not a subclass of the 'Output' class.") + + def get_target_list(self, file_base_path: str = None, file_service: str = None): + """ + Gets list of output targets. + """ + targets = [] + fs = FileOutput.find_file_service(file_service) + for output in self: + for index in output.statement_indexes: + location = output.name + if output.output_type != OutputType.DATABASE_TABLE: # for files + if not output.file_service_identifier: + file_service_identifier = fs.get('identifier') + else: + file_service_identifier = output.file_service_identifier + + if not output.file_base_path: + if file_base_path: + location = file_base_path.rstrip("/") + "/" + output.file + else: + location = FileOutput.build_file_base_path().rstrip("/") + "/" + output.file + else: + location = output.file_base_path.rstrip("/") + "/" + output.file + + location = file_service_identifier + ":" + location; + targets.append({'location': location, 'type': output.output_type, 'resultNumber': index}) + return targets + + @staticmethod + def get_default(): + """ + Gets an OutputList with one element, consisting of a JSON FileOutput object. + """ + return Outputs(FileOutput(name="result.json", output_type=OutputType.FILE_JSON, statement_indexes = [1])) + + +class RDBJob: + + _JOB_STATUS_MAP = {1: "PENDING", 2: "QUEUED", 4: "ACCEPTED", 8: "STARTED", 16: "FINISHED", 32: "SUCCESS", + 64: "ERROR", 128: "CANCELED"} + + def __init__(self, job): + """ + Contains the definition of a job consisting on a query run in a Relational Database (RDB). + + :param job: can be the job ID (string), or a dictionary containing all the attributes of an RDBJob object. + """ + if type(job) != dict: + job = Jobs.getJobDescription(job) + + self.id = job.get('id') + self.alias = job.get('submitterDID') if job.get('submitterDID') is not None else job.get('alias') + self._submitter_trust_id = job.get('submitterTrustId') if job.get('submitterTrustId') is not None else \ + job.get('_submitter_trust_id') + self._run_by_uuid = job.get('runByUUID') if job.get('runByUUID') is not None else job.get('_run_by_uuid') + self.submission_time = self._get_datetime(job.get('submissionTime') if job.get('submissionTime') is not None + else job.get('submission_time')) + self.start_time = self._get_datetime(job.get('startTime') if job.get('startTime') is not None + else job.get('start_time')) + self.end_time = self._get_datetime(job.get('endTime') if job.get('endTime') is not None + else job.get('end_time')) + self.duration = job.get('duration') + self.timeout = job.get('timeout') + self._messages = job.get('messages') if job.get('messages') is not None else job.get('_messages') + self.message_list = [m.get("content") for m in self._messages] if self._messages is not None else [] + self.status = job.get('status') + self.status_string = RDBJob.get_job_status(job.get('status')) + self._results_folder_uri = job.get('resultsFolderURI') if job.get('resultsFolderURI') is not None \ + else job.get('_results_folder_uri') + self._type = job.get('type') if job.get('type') is not None else job.get('_type') + self.user_name = job.get('username') if job.get('username') is not None else job.get('user_name') + self.input_sql = job.get('inputSql') if job.get('inputSql') is not None else job.get('input_sql') + self.targets = job.get('targets') + self.database_name = job.get('databaseContextName') \ + if job.get('databaseContextName') is not None else job.get('database_name') + self._rdb_resource_context_uuid = job.get('rdbResourceContextUUID') \ + if job.get('rdbResourceContextUUID') is not None else job.get('_rdb_resource_context_uuid') + self.rdb_compute_domain_name = job.get('rdbDomainName') if job.get('rdbDomainName') is not None \ + else job.get('rdb_compute_domain_name') + self.rdb_compute_domain_id = job.get('rdbDomainId') if job.get('rdbDomainId') is not None \ + else job.get('rdb_compute_domain_id') + self.outputs = self._get_outputs() + self.get_job_status = self._get_job_status_string + + def get_metadata(self, result_format="pandas") -> pd.DataFrame: + """ + Gets this RDBJob's metadata as a Pandas DataFrame. + """ + data = [] + column_names = [] + if result_format == "pandas": + for attr, value in self.__dict__.items(): + import inspect + if not attr.startswith("_") and not inspect.ismethod(value): # no private members nor methods + data.append(value) + column_names.append(attr) + df = pd.DataFrame([data], columns=column_names) + df.name = str(RDBJob) + return df + elif result_format == "dict": + return self.__dict__ + else: + raise Exception("Invalid value for input parameter 'result_format'.") + + @staticmethod + def get_job_status(status: int) -> str: + """ + Gets job status string from its integer representation. + """ + return RDBJob._JOB_STATUS_MAP.get(status) + + @staticmethod + def get_job(job_id: int): + """ + Gets RDBJob object from its Id. + """ + return RDBJob(Jobs.getJobDescription(job_id)) + + def cancel(self): + """ + Cancels this RDBJob. + """ + Jobs.cancelJob(self.id) + + def refresh(self): + """ + Refreshes metadata and info of this RDBJob. + """ + self.__init__(Jobs.getJobDescription(self.id)) + + def _get_job_status_string(self) -> str: + return RDBJob.get_job_status(self.status) + + def _get_outputs(self) -> Outputs: + output_targets = {} + for t in self.targets: + i = (t['location'], t['type']) + if i not in output_targets: + output_targets[i] = [t['resultNumber']] + else: + output_targets[i].append(t['resultNumber']) + + outputs = Outputs() + for k in output_targets: + if k[1] == OutputType.DATABASE_TABLE: + p = k[0].split(".") + outputs.append(DatabaseTableOutput(table=p[3], database=p[1], rdb_compute_domain=p[0], schema=p[2], + statement_indexes=output_targets[k])) + else: + file_parts = k[0].split(":") + file_service_identifier = file_parts[0] + name = file_parts[1] + outputs.append(FileOutput(name=name, output_type=k[1], statement_indexes=output_targets[k], + file_service=file_service_identifier)) + + return outputs + + def _get_output_from_index(self, ind: int): + if ind > len(self.outputs) - 1: + raise ValueError("Index is outside of the index range in the outputs list.") + return self.outputs[ind] + + def get_output_path(self, output: Union[Output, int] = 0) -> str: + """ + Gets output path on SciServer's filesystem, if output type is a file. + """ + out = self._get_output_from_index(output) if isinstance(output, int) else output + if out.output_type == OutputType.DATABASE_TABLE: + raise TypeError("Output is not a file but a database") + return out.get_path() + + def get_output_as_string(self, output: Union[Output, int, str] = None): + """ + Gets content of output file in SciServer's filesystem as a string. + """ + if not isinstance(output, str): + out = self._get_output_from_index(output) if isinstance(output, int) else output + file_path = self.get_output_path(out) + else: + file_path = output + + if Config.isSciServerComputeEnvironment(): + with open(file_path, ) as f: + data = f.read() + else: + if isinstance(output, str): + raise Exception(f"Cannot find file_path {output} in local file system.") + + fs = FileOutput.find_file_service(out.file_service_identifier) + path = out.file_service_path + data = Files.download(fs, path, format="txt", quiet=True) + + return data + + def get_json_output(self, output: Union[Output, int, str] = 0) -> dict: + """ + Gets content of output file in SciServer's filesystem as a dictionary. + """ + data_dict = json.loads(self.get_output_as_string(output)) + return data_dict.get("Result") + + def get_dataframe_from_output(self, output: Union[Output, int] = 0, result_index: int = 0) -> pd.DataFrame: + """ + Gets query output as a Pandas DatFrame. + """ + out = self._get_output_from_index(output) if isinstance(output, int) else output + if out.output_type == OutputType.FILE_JSON: + result = self.get_json_output(out)[result_index] + df = pd.DataFrame(result['Data'], columns=result['ColumnNames']) + df.name = result['TableName'] + elif out.output_type == OutputType.FILE_CSV: + df = pd.read_csv(out.get_path(), skiprows=1) + elif out.output_type == OutputType.DATABASE_TABLE: + sq = SciQuery(rdb_compute_domain=out.rdb_compute_domain_name, database=out.database) + query = f"select * from {out.table};" + df = sq.execute_query(query, write_job_id=False) + else: + raise Exception(f"Output type {out.output_type} not supported") + return df + + def _get_datetime(self, time): + return datetime.fromtimestamp(time / 1000.0) if time is not None else None + + def __str__(self): + return "RDB Job of id={}".format(self.id) + + def __repr__(self): + return "RDBJob(id={})".format(self.id) + + +class Database: + + def __init__(self, rdb_compute_domain: Union[str, int, dict], database: Union[str, int, dict]): + """ + Defines a database context where users can run sql queries. + + :param rdb_compute_domain: Parameter that identifies the relation database domain or environment that + contains the database. Could be either its name (string), ID (integer), or a dictionary containing + the attributes of the domain. + :param database: defines the database. Can be either the database name (string), ID (integer), or a dictionary + containing all the attributes of an object of class Database. + """ + if type(database) not in [str, int, dict]: + raise TypeError("Invalid type for input parameter 'database'.") + + if type(rdb_compute_domain) not in [str, int, dict]: + raise TypeError("Invalid type for input parameter 'rdb_compute_domain'.") + + if type(rdb_compute_domain) != dict: + domain = RDBComputeDomain(rdb_compute_domain) + else: + domain = rdb_compute_domain + + if type(database) != dict: + dbs = domain.get('databaseContexts') or domain.get('databases') + if type(database) == str: + database = [db for db in dbs if db.get('name') == database] + else: + database = [db for db in dbs if db.get('id') == database] + + if len(database) == 0: + raise NameError("Unable to find database.") + else: + database = database[0] + + self.id = database.get('id') + self._racm_id = database.get('_racm_id') if database.get('_racm_id') is not None else database.get('racmId') + self.name = database.get('name') if database.get('name') is not None else database.get('contextName') + self.description = database.get('description') + self.vendor = database.get('vendor') + self.schemas = database.get('dbSchemas') if database.get('dbSchemas') is not None else database.get('schemas') + self.rdb_compute_domain_name = domain.get('name') if domain.get('name') is not None else \ + domain.get('displayName') + self.rdb_compute_domain_id = domain.get('id') + + def get_metadata(self) -> pd.DataFrame: + """ + Gets Database metadata. + """ + data = [] + column_names = ['database_name', 'database_description', 'database_vendor', 'database_id', + 'rdb_compute_domain_name', 'rdb_compute_domain_id'] + data.append([self.name, self.description, self.vendor, self.id, self.rdb_compute_domain_name, + self.rdb_compute_domain_id]) + df = pd.DataFrame(data=data, columns=column_names, index=[self.id]) + df = df.astype({"rdb_compute_domain_id": int, "database_id": int}) + return df + + def __str__(self): + return "Database of name={}, id={} and rdb_compute_domain_name={}".format(self.name, self.id, + self.rdb_compute_domain_name) + + def __repr__(self): + return "Database(name={}, id={}, rdb_compute_domain_name={})".format(self.name, self.id, + self.rdb_compute_domain_name) + + +class RDBComputeDomain: + + def __init__(self, rdb_compute_domain: Union[str, int, dict]): + """ + Creates an instance of an RDBComputeDomain, which defines a domain or environment with databases that users. + are able to query. + + :param rdb_compute_domain: Parameter that identifies the domain. Could be either its name (string), + ID (integer), or a dictionary containing all the attributes of the domain. + """ + if type(rdb_compute_domain) not in [str, int, dict]: + raise TypeError("Invalid type for input parameter 'rdb_compute_domain'.") + + if type(rdb_compute_domain) != dict: + domains = SciQuery.get_rdb_compute_domains("dict") + if type(rdb_compute_domain) == str: + domain = [d for d in domains if d.get('name') == rdb_compute_domain] + elif type(rdb_compute_domain) == int: + domain = [d for d in domains if d.get('id') == rdb_compute_domain] + else: + raise TypeError("Invalid type for input parameter 'rdb_compute_domain'.") + + if len(domain) > 0: + rdb_compute_domain = domain[0] + else: + raise NameError("Unable to find rdbComputeDomain {0}.".format(rdb_compute_domain)) + + self.id = rdb_compute_domain.get('id') + self._racm_id = rdb_compute_domain.get('_racm_id') if rdb_compute_domain.get('_racm_id') is not None else \ + rdb_compute_domain.get('racmId') + self.name = rdb_compute_domain.get('name') if rdb_compute_domain.get('name') is not None else \ + rdb_compute_domain.get('displayName') + self.description = rdb_compute_domain.get('description') + dbs = [] + databases = rdb_compute_domain.get('dbContexts') if \ + rdb_compute_domain.get('dbContexts') is not None else rdb_compute_domain.get('databases') + for db_name, db_dict in databases.items(): + dbs.append(Database(rdb_compute_domain, db_dict)) + self.databases = dbs + + def get_database_names(self) -> list: + """ + Gets a list of the names of databases in an RDBComputeDomain. + + :return: list of database names (strings) + :example: dbnames = SciQuery.get_database_names(rdbComputeDomainName); + + .. seealso:: SciQuery.get_databases_metadata + """ + return [db.name for db in self.databases] + + def get_database(self, database: Union[str, int, dict, Database]) -> Database: + """ + Gets Database + """ + if type(database) == str: + dbs = [db for db in self.databases if db.name == database] + elif type(database) == int: + dbs = [db for db in self.databases if db.id == database] + elif type(database) == dict: + dbs = [db for db in self.databases if db.id == database.get('id')] + elif isinstance(database, Database): + return self.get_database(database.id) + else: + raise TypeError("Invalid type for input parameter 'database'.") + if len(dbs) == 0: + raise NameError("Database not found in list of available databases.") + else: + return dbs[0] + + def get_default_database(self) -> Database: + """ + Gets default database. + """ + dbs = [db for db in self.databases if db.name == SciQuery.get_mydb_name()] + if len(dbs) > 0: + return dbs[0] + elif len(self.databases) > 0: + return self.databases[0] + else: + raise Exception("No default database available.") + + def get_metadata(self, do_include_databases: bool = False) -> pd.DataFrame: + """ + Gets metadata of this RDBComputeDomain. + """ + + column_names = ['rdb_compute_domain_name', 'rdb_compute_domain_description', 'rdb_compute_domain_id'] + data = [[self.name, self.description, self.id]] + domain_metadata = pd.DataFrame(data=data, columns=column_names) + + if do_include_databases: + db_metadata = self.get_databases_metadata() + # https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.join.html + domain_metadata = pd.merge(domain_metadata, db_metadata, how="outer", + left_on=["rdb_compute_domain_id", "rdb_compute_domain_name"], + right_on=["rdb_compute_domain_id", "rdb_compute_domain_name"]) + domain_metadata.sort_values(by=['rdb_compute_domain_name', 'database_name'], inplace=True) + domain_metadata = domain_metadata.astype({"database_id": int}) + else: + domain_metadata.sort_values(by=['rdb_compute_domain_name'], inplace=True) + + domain_metadata = domain_metadata.astype({"rdb_compute_domain_id": int}) + return domain_metadata + + def get_databases_metadata(self) -> pd.DataFrame: + """ + Gets metadata of the databases in this RDBComputeDomain. + + :return: pandas dataframe with associated metadata. + + .. seealso:: SciQuery.get_database_names + """ + dfs = [db.get_metadata() for db in self.databases] + dfs = pd.concat(dfs, ignore_index=True) + dfs.sort_values(by="database_name", inplace=True) + return dfs + + def __str__(self): + return "RDBComputeDomain of name={} and id={}".format(self.name, self.id) + + def __repr__(self): + return "RDBComputeDomain(name={}, id={})".format(self.name, self.id) + + +class RDBComputeDomains(list): + + def __init__(self, rdb_compute_domains: Union[Iterable, RDBComputeDomain]): + """ + Defines a list of RDBComputeDomains, which are domains or environments with databases that users are able to + query. + + :param rdb_compute_domains: Parameter that identifies a list of RDBComputeDomain objects. + Could be either single RDBComputeDomain object, or an iterable containing multiple RDBComputeDomain objects. + """ + super().__init__() + domains = rdb_compute_domains if isinstance(rdb_compute_domains, Iterable) else [rdb_compute_domains] + for d in domains: + if isinstance(d, RDBComputeDomain): + self.append(d) + else: + raise NameError("Input object is not of class RDBComputeDomain.") + + def get_rdb_compute_domain(self, rdb_compute_domain: Union[str, int, dict, RDBComputeDomain]) -> RDBComputeDomain: + """ + Gets RDBComputeDomain from this list + """ + if type(rdb_compute_domain) == str: + domains = [d for d in self if d.name == rdb_compute_domain] + elif type(rdb_compute_domain) == int: + domains = [d for d in self if d.id == rdb_compute_domain] + elif type(rdb_compute_domain) == dict: + domains = [d for d in self if d.id == rdb_compute_domain.get('id')] + elif isinstance(rdb_compute_domain, RDBComputeDomain): + return self.get_rdb_compute_domain(rdb_compute_domain.id) + else: + raise TypeError("Invalid type for input parameter 'rdb_compute_domain'.") + + if len(domains) == 0: + raise NameError("RDBComputeDomain not found in list of available rdbComputeDomains") + else: + return domains[0] + + def get_default_rdb_compute_domain(self) -> RDBComputeDomain: + """ + Gets default RDBComputeDomain from this list. + """ + domains = [domain for domain in self if len(domain.databases) > 0] + if len(domains) > 0: + doms = [dom for dom in domains if dom.get_default_database().name == SciQuery.get_mydb_name()] + if len(doms) > 0: + return doms[0] + return domains[0] + else: + raise Exception("No RDBComputeDomain available with a database.") + + +class SciQuery: + + def __init__(self, + rdb_compute_domain: Union[str, int, dict, RDBComputeDomain] = None, + database: Union[str, int, dict, Database] = None, + file_service: Union[str, dict] = None, + results_base_path: str = None, + outputs: Outputs = None, + verbose: bool = True, + hard_fail: bool = False, + poll_time: float = 1.0 + ): + """ + Created an instance of the SciQuery app for querying relational databases. + + :param rdb_compute_domain: defines a domain or environment of multiple databases where users can run queries. + Can be either the domain's name (string), ID (integer), an object of class RDBComputeDomain, or a dictionary + containing all the attributes of an object of class RDBComputeDomain. If set to None, a default value will + be assigned to it. + :param database: defines the database where the queries are executed in. + Can be either the database name (string), ID (integer), an object of class Database, or a dictionary + containing all the attributes of an object of class Database. If set to None, a default value will be + assigned to it. + :param file_service: a File Service defines an available file system where query result sets can be written + into. This parameter can be it name or identifier (string), or a dictionary defining a file service. + If set to None, a default value will be assigned to it. + :param results_base_path: base path (string) of the directory where the query results are written into. + Can be constructed by using FileOutput.build_file_base_path(). If set to None, a default value will be + assigned to it at the moment of running a sql query. + :param outputs: Defines the query(ies) output(s). Can be an object derived from the Output base class (such as + FileOutput or DatabaseTableOutput), or a list of those. If set to None, a default value (json file output) + will be assigned to it. + :param verbose: Boolean parameter. If True, warning messages will be written in case of errors, in the case when + the hard_fail parameter is set to False. If False, nothing will be written. + :param hard_fail: Boolean parameter. If True, exceptions will be raised in case of errors during instantiation. + If False, then no exceptions are raised, and warnings might be showed instead + (depending on the value of the verbose parameter). + :param poll_time: time (float) in seconds between consecutive requests for updates in the jobs status. + """ + + self.user = SciQuery.get_user() + self.verbose = verbose + self.hard_fail = hard_fail + self.poll_time = poll_time + self._file_service = None + self._results_base_path = None + self._outputs = None + self._rdb_compute_domains = None + self._rdb_compute_domain = None + self._database = None + self.refresh_date = None + self.set(rdb_compute_domain, database, file_service, results_base_path, outputs, verbose, hard_fail, poll_time) + + @staticmethod + def get_token() -> str: + """ + Gets user's auth token. + """ + token = Authentication.getToken() + if token is None or token == "": + raise Exception("User not has not logged into SciServer. Use 'Authentication.login'.") + return token + + @staticmethod + def get_user() -> Authentication.KeystoneUser: + """ + Gets logged Keystone user info. + """ + token = SciQuery.get_token() + user = Authentication.getKeystoneUserWithToken(token) + user.token = token + return user + + def set(self, + rdb_compute_domain: Union[str, int, dict, RDBComputeDomain] = None, + database: Union[str, int, dict, Database] = None, + file_service: Union[str, dict] = None, + results_base_path: str = None, + outputs: Outputs = None, + verbose: bool = None, + hard_fail: bool = None, + poll_time: float = None + ): + """ + Sets or refreshes the parameters in the SciQuery object, all at once. + + :param rdb_compute_domain: defines a domain or environment of multiple databases where users can run queries. + Can be either the domain's name (string), ID (integer), an object of class RDBComputeDomain, or a dictionary + containing all the attributes of an object of class RDBComputeDomain. If set to None, the current value + is refreshed. + :param database: defines the database where the queries are executed in. + Can be either the database name (string), ID (integer), an object of class Database, or a dictionary + containing all the attributes of an object of class Database. If set to None, the current value is + refreshed. + :param file_service: a File Service defines an available file system where query result sets can be written + into. This parameter can be it name or identifier (string), or a dictionary defining a file service. + If set to None, the current value is refreshed. + :param results_base_path: base path (string) of the directory where the query results are written into. + Can be constructed by using FileOutput.build_file_base_path(). + :param outputs: Defines the query(ies) output(s). Can be a list of Output objects, + or a single object of class Outputs. If set to None, a default value (json file output) will be assigned to + it. + :param verbose: Boolean parameter. If True, warning messages will be written in case of errors, in the case when + the hard_fail parameter is set to False. If False, nothing will be written. + :param hard_fail: Boolean parameter. If True, exceptions will be raised in case of errors during instantiation. + If False, then no exceptions are raised, and warnings might be showed instead + (depending on the value of the verbose parameter). + :param poll_time: time (float) in seconds between consecutive requests for updates in the jobs status. + """ + + self.verbose = verbose if verbose else self.verbose + self.hard_fail = hard_fail if hard_fail else self.hard_fail + self.poll_time = poll_time if poll_time else self.poll_time + + # set or refresh current _rdb_compute_domains + try: + self.rdb_compute_domains = SciQuery.get_rdb_compute_domains('class') + except Exception as ex: + self._handle_exception(NameError(ex), "Unable to set or refresh rdb_compute_domains.") + # nothing else to do: + return + + try: + if self.rdb_compute_domain is None: + self.rdb_compute_domain = rdb_compute_domain if rdb_compute_domain else \ + self.get_default_rdb_compute_domain() + else: + self.rdb_compute_domain = rdb_compute_domain if rdb_compute_domain else self.rdb_compute_domain + except Exception as ex: + self._handle_exception(NameError(ex), "Unable to set or refresh rdb_compute_domain.") + + try: + if self.database is None: + self.database = database if database else self.get_default_database() + else: + self.database = database if database else self.database + except Exception as ex: + self._handle_exception(NameError(ex), "Unable to set or refresh database.") + + try: + if self.file_service is None: + self.file_service = file_service if file_service else self.get_default_file_service() + else: + self.file_service = file_service if file_service else self.file_service + except Exception as ex: + self._handle_exception(NameError(ex), "Unable to set or refresh file_service.") + + try: + if self.outputs is None: + self.outputs = outputs if outputs else self.get_default_outputs() + else: + self.outputs = outputs if outputs else self.outputs + except Exception as ex: + self._handle_exception(NameError(ex), "Unable to set or refresh outputs.") + + try: + self.results_base_path = results_base_path + except Exception as ex: + self._handle_exception(NameError(ex), "Unable to set results_base_path.") + + self.refresh_date = datetime.now() + + def _handle_exception(self, exception: Exception, extra_message: str = ""): + + message = extra_message + " Error: " + str(exception) if extra_message else str(exception) + if self.hard_fail: + exception.message = message + raise exception + elif self.verbose: + warnings.warn(message) + + def refresh(self): + """ + Refreshes SciQuery instance. + """ + self.set(verbose=self.verbose, hard_fail=self.hard_fail) + + @staticmethod + def get_mydb_name(owner_name: str = None) -> str: + """ + Returns name of mydb based on the owner's name. + """ + if not owner_name: + owner_name = SciQuery.get_user().userName + return "mydb:" + owner_name + + @staticmethod + def get_rdb_compute_domains(result_format: str = 'class') -> RDBComputeDomains: + """ + Gets a list of all registered Relational Database (RDB) compute domains that the user has access to. + + :param result_format: If set to "class", then the returned value will be of class RDBComputeDomains. + If set to "dict", then the return value will be a list of dictionaries, each of them containing the + attributes of an RDBComputeDomain object. + :return: an object of class RDBComputeDomains, or a list of dictionaries, each of them containing the + attributes of an RDBComputeDomain object. + """ + token = SciQuery.get_user().token + + if Config.isSciServerComputeEnvironment(): + task_name = "Compute.SciScript-Python.SciQuery.get_rdb_compute_domains" + else: + task_name = "SciScript-Python.SciQuery.get_rdb_compute_domains" + + url = Config.SciqueryURL + "/api/info/domain?TaskName=" + task_name + headers = {'X-Auth-Token': token, "Content-Type": "application/json"} + res = requests.get(url, headers=headers, stream=True) + if res.status_code != 200: + raise Exception( + "Error when getting RDB Compute Domains from the SciQuery API.\nHttp Response from the SciQuery API " + "returned status code " + str(res.status_code) + ":\n" + res.content.decode()) + else: + arr = json.loads(res.content.decode()) + if result_format == 'class': + return RDBComputeDomains([RDBComputeDomain(d) for d in arr]) + else: + return arr + + # rdb_compute_domains --------------------------------------------------- + + @property + def rdb_compute_domains(self) -> RDBComputeDomains: + """ + Property defining the RDBComputeDomains available in this SciQuery instance. + """ + return self._rdb_compute_domains + + @rdb_compute_domains.setter + def rdb_compute_domains(self, rdb_compute_domains: RDBComputeDomains): + if not isinstance(rdb_compute_domains, RDBComputeDomains): + raise Exception("'rdb_compute_domains' should be of class RDBComputeDomains.") + self._rdb_compute_domains = rdb_compute_domains + + # rdb_compute_domain --------------------------------------------------- + + @property + def rdb_compute_domain(self) -> RDBComputeDomain: + """ + Property defining the selected RDBComputeDomain in this SciQuery instance. + """ + return self._rdb_compute_domain + + @rdb_compute_domain.setter + def rdb_compute_domain(self, rdb_compute_domain: Union[str, int, dict, RDBComputeDomain]): + if rdb_compute_domain is None: + raise Exception("'rdb_compute_domain' cannot be set to None.") + self._rdb_compute_domain = self.get_rdb_compute_domain(rdb_compute_domain) + + def get_rdb_compute_domain(self, rdb_compute_domain: Union[str, int, dict, RDBComputeDomain] = None) \ + -> RDBComputeDomain: + """ + Returns an object of class RDBComputeDomain, either defined by the input name or identifiers, or that + which is set in the SciQuery instance. + + :param rdb_compute_domain: defines a domain or environment of multiple databases where users can run queries. + Can be either the domain's name (string), ID (integer), an object of class RDBComputeDomain, or a dictionary + containing all the attributes of an object of class RDBComputeDomain. If set to None, then the currently set + value of rdb_compute_domain in the SciQuery object is returned. + :return: Object of class RDBComputeDomain. + """ + if rdb_compute_domain is None: + return self._rdb_compute_domain + return self.rdb_compute_domains.get_rdb_compute_domain(rdb_compute_domain) + + def get_default_rdb_compute_domain(self): + """ + Gets default RDBComputeDomain. + """ + return self.rdb_compute_domains.get_default_rdb_compute_domain() + + # database --------------------------------------------------- + + @property + def database(self) -> Database: + """ + Property defining the selected database in this SciQuery instance. + """ + return self._database + + @database.setter + def database(self, database: Union[str, int, dict, Database]): + if database is None: + raise Exception("'database' cannot be set to None.") + self._database = self.get_database(database, self.rdb_compute_domain) + + def get_database(self, + database: Union[str, int, dict, Database] = None, + rdb_compute_domain: Union[str, int, dict, RDBComputeDomain] = None) -> Database: + """ + Returns an object of class Database, either defined by the input name or identifiers, or that + which is set in the SciQuery instance. + + :param database: identifies the database, which this function returns as an object of class Database. + Can be either the database name (string), ID (integer), an object of class Database, or a dictionary + containing all the attributes of an object of class Database. If set to None, then the currently set value + of database in the SciQuery object is returned. + :param rdb_compute_domain: defines a domain or environment of multiple databases where users can run queries, + and that contains the database. Can be either the domain's name (string), ID (integer), an object of class + RDBComputeDomain, or a dictionary containing all the attributes of an object of class RDBComputeDomain. + If set to None, then the currently set value of rdb_compute_domain in the SciQuery object is internally + used. + :return: Object of class Database + """ + if database is None: + return self._database + return self.get_rdb_compute_domain(rdb_compute_domain).get_database(database) + + def get_default_database(self, rdb_compute_domain: Union[str, int, dict, RDBComputeDomain] = None) -> Database: + """ + Gets default database. + """ + domain = self.get_default_rdb_compute_domain() if rdb_compute_domain is None \ + else self.get_rdb_compute_domain(rdb_compute_domain) + return domain.get_default_database() + + # file_service --------------------------------------------------- + + @property + def file_service(self) -> dict: + """ + Property defining the selected FileService available in this SciQuery instance. + """ + return self._file_service + + @file_service.setter + def file_service(self, file_service: Union[str, dict]): + if file_service is None: + raise Exception("'file_service' cannot be set to None.") + self._file_service = self.get_file_service(file_service) + + def get_file_service(self, file_service: Union[str, dict] = None) -> dict: + """ + Returns the definition of a file service as a dictionary, either defined by the input name or identifiers, + or that which is set in the SciQuery instance. + + :param file_service: name or identifier (string) of a file service, or the dictionary with its definition. + If set to None, then the currently set value of file_service in the SciQuery object is returned. + :return: dictionary with the definition of a file service. + """ + if file_service is None: + return self._file_service + return FileOutput.find_file_service(file_service) + + def get_default_file_service(self) -> dict: + """ + Gets default FiLeService + """ + return FileOutput.find_file_service() + + # results_base_path --------------------------------------------------- + + @property + def results_base_path(self) -> str: + """ + Property defining the file output base path in SciServer's filesystem for this SciQuery instance. + """ + return self._results_base_path + + @results_base_path.setter + def results_base_path(self, results_base_path: str): + # if results_base_path is None or not results_base_path.startswith(Config.ComputeWorkDir): + # raise Exception(f"The string 'results_base_path' must start with {Config.ComputeWorkDir}") + self._results_base_path = results_base_path + + def get_results_base_path(self) -> str: + """ + Gets results base path on SciServer's filesystem. + """ + return self._results_base_path + + def get_default_results_base_path(self, add_date_ending=True) -> str: + """ + Gets default results base path on SciServer's filesystem. + """ + return FileOutput.build_file_base_path(add_date_ending=add_date_ending) + + # outputs ------------------------------------------------------------- + + @property + def outputs(self) -> Outputs: + """ + Property defining a list of query result Output objects. + """ + return self._outputs + + @outputs.setter + def outputs(self, outputs: Union[Outputs, Output]): + if outputs is None: + raise Exception("'outputs' cannot be set to None.") + self._outputs = self.get_outputs(outputs) + + def get_outputs(self, outputs: Union[Outputs, Output] = None) -> Outputs: + """ + Returns an object of class Outputs, either defined by the inputs parameters, or that + which is set in the SciQuery instance. + + :param outputs: object of class Outputs, or iterable of output objects. If set to None, then the currently + set value of outputs in the SciQuery object is returned. + :return: object of class Outputs. + """ + if outputs is None: + return self.outputs + return Outputs(outputs) + + def get_default_outputs(self) -> Outputs: + """ + Gets default query outputs. + """ + return Outputs.get_default() + + # --------------------------------------------------------------------------------------------- + # Running Queries ----------------------------------------------------------------------------- + # --------------------------------------------------------------------------------------------- + + def submit_query_job(self, + sql_query: str, + database: Union[str, int, dict, Database] = None, + outputs: Union[Outputs, Output] = None, + results_base_path: str = None, + rdb_compute_domain: Union[str, int, dict, RDBComputeDomain] = None, + file_service: str = None, + job_alias: str = "") -> int: + """ + Submits a sql query for execution (as an asynchronous job) inside a relational database (RDB) compute domain. + + :param sql_query: sql query (string) + :param database: defines the database where the sql query is executed in. + Can be either the database name (string), ID (integer), an object of class Database, or a dictionary + containing all the attributes of an object of class Database. If set to None, then the current value of + the database field in this SciQuery instance will be used. + :param outputs: Defines the query(ies) output(s). Can be an object derived from the Output base class (such as + FileOutput or DatabaseTableOutput), or a list of those. If set to None, then the current value of the + outputs field in this SciQuery instance will be used. + :param results_base_path: full path to results folder (string) where query output tables are written into. + E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder . If set to None, then its current + value in this SciQuery instance will be used. If that value is None, then a default folder will be set + automatically. + :param rdb_compute_domain: defines a domain or environment of multiple databases where users can run queries, + and that contains the database. Can be either the domain's name (string), ID (integer), an object of class + RDBComputeDomain, or a dictionary containing all the attributes of an object of class RDBComputeDomain. + If set to None, then the currently set value of rdb_compute_domain in the SciQuery object is internally + used. + :param file_service: a File Service defines an available file system where query result sets can be written + into. This parameter can be its name or identifier (string), or a dictionary defining a file service. + If set to None, then the currently set value of file_service in the SciQuery object is internally used. + :param job_alias: alias (string) of job, defined by the user. + :return: the ID (string) that labels the job. + """ + + domain = self.get_rdb_compute_domain(rdb_compute_domain) + db = self.get_database(database, domain) + fs = self.get_file_service(file_service) + outputs = self.get_outputs(outputs) + results_base_path = results_base_path if results_base_path else self.get_results_base_path() + if not results_base_path: + results_base_path = self.get_default_results_base_path() + + targets = outputs.get_target_list(results_base_path, fs.get('identifier')) + + job_model = { + "inputSql": sql_query, + "submitterDID": job_alias, + "databaseContextName": db.name, + "rdbDomainId": domain.id, + "targets": targets, + "resultsFolderURI": fs['identifier'] + ":" + results_base_path + } + + if Config.isSciServerComputeEnvironment(): + task_name = "Compute.SciScript-Python.SciQuery.submit_query_job" + else: + task_name = "SciScript-Python.SciQuery.submit_query_job" + + data = json.dumps(job_model).encode() + url = Config.SciqueryURL + "/api/jobs/" + str(domain._racm_id) + "?TaskName=" + task_name; + headers = {'X-Auth-Token': self.user.token, "Content-Type": "application/json"} + res = requests.post(url, data=data, headers=headers, stream=True) + if res.status_code < 200 or res.status_code >= 300: + raise Exception("Error when submitting a job to the SciQuery API.\nHttp Response from SciQuery API " + + "returned status code " + str(res.status_code) + ":\n" + res.content.decode()); + else: + return res.content.decode() + + def execute_query(self, + sql_query, + database: Union[str, int, dict, Database] = None, + results_base_path: str = None, + rdb_compute_domain: Union[str, int, dict, RDBComputeDomain] = None, + job_alias: str = "", + file_service: str = None, + write_job_id = True) -> pd.DataFrame: + """ + Returns the query result (as a Pandas data frame) of a sql query submitted as a job to a + relational database (RDB) compute domain. + + :param sql_query: sql query (string) + :param database: defines the database where the sql query is executed in. + Can be either the database name (string), ID (integer), an object of class Database, or a dictionary + containing all the attributes of an object of class Database. If set to None, then the current value of + the database field in this SciQuery instance will be used. + :param results_base_path: full path to results folder (string) where query output tables are written into. + E.g.: /home/idies/workspace/rootVOlume/username/userVolume/jobsFolder . If set to None, then its current + value in this SciQuery instance will be used. If that value is None, then a default folder will be set + automatically. + :param rdb_compute_domain: defines a domain or environment of multiple databases where users can run queries, + and that contains the database. Can be either the domain's name (string), ID (integer), an object of class + RDBComputeDomain, or a dictionary containing all the attributes of an object of class RDBComputeDomain. + If set to None, then the currently set value of rdb_compute_domain in the SciQuery object is internally used. + :param job_alias: alias (string) of job, defined by the user. + :param file_service: a File Service defines an available file system where query result sets can be written + into. This parameter can be its name or identifier (string), or a dictionary defining a file service. + If set to None, then the currently set value of file_service in the SciQuery object is internally used. + :param write_job_id: if True, the job id will be written on the screen, just before returning the result. + The job id won;t be written if write_job_id = False. + into. This parameter can be its name or identifier (string), or a dictionary defining a file service. + If set to None, then the currently set value of file_service in the SciQuery object is internally used. + + :return: Pandas data frame containing the result of the query. + """ + output = FileOutput("result1.json", OutputType.FILE_JSON, 1) + job_alias = job_alias if job_alias else "synchronous query" + job_id = self.submit_query_job(sql_query=sql_query, rdb_compute_domain=rdb_compute_domain, database=database, + outputs=output, + results_base_path=results_base_path, + job_alias=job_alias, + file_service=file_service) + if write_job_id: + print("Query was submitted as a job with id = " + job_id) + job = self.wait_for_job(job_id, verbose=False) + if job.status > 32: + messages = ". ".join(job.message_list) + if job.status == 64: + raise Exception("Query ended with an error. " + messages) + if job.status == 128: + raise Exception("Query was cancelled. " + messages) + + df = job.get_dataframe_from_output(0) + return df + + @staticmethod + def get_jobs_list(top=5, open=None, start=None, end=None, result_format="pandas") \ + -> Union[pd.DataFrame, list]: + """ + Gets the list of SciQuery Jobs submitted by the user. + + :param top: top number of jobs (integer) returned. If top=None, then all jobs are returned. + :param open: If set to 'True', then only returns jobs that have not finished executing and wrapped up + (status <= FINISHED). If set to 'False' then only returns jobs that are still running. If set to 'None', + then returns both finished and unfinished jobs. + :param start: The earliest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. + If set to 'None', then there is no lower bound on date. + :param end: The latest date (inclusive) to search for jobs, in string format yyyy-MM-dd hh:mm:ss.SSS. + If set to 'None', then there is no upper bound on date. + :param result_format: string defining the return format. "pandas" for a pandas dataframe and "list" + for a list of RDBJob objects. + :return: pandas dataframe, or list of RDBJob objects or, each containing the definition of a submitted job. + """ + job_dict_list = Jobs.getJobsList(top=top, open=open, start=start, end=end, type='rdb') + jobs = [] + for job_dict in job_dict_list: + j = RDBJob(job_dict) + jobs.append(j if format == "list" else j.get_metadata()) + if result_format == "pandas": + jobs = pd.concat(jobs, ignore_index=True) + return jobs + + @staticmethod + def get_job(job_id): + """ + Gets the definition of the job as a RDBJob object. + + :param job_id: Id of job + :return: RDBJob object containing the description or definition of the job. + """ + return RDBJob.get_job(job_id) + + @staticmethod + def get_job_status(job_id): + """ + Gets a dictionary with the job status as an integer value, together with its semantic meaning. The integer value + is a power of 2, that is, + 1:PENDING, 2:QUEUED, 4:ACCEPTED, 8:STARTED, 16:FINISHED, 32:SUCCESS, 64:ERROR, 128:CANCELED + + :param job_id: Id of job (string). + :return: dictionary with the integer value of the job status, as well as its semantic meaning. + """ + return Jobs.getJobStatus(job_id) + + @staticmethod + def cancel_job(job): + """ + Cancels the execution of a job. + + :param job: id of the job (string), or object of class RDBJob. + """ + if isinstance(job, str): + Jobs.cancelJob(job) + elif isinstance(job, RDBJob): + job.cancel() + else: + raise NameError("Invalid type for input parameter 'job'.") + + def wait_for_job(self, job_id, verbose=False): + """ + Queries the job status regularly and waits until the job is completed. + + :param job_id: id of job (integer) + :param verbose: if True, will print "wait" messages on the screen while the job is still running. If False, it + will suppress the printing of messages on the screen. + :return: After the job is finished, returns an object of class RDBJob, containing the job definition. + :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that + purpose). + Throws an exception if the HTTP request to the JOBM API returns an error. + """ + t = max(0.5, self.poll_time) + wait_message = "Waiting" + while True: + job_desc = Jobs.getJobDescription(job_id) + if job_desc.get("status") >= 32: + return RDBJob(job_desc) + else: + if verbose: + wait_message += "." + print(wait_message, end="\r") + time.sleep(t) + + # METADATA ------------------------------------------------------------------------------------------------- + + def get_rdb_compute_domains_metadata(self, do_include_databases=False): + """ + Gets metadata related to all relational database (RDB) compute domains (RDBComputeDomains) available. + + :param do_include_databases: Boolean parameter. If True, it will return metadata related to all available + databases in each RDBComputeDomain as well. + :return: pandas dataframe containing associated metadata. + """ + dfs = [] + for domain in self.rdb_compute_domains: + dfs.append(domain.get_metadata(do_include_databases)) + + df = pd.concat(dfs, ignore_index=True) + df.sort_values(by="rdb_compute_domain_name", inplace=True) + return df + + def get_rdb_compute_domain_names(self): + """ + Returns the names of the RDB compute domains available to the user. + + :return: an array of strings, each being the name of a rdb compute domain available to the user. + """ + return [d.name for d in self.rdb_compute_domains] + + def get_rdb_compute_domain_metadata(self, rdb_compute_domain=None, do_include_databases=False): + """ + Gets metadata of an rdb_compute_domain. + """ + return self.get_rdb_compute_domain(rdb_compute_domain).get_metadata(do_include_databases) + + def get_databases_metadata(self, rdb_compute_domain=None): + """ + Gets metadata (name and description) of databases in an RDBComputeDomain. + """ + rdb_compute_domain = self.get_rdb_compute_domain(rdb_compute_domain) + if isinstance(rdb_compute_domain, Iterable): + dfs = [d.get_databases_metadata() for d in rdb_compute_domain] + return pd.concat(dfs, ignore_index=True) + else: + return rdb_compute_domain.get_databases_metadata() + + def get_database_metadata(self, database=None, rdb_compute_domain=None): + """ + Gets database metadata. + """ + rdb_compute_domain = self.get_rdb_compute_domain(rdb_compute_domain) + database = self.get_database(database, rdb_compute_domain) + return database.get_metadata() + + def get_database_names(self, rdb_compute_domain=None): + """ + Gets a list of the names of the databases in this RDBComputeDomain. + + :return: array of database names (strings) + """ + rdb_compute_domain = self.get_rdb_compute_domain(rdb_compute_domain) + return rdb_compute_domain.get_database_names() + + def _get_metadata(self, rdb_compute_domain, database, resource_name="", metadata_type=""): + """ + Utility function for the use of other metadata functions. + """ + + rdb_compute_domain = self.get_rdb_compute_domain(rdb_compute_domain) + database = self.get_database(database, rdb_compute_domain.id) + + if metadata_type not in [t for t in dir(_MetadataType) if not t.startswith("__")]: + raise TypeError("Invalid type for input parameter 'metadata_type'.") + + if Config.isSciServerComputeEnvironment(): + task_name = "Compute.SciScript-Python.Sciquery.get_metadata_" + metadata_type + else: + task_name = "SciScript-Python.Sciquery.get_metadata_" + metadata_type + + url = Config.SciqueryURL + "/api/metadata/{0}/{1}/".format(rdb_compute_domain._racm_id, database.name) + if metadata_type == _MetadataType.TABLES: + url += "tables" + elif metadata_type == _MetadataType.VIEWS: + url += "views" + elif metadata_type == _MetadataType.ROUTINES: + url += "routines" + elif metadata_type == _MetadataType.COLUMNS: + url += "{0}/{1}".format(resource_name, "columns") + elif metadata_type == _MetadataType.PARAMETERS: + url += "{0}/{1}".format(resource_name, "parameters") + elif metadata_type == _MetadataType.CONSTRAINTS: + url += "{0}/{1}".format(resource_name, "constraints") + else: + raise ValueError("Wrong metadata_type parameter value of " + metadata_type) + + url += "?taskName=" + task_name + headers = {'X-Auth-Token': self.user.token} + res = requests.get(url, headers=headers, stream=True) + + if res.status_code < 200 or res.status_code >= 300: + raise Exception("Error when getting metadata from SciQuery API.\nHttp Response from SciQuery API " + + "returned status code " + str(res.status_code) + ":\n" + res.content.decode()) + else: + res = json.loads(res.content.decode()) + result = res['Result'][0] + df = pd.DataFrame(result['Data'], columns=[c.upper() for c in result['ColumnNames']]) + df.name = result['TableName'] + return df + + def get_tables_metadata(self, database=None, rdb_compute_domain=None): + """ + Gets metadata related to tables in a particular database belonging to an RDBComputeDomain. + """ + return self._get_metadata(rdb_compute_domain, database, metadata_type=_MetadataType.TABLES) + + def get_table_names(self, database=None, rdb_compute_domain=None): + """ + Gets a list of the names of tables in a particular database belonging to an RDBComputeDomain. + """ + tables = self.get_tables_metadata(database, rdb_compute_domain) + return [name for name in tables['TABLE_NAME']] + + def get_views_metadata(self, database=None, rdb_compute_domain=None): + """ + Gets metadata related to views in a particular database belonging to an RDBComputeDomain. + """ + return self._get_metadata(rdb_compute_domain, database, metadata_type=_MetadataType.VIEWS) + + def get_view_names(self, database=None, rdb_compute_domain=None): + """ + Gets a list of the names of views in a particular database belonging to an RDBComputeDomain. + """ + tables = self.get_views_metadata(database, rdb_compute_domain) + return [name for name in tables['TABLE_NAME']] + + def get_routines_metadata(self, database=None, rdb_compute_domain=None): + """ + Gets metadata related to routines or functions in a particular database belonging to an RDBComputeDomain. + """ + return self._get_metadata(rdb_compute_domain, database, metadata_type=_MetadataType.ROUTINES) + + def get_routine_names(self, database=None, rdb_compute_domain=None): + """ + Gets a list of the names of routines or functions in a particular database belonging to an RDBComputeDomain. + """ + routines = self.get_routines_metadata(database, rdb_compute_domain) + return [routine_name for routine_name in routines['ROUTINE_NAME']] + + def get_columns_metadata(self, table_name, database=None, rdb_compute_domain=None): + """ + Gets metadata related to columns in a particular database table belonging to an RDBComputeDomain. + """ + return self._get_metadata(rdb_compute_domain, database, table_name, metadata_type=_MetadataType.COLUMNS) + + def get_column_names(self, table_name, database=None, rdb_compute_domain=None): + """ + Gets a list of the names of table columns in a particular database belonging to an RDBComputeDomain. + """ + columns = self.get_columns_metadata(table_name, database, rdb_compute_domain) + return [columnName for columnName in columns['COLUMN_NAME']] + + def get_constraints_metadata(self, table_name, database=None, rdb_compute_domain=None): + """ + Gets metadata related to table constraints in a particular database table belonging to an RDBComputeDomain. + """ + return self._get_metadata(rdb_compute_domain, database, table_name, metadata_type=_MetadataType.CONSTRAINTS) + + def get_constraint_names(self, table_name, database=None, rdb_compute_domain=None): + """ + Gets a list of the names of table constraints in a particular database belonging to an RDBComputeDomain. + """ + constraints = self.get_constraints_metadata(table_name, database, rdb_compute_domain) + return [constraintName for constraintName in constraints['CONSTRAINT_NAME']] + + def get_routine_parameters_metadata(self, routine_name, database=None, rdb_compute_domain=None): + """ + Gets metadata related to routine parameters in a particular database belonging to an RDBComputeDomain. + """ + return self._get_metadata(rdb_compute_domain, database, routine_name, metadata_type=_MetadataType.PARAMETERS) + + def get_routine_parameter_names(self, routine_name, database=None, rdb_compute_domain=None): + """ + Gets a list of the names of routine parameters in a particular database belonging to an RDBComputeDomain. + """ + parameters = self.get_routine_parameters_metadata(routine_name, database, rdb_compute_domain) + return [name for name in parameters['PARAMETER_NAME']] + + def __str__(self): + return "SciQuery instance with rdb_compute_domains = {})".format(self._rdb_compute_domains) + + def __repr__(self): + return "SciQuery(rdb_compute_domains = {})".format(self._rdb_compute_domains) + + +class _MetadataType: + """ + Contains a set of metadata types. + """ + TABLES = "TABLES" + VIEWS = "VIEWS" + COLUMNS = "COLUMNS" + ROUTINES = "ROUTINES" + CONSTRAINTS = "CONSTRAINTS" + PARAMETERS = "PARAMETERS" diff --git a/py3/SciServer/SkyQuery.py b/py3/SciServer/SkyQuery.py index 35ecee1..9645262 100644 --- a/py3/SciServer/SkyQuery.py +++ b/py3/SciServer/SkyQuery.py @@ -27,7 +27,14 @@ def getJobStatus(jobId): """ token = Authentication.getToken() if token is not None and token != "": - statusURL = Config.SkyQueryUrl + '/Jobs.svc/jobs/' + str(jobId) + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.getJobStatus" + else: + taskName = "SciScript-Python.SkyQuery.getJobStatus" + + statusURL = Config.SkyQueryUrl + '/Jobs.svc/jobs/' + str(jobId) + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -57,7 +64,13 @@ def cancelJob(jobId): token = Authentication.getToken() if token is not None and token != "": - statusURL = Config.SkyQueryUrl + '/Jobs.svc/jobs/' + jobId + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.cancelJob" + else: + taskName = "SciScript-Python.SkyQuery.cancelJob" + + statusURL = Config.SkyQueryUrl + '/Jobs.svc/jobs/' + jobId + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -93,7 +106,14 @@ def listQueues(): """ token = Authentication.getToken() if token is not None and token != "": - jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues' + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.listQueues" + else: + taskName = "SciScript-Python.SkyQuery.listQueues" + + jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -113,7 +133,7 @@ def getQueueInfo(queue): """ Returns information about a particular job queue (more info in http://www.voservices.net/skyquery). - :param queue: queue name (string) + :param queue: queue name (string) :return: a dictionary containing information associated to the queue. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error. :example: queueInfo = SkyQuery.getQueueInfo('quick') @@ -123,7 +143,13 @@ def getQueueInfo(queue): token = Authentication.getToken() if token is not None and token != "": - jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.getQueueInfo" + else: + taskName = "SciScript-Python.SkyQuery.getQueueInfo" + + jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -154,7 +180,13 @@ def submitJob(query, queue='quick'): token = Authentication.getToken() if token is not None and token != "": - jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + '/jobs' + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.submitJob" + else: + taskName = "SciScript-Python.SkyQuery.submitJob" + + jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + '/jobs' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -174,12 +206,13 @@ def submitJob(query, queue='quick'): raise Exception("User token is not defined. First log into SciServer.") -def waitForJob(jobId, verbose=True): +def waitForJob(jobId, verbose=False, pollTime = 5): """ - Queries the job status from SkyQuery every 2 seconds and waits for the SkyQuery job to be completed. + Queries regularly the job status and waits until the job is completed. :param jobId: id of job (integer) - :param verbose: if True, will print "wait" messages on the screen while the job is not done. If False, will suppress printing messages on the screen. + :param verbose: if True, will print "wait" messages on the screen while the job is still running. If False, will suppress the printing of messages on the screen. + :param pollTime: idle time interval (integer, in seconds) before querying again for the job status. Minimum value allowed is 5 seconds. :return: After the job is finished, returns a dictionary object containing the job status and related metadata. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error. :example: SkyQuery.waitForJob(SkyQuery.submitJob("select 1")) @@ -187,8 +220,8 @@ def waitForJob(jobId, verbose=True): .. seealso:: SkyQuery.submitJob, SkyQuery.getJobStatus. """ try: + minPollTime = 5 # in seconds complete = False - waitingStr = "Waiting..." back = "\b" * len(waitingStr) if verbose: @@ -199,13 +232,13 @@ def waitForJob(jobId, verbose=True): #print(back, end="") print(waitingStr, end="") jobDesc = getJobStatus(jobId) - if jobDesc['status'] == 'completed': + if jobDesc.get('dateFinished') is not None: complete = True if verbose: #print(back, end="") print("Done!") else: - time.sleep(2) + time.sleep(max(minPollTime,pollTime)); return jobDesc except Exception as e: @@ -226,7 +259,14 @@ def listJobs(queue="quick"): """ token = Authentication.getToken() if token is not None and token != "": - jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + '/jobs?' + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.listJobs" + else: + taskName = "SciScript-Python.SkyQuery.listJobs" + + jobsURL = Config.SkyQueryUrl + '/Jobs.svc/queues/' + queue + '/jobs' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -261,7 +301,13 @@ def listAllDatasets(): token = Authentication.getToken() if token is not None and token != "": - schemaURL = Config.SkyQueryUrl + '/Schema.svc/datasets' + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.listAllDatasets" + else: + taskName = "SciScript-Python.SkyQuery.listAllDatasets" + + schemaURL = Config.SkyQueryUrl + '/Schema.svc/datasets' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -290,7 +336,14 @@ def getDatasetInfo(datasetName="MyDB"): """ token = Authentication.getToken() if token is not None and token != "": - schemaURL = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.getDatasetInfo" + else: + taskName = "SciScript-Python.SkyQuery.getDatasetInfo" + + schemaURL = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -318,7 +371,14 @@ def listDatasetTables(datasetName="MyDB"): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables' + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.listDatasetTables" + else: + taskName = "SciScript-Python.SkyQuery.listDatasetTables" + + url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -348,7 +408,14 @@ def getTableInfo(tableName, datasetName="MyDB"): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables/' + tableName + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.getTableInfo" + else: + taskName = "SciScript-Python.SkyQuery.getTableInfo" + + url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables/' + tableName + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -377,7 +444,14 @@ def listTableColumns(tableName, datasetName="MyDB"): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables/' + tableName + '/columns' + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.listTableColumns" + else: + taskName = "SciScript-Python.SkyQuery.listTableColumns" + + url = Config.SkyQueryUrl + '/Schema.svc/datasets/' + datasetName +'/tables/' + tableName + '/columns' + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -412,7 +486,13 @@ def getTable(tableName, datasetName="MyDB", top = None): token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.getTable" + else: + taskName = "SciScript-Python.SkyQuery.getTable" + + url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + "?TaskName=" + taskName if top != None and top != "": url = url + '?top=' + str(top) @@ -443,7 +523,14 @@ def dropTable(tableName, datasetName="MyDB"): """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.dropTable" + else: + taskName = "SciScript-Python.SkyQuery.dropTable" + + url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + "?TaskName=" + taskName headers = {'Content-Type': 'application/json','Accept': 'application/json'} headers['X-Auth-Token']= token @@ -468,13 +555,20 @@ def uploadTable(uploadData, tableName, datasetName="MyDB", format="csv"): :param format: format of the 'data' parameter. Set to 'csv' for now. :return: returns True if the table was uploaded successfully. :raises: Throws an exception if the user is not logged into SciServer (use Authentication.login for that purpose). Throws an exception if the HTTP request to the SkyQuery API returns an error. - :example: result = SkyQuery.uploadTable("Column1,Column2\n4.5,5.5\n", tableName="myTable", datasetName="MyDB", format="csv") + :example: result = SkyQuery.uploadTable("Column1,Column2\\n4.5,5.5\\n", tableName="myTable", datasetName="MyDB", format="csv") .. seealso:: SkyQuery.listQueues, SkyQuery.listAllDatasets, SkyQuery.getDatasetInfo, SkyQuery.listDatasetTables, SkyQuery.getTableInfo, SkyQuery.getTable, SkyQuery.submitJob """ token = Authentication.getToken() if token is not None and token != "": - url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + + taskName = ""; + if Config.isSciServerComputeEnvironment(): + taskName = "Compute.SciScript-Python.SkyQuery.uploadTable" + else: + taskName = "SciScript-Python.SkyQuery.uploadTable" + + url = Config.SkyQueryUrl + '/Data.svc/' + datasetName +'/' + tableName + "?TaskName=" + taskName ctype = "" if format == "csv": ctype = 'text/csv' diff --git a/py3/SciServer/SkyServer.py b/py3/SciServer/SkyServer.py index a664723..ad05c27 100644 --- a/py3/SciServer/SkyServer.py +++ b/py3/SciServer/SkyServer.py @@ -36,9 +36,9 @@ def sqlSearch(sql, dataRelease=None): url = url + 'cmd=' + sql + '&' if Config.isSciServerComputeEnvironment(): - url = url + "TaskName=Compute.SciScript-Python.SkyServer.sqlSearch&" + url = url + "TaskName=Compute.SciScript-Python.SkyServer.sqlSearch" else: - url = url + "TaskName=SciScript-Python.SkyServer.sqlSearch&" + url = url + "TaskName=SciScript-Python.SkyServer.sqlSearch" #url = urllib.quote_plus(url) acceptHeader = "text/plain" @@ -60,12 +60,11 @@ def getJpegImgCutout(ra, dec, scale=0.7, width=512, height=512, opt="", query="" """ Gets a rectangular image cutout from a region of the sky in SDSS, centered at (ra,dec). Return type is numpy.ndarray.\n - :param ra: Right Ascension of the image's center. - :param dec: Declination of the image's center. - :param scale: scale of the image, measured in [arcsec/pix] - :param width: Right Ascension of the image's center. - :param ra: Right Ascension of the image's center. - :param height: Height of the image, measured in [pix]. + :param ra: Right Ascension of the image's center in floating point ICRS J2000 coordinates. + :param dec: Declination of the image's center in floating point ICRS J2000 coordinates. + :param scale: Scale of the image, measured in [arcsec/pix] + :param width: Width of image, measured in pixels (width in arcsec = width * scale). If larger than 2048, silently truncated to 2048. + :param height: Height of image, measured in pixels (width in arcsec = width * scale). If larger than 2048, silently truncated to 2048. :param opt: Optional drawing options, expressed as concatenation of letters (string). The letters options are \n \t"G": Grid. Draw a N-S E-W grid through the center\n \t"L": Label. Draw the name, scale, ra, and dec on image.\n @@ -114,9 +113,9 @@ def getJpegImgCutout(ra, dec, scale=0.7, width=512, height=512, opt="", query="" url = url + 'query=' + query + '&' if Config.isSciServerComputeEnvironment(): - url = url + "TaskName=Compute.SciScript-Python.SkyServer.getJpegImgCutout&" + url = url + "TaskName=Compute.SciScript-Python.SkyServer.getJpegImgCutout" else: - url = url + "TaskName=SciScript-Python.SkyServer.getJpegImgCutout&" + url = url + "TaskName=SciScript-Python.SkyServer.getJpegImgCutout" #url = urllib.quote_plus(url) acceptHeader = "text/plain" @@ -172,9 +171,9 @@ def radialSearch(ra, dec, radius=1, coordType="equatorial", whichPhotometry="opt url = url + 'limit=' + limit + '&' if Config.isSciServerComputeEnvironment(): - url = url + "TaskName=Compute.SciScript-Python.SkyServer.radialSearch&" + url = url + "TaskName=Compute.SciScript-Python.SkyServer.radialSearch" else: - url = url + "TaskName=SciScript-Python.SkyServer.radialSearch&" + url = url + "TaskName=SciScript-Python.SkyServer.radialSearch" #url = urllib.quote_plus(url) acceptHeader = "text/plain" @@ -231,9 +230,9 @@ def rectangularSearch(min_ra, max_ra, min_dec, max_dec, coordType="equatorial", url = url + 'limit=' + limit + '&' if Config.isSciServerComputeEnvironment(): - url = url + "TaskName=Compute.SciScript-Python.SkyServer.rectangularSearch&" + url = url + "TaskName=Compute.SciScript-Python.SkyServer.rectangularSearch" else: - url = url + "TaskName=SciScript-Python.SkyServer.rectangularSearch&" + url = url + "TaskName=SciScript-Python.SkyServer.rectangularSearch" #url = urllib.quote_plus(url) acceptHeader = "text/plain" @@ -319,9 +318,9 @@ def objectSearch(objId=None, specObjId=None, apogee_id=None, apstar_id=None, ra= url = url + 'obj=' + str(obj) + '&'; if Config.isSciServerComputeEnvironment(): - url = url + "TaskName=Compute.SciScript-Python.SkyServer.objectSearch&" + url = url + "TaskName=Compute.SciScript-Python.SkyServer.objectSearch" else: - url = url + "TaskName=SciScript-Python.SkyServer.objectSearch&" + url = url + "TaskName=SciScript-Python.SkyServer.objectSearch" #url = urllib.quote_plus(url) acceptHeader = "text/plain" diff --git a/py3/SciServer/__init__.py b/py3/SciServer/__init__.py index a7f4476..87f1fcb 100644 --- a/py3/SciServer/__init__.py +++ b/py3/SciServer/__init__.py @@ -5,21 +5,34 @@ Some SciServer tools you can access with this package:\n -* `Login Portal `_: Single sign-on portal to all SciServer applications. +* `Login Portal `_: Single sign-on portal to all SciServer applications. \t\tAlthough some tools accept anonymous access, you can use Authentication.login to login and access the tools and your own data and environment (after registering in the Login Portal). If you are running this package in a Jupyter Notebook in the SciServer-Compute environment, the use of Authentication.login is not necessary since it's done automatically. -* `CasJobs `_: Database storage and querying. -\t\tYou can have access big databases, as well as save your data tables in your own database called 'MyDB'. The user can run synchronous or asynchronous SQL queries and get the result back as an R data-frame (using CasJobs.executeQuery or CasJobs.submitJob, respectively). Uploading of CSV files or R data-frames into a database table can be done using CasJobs.uploadCSVToTable and CasJobs.uploadDataFrameToTable, respectively. +* `CasJobs `_: Database storage and querying. +\t\tYou can have access big databases, as well as saving your data tables in your own database called 'MyDB'. The user can run synchronous or asynchronous SQL queries and get the result back as a Pandas data-frame (using CasJobs.executeQuery or CasJobs.submitJob, respectively). Uploading of CSV files or data-frames into a database table can be done using CasJobs.uploadCSVToTable and CasJobs.uploadDataFrameToTable, respectively. -* `SciDrive `_: Drag-and-drop file storage and sharing. +* `SciQuery`: Advanced database storage and querying. +\t\tSciQuery expands the capabilities of CasJobs, as it is designed to work with databases loaded in several database management systems beyond MS SQL Server, such as PostgreSQL and MySQL. Query result tables can also be directly written as JSON or CSV files in the SciServer filesystem. + +* `SciDrive `_: Drag-and-drop file storage and sharing. \t\tYou can create directories in SciDrive using SciDrive.createContainer, upload a file to SciDrive using SciDrive.upload, and share its URL with your collaborators by using SciDrive.publicUrl. -* `SkyServer `_: Access to the SDSS astronomical survey. +* `SkyServer `_: Access to the SDSS astronomical survey. \t\tYou can query the SDSS database using SkyServer.sqlSearch, run cone searches using SkyServer.radialSearch, or get cutout images from the sky using SkyServer.getJpegImgCutout, between other tasks. -* `SkyQuery `_: Cross-match of astronomical source catalogs. +* `SkyQuery `_: Cross-match of astronomical source catalogs. \t\tYou can use this scalable database system for uploading your own catalogs and cross-matching them against huge astronomical source catalogs, or even cross-matching huge catalogs against each other!. Use SkyQuery.submitJob to run the cross-match, and use SkyQuery.listAllDatasets, SkyQuery.listDatasetTables and SkyQuery.listTableColumns to browse the catalogs and database schema. +* `Compute Jobs `_: Submission of Jupyter notebooks or shell commands as jobs +\t\tYou can execute whole Jupyter notebooks and shell commands as asynchronous batch jobs, as well as synchronous jobs. + +* `SciQuery Jobs `_: Submission of SQL queries. +\t\tYou can execute synchronous or s synchronous SQL queries against Postgres and other database backends. + +* `Files `_: Interaction with the SciServer file system. +\t\tYou can upload, and download data into your own file space in SciServer, as well as share data with your collaborators, between other things. + + **References** * SciServer-Python repository: http://www.github.com/sciserver/SciScript-Python @@ -31,7 +44,7 @@ **Authors**: Gerard Lemson , Manuchehr Taghizadeh-Popp -**Version**: sciserver-v1.10.0 +**Version**: sciserver-v2.2.0 """ diff --git a/py3/setup.py b/py3/setup.py index 33d3959..26e8abf 100644 --- a/py3/setup.py +++ b/py3/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages setup( name = "SciServer", - version = "1.10.0", + version = "2.2.0", packages = find_packages(), ) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..cb22f83 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +cachetools \ No newline at end of file