Commit 5cc45249 authored by Gao, Shang's avatar Gao, Shang
Browse files

updated all experiments to output prediction csv

parent 0f5212a6

crossbow/crossbowBase.py

+20 −0
Original line number Diff line number Diff line
@@ -209,3 +209,23 @@ class crossbowBase(object): 
            rsc_metadata['description'] = new_description



        self.ckan.action.resource_update(**rsc_metadata)

        

    def check_resource_exists(self,package,resource):

        '''

        check if a resource already exists on the CKAN server



        parameters:

          - package: string

            name of package

          - resource: string

            name of resource

            

        output: boolean

            whether or not the resource already exists on the CKAN server

        '''

        pkg_metadata = self.ckan.action.package_show(id=package)

        rsc_metadata = [metadata for metadata in pkg_metadata['resources'] \

                        if metadata['name']==resource]

        if len(rsc_metadata) > 0:

            return True

        return False

crossbow/crossbowGlobus.py

+4 −4
Original line number Diff line number Diff line
@@ -54,6 +54,9 @@ class crossbowGlobus(crossbowBase): 
      - get_resource_details(package,resource)

        prints and returns details for a specific resource

        

      - check_resource_exists(package,resource)

        check if a resource already exists on the CKAN server



      - upload_resource(package,resource_path,source_endpoint=olcfatlas,

                        description=None)

        upload a resource at a globus endpoint to an existing package
@@ -186,10 +189,7 @@ class crossbowGlobus(crossbowBase): 
            raise Exception("Could not find specified file")

            

        #make sure file is not already on ckan

        pkg_metadata = self.ckan.action.package_show(id=package)

        rsc_metadata = [metadata for metadata in pkg_metadata['resources'] \

                        if metadata['name']==filename]

        if len(rsc_metadata) > 0:

        if self.check_resource_exists(package,filename):

            raise Exception("File already exists on CKAN, delete the existing file first")



        #get file info

crossbow/crossbowMount.py

+4 −1
Original line number Diff line number Diff line
@@ -51,6 +51,9 @@ class crossbowMount(crossbowBase): 
      - get_resource_details(package,resource)

        prints and returns details for a specific resource

        

      - check_resource_exists(package,resource)

        check if a resource already exists on the CKAN server



      - upload_resource(package,resource,resource_path,description=None)

        upload a local resource to an existing package

        resource is copied to CROSSBOW_NFS/package/filename
@@ -123,7 +126,7 @@ class crossbowMount(crossbowBase): 
        url = "file://CROSSBOW_NFS/%s/%s" % (package,filename)



        #check if file already exists on nfs

        if os.path.isfile(nfs_path):

        if self.check_resource_exists(package,filename):

            raise Exception("File already exists on CKAN, delete the existing file first")



        #create directory if necessary

tests/test_crossbowGlobus.py

+19 −16
Original line number Diff line number Diff line
@@ -11,9 +11,9 @@ testing_endpoint = '6a3ced3a-0d9a-11e7-bb38-22000b9a448b' 
globus_crossbow_path = '/~/Desktop/crossbow/'



def setup_module():

    #create cbowMount class for testing

    global cbowMount

    cbowMount = crossbowGlobus(

    #create cbowGlobus class for testing

    global cbowGlobus

    cbowGlobus = crossbowGlobus(

            api_key="eaabd7d9-3cb4-4014-85fe-73736e658472",

            token_file='./crossbow/refresh-tokens.json')
@@ -47,40 +47,43 @@ def setup_module(): 
    tempfile.close()



def test_list_packages():

    assert "nosetests1" in cbowMount.list_packages()

    assert "nosetests1" in cbowGlobus.list_packages()



def test_get_package_details():

    assert cbowMount.get_package_details("nosetests1")['title'] == "nosetests1"

    assert cbowGlobus.get_package_details("nosetests1")['title'] == "nosetests1"



def test_create_package():

    cbowMount.create_package(

    cbowGlobus.create_package(

            "nosetests2",owner_org="test",title="nosetests2",description="nosetests2",

            author="nosetests2")

    assert "nosetests2" in ckan.action.package_list()



def test_edit_package():

    cbowMount.edit_package("nosetests1","test",title="nosetests1b")

    cbowGlobus.edit_package("nosetests1","test",title="nosetests1b")

    metadata = ckan.action.package_show(id="nosetests1")

    resources = [resource['name'] for resource in metadata['resources']]

    assert metadata["title"] == "nosetests1b"

    assert "testfile.csv" in resources



def test_list_resources():

    assert "testfile.csv" in cbowMount.list_resources("nosetests1")

    assert "testfile.dcd" in cbowMount.list_resources("nosetests1")

    assert "testfile.csv" in cbowGlobus.list_resources("nosetests1")

    assert "testfile.dcd" in cbowGlobus.list_resources("nosetests1")



def test_list_dcd_resources():

    assert "testfile.dcd" in cbowMount.list_dcd_resources("nosetests1")

    assert "testfile.dcd" in cbowGlobus.list_dcd_resources("nosetests1")



def test_list_non_dcd_resources():

    assert "testfile.csv" in cbowMount.list_non_dcd_resources("nosetests1")

    assert "testfile.csv" in cbowGlobus.list_non_dcd_resources("nosetests1")



def test_get_resource_details():

    assert cbowMount.get_resource_details("nosetests1","testfile.csv")['url'] \

    assert cbowGlobus.get_resource_details("nosetests1","testfile.csv")['url'] \

           == "file://CROSSBOW_NFS/nosetests1/testfile.csv"

           

def test_check_resource_exists():

    assert cbowGlobus.check_resource_exists("nosetests1","testfile.csv") == True



def test_upload_resource():

    cbowMount.upload_resource(

    cbowGlobus.upload_resource(

            "nosetests1",globus_crossbow_path+"/testfile2.dcd",

            source_endpoint=testing_endpoint,description="myresource3")

    time.sleep(15)
@@ -92,7 +95,7 @@ def test_upload_resource(): 
    assert os.path.isfile('/data/nosetests1/testfile2.dcd')



def test_edit_resource_description():

    cbowMount.edit_resource_description("nosetests1","testfile.csv",

    cbowGlobus.edit_resource_description("nosetests1","testfile.csv",

                                        new_description="myresource1b")

    pkg_metadata = ckan.action.package_show(id="nosetests1")

    rsc_metadata = [metadata for metadata in pkg_metadata['resources'] \
@@ -100,14 +103,14 @@ def test_edit_resource_description(): 
    assert rsc_metadata['description'] == "myresource1b"



def test_download_resource():

    cbowMount.download_resource("nosetests1","testfile.csv",

    cbowGlobus.download_resource("nosetests1","testfile.csv",

                                dest_endpoint=testing_endpoint,

                                dest_path=globus_crossbow_path,

                                wait_for_download=True,timeout=15)

    assert os.path.isfile('testfile.csv')



def test_delete_resource():

    cbowMount.delete_resource("nosetests1","testfile.csv")

    cbowGlobus.delete_resource("nosetests1","testfile.csv")

    time.sleep(5)

    metadata = ckan.action.package_show(id="nosetests1")

    resources = [resource['name'] for resource in metadata['resources']]

tests/test_crossbowMount.py

+3 −0
Original line number Diff line number Diff line
@@ -75,6 +75,9 @@ def test_get_resource_details(): 
    assert cbowMount.get_resource_details("nosetests1","testfile.csv")['url'] \

           == "file://CROSSBOW_NFS/nosetests1/testfile.csv"

           

def test_check_resource_exists():

    assert cbowMount.check_resource_exists("nosetests1","testfile.csv") == True



def test_upload_resource():

    cbowMount.upload_resource(

            "nosetests1","./testfile2.dcd",description="myresource3")