Commit 224d311c authored by Gao, Shang's avatar Gao, Shang
Browse files

added *args before **args, details methods now both return and print

parent 3859b3cf

crossbow/crossbow.py

+41 −24
Original line number Diff line number Diff line
@@ -22,13 +22,16 @@ class crossbowBase(object): 
        '''

        return self.ckan.action.package_list()



    def print_package_details(self,package):

    def get_package_details(self,package):

        '''

        prints details for a specific package

        prints and returns details for a specific package



        parameters:

          - package: string

            name of package



        output: dictionary

            metadata for specific package

        '''

        metadata = self.ckan.action.package_show(id=package)

        print "package name: %s" % package
@@ -44,6 +47,8 @@ class crossbowBase(object): 
        print "resources: ", self.list_resources(package)

        print "tags: %s" % metadata['tags']



        return metadata



    def create_package(self,package,owner_org,title=None,description=None,author=None,author_email=None,

        maintainer=None,maintainer_email=None,version=None,tags=[]):

        '''
@@ -77,7 +82,7 @@ class crossbowBase(object): 
            author_email=author_email,maintainer=maintainer,maintainer_email=maintainer_email,

            notes=description,version=version,tags=tags)



    def edit_package(self,package,**kwargs):

    def edit_package(self,package,*args,**kwargs):

        '''

        edit an existing package (dataset)
@@ -86,6 +91,10 @@ class crossbowBase(object): 
            name of package

          see create_package() function for additional parameters

        '''

        #add args to kwargs dic

        keywords = ['owner_org','title','notes','author','author_email','maintainer','maintainer_email','version','tags']

        for idx,arg in enumerate(args):

            kwargs[keywords[idx]] = arg

        

        #replace kwargs keywords that don't match ckanapi keywords 

        if 'description' in kwargs:
@@ -116,15 +125,18 @@ class crossbowBase(object): 
        resources = [resource['name'] for resource in metadata['resources']]

        return resources



    def print_resource_details(self,package,resource):

    def get_resource_details(self,package,resource):

        '''

        prints details for a specific resource

        prints and returns details for a specific resource



        parameters:

          - package: string

            name of package

          - resource: string

            name of resource



        output: dictionary

            metadata for specific resource

        '''

        pkg_metadata = self.ckan.action.package_show(id=package)

        rsc_metadata = [metadata if metadata['name']==resource else None for metadata in pkg_metadata['resources']][0]
@@ -137,6 +149,8 @@ class crossbowBase(object): 
        print "created: %s" % rsc_metadata['created']

        print "last modified: %s" % metadata['last_modified']



        return rsc_metadata



class crossbowMount(crossbowBase):

    '''

    oak ridge national labs cross-platform big-data operational workflow
@@ -159,25 +173,25 @@ class crossbowMount(crossbowBase): 
      - list_packages()

        returns all packages (datasets) available on CKAN



      - print_package_details(package)

        prints details for a specific package

      - get_package_details(package)

        prints and returns details for a specific package



      - create_package(package,title,description,author,author_email,maintainer,maintainer_email,version,tags)

        creates a new package (dataset)



      - edit_package(package,**kwargs)

      - edit_package(package,*args,**kwargs)

        edit an existing package (dataset)



      - list_resources(package)

        returns all resources (scripts, datasets, and other) for a particular package



      - print_resource_details(package,resource)

        prints details for a specific resource

      - get_resource_details(package,resource)

        prints and returns details for a specific resource



      - add_resource(package,resource,resource_URI,description,format,size,autodetect)

      - add_resource(package,resource,resource_URI,description)

        add a resource located on local NFS mount to an existing package



      - edit_resource(package,resource,**kwargs)

      - edit_resource(package,resource,*args,**kwargs)

        edit an existing resource



      - get_resource_path(package,resource)
@@ -202,8 +216,7 @@ class crossbowMount(crossbowBase): 


        super(crossbowMount, self).__init__(api_key,CKAN_url,user_agent)



    def add_resource(self,package,resource,resource_path,description=None,format=None,

        size=None,autodetect=True):

    def add_resource(self,package,resource,resource_path,description=None):

        '''

        add a resource located on local NFS mount to an existing package
@@ -216,12 +229,6 @@ class crossbowMount(crossbowBase): 
            full path to resource, must be on NFS mount

          - description: string (optional)

            description of resource

          - format: string (optional)

            file format of resource (e.g. "DCD")

          - size: int (optional)

            size of resource

          - autodetect: boolean (default: True)

            if True, autodetect file format and size

        '''

        #check if file is on NFS

        if not os.path.isfile(resource_path):
@@ -229,7 +236,7 @@ class crossbowMount(crossbowBase): 
        if self.NFS_path not in resource_path:

            raise Exception("File must be located on NFS mount")



        if autodetect==True:

        #autodetect file info

        size = os.path.getsize(resource_path)

        _, format = os.path.splitext(resource_path)
@@ -239,7 +246,7 @@ class crossbowMount(crossbowBase): 
        self.ckan.action.resource_create(package_id=package,url=resource_path,name=resource,

            description=description,format=format,size=size)



    def edit_resource(self,package,resource,**kwargs):

    def edit_resource(self,package,resource,*args,**kwargs):

        '''

        edit an existing resource
@@ -254,6 +261,16 @@ class crossbowMount(crossbowBase): 
        pkg_metadata = self.ckan.action.package_show(id=package)

        rsc_metadata = [metadata if metadata['name']==resource else None for metadata in pkg_metadata['resources']][0]



        #add args to kwargs dic

        keywords = ['resource_path','description']

        for idx,arg in enumerate(args):

            kwargs[keywords[idx]] = arg



        #autodetect file info

        if 'resource_path' in kwargs.keys():

            kwargs['size'] = os.path.getsize(kwargs['resource_path'])

            _, kwargs['format'] = os.path.splitext(kwargs['resource_path'])



        #replace kwargs keywords that don't match ckanapi keywords

        if 'resource_path' in kwargs:

tests/test_crossbow.py

+12 −5
Original line number Diff line number Diff line
@@ -26,12 +26,15 @@ def setup_module(): 
def test_list_packages():

    assert "nosetests1" in cbowMount.list_packages()



def test_get_package_details():

    assert cbowMount.get_package_details("nosetests1")['title'] == "nosetests1"



def test_create_package():

    cbowMount.create_package("nosetests2",owner_org="test",title="nosetests2",description="nosetests2",author="nosetests2")

    assert "nosetests2" in ckan.action.package_list()



def test_edit_package():

    cbowMount.edit_package("nosetests1",title="nosetests1b")

    cbowMount.edit_package("nosetests1","test",title="nosetests1b")

    metadata = ckan.action.package_show(id="nosetests1")

    resources = [resource['name'] for resource in metadata['resources']]

    assert metadata["title"] == "nosetests1b"
@@ -40,6 +43,9 @@ def test_edit_package(): 
def test_list_resources():

    assert "myresource1" in cbowMount.list_resources("nosetests1")



def test_get_resource_details():

    assert cbowMount.get_resource_details("nosetests1","myresource1")['url'] == "file://CROSSBOW_NFS/testfile.csv"



def test_add_resource():

    cbowMount.add_resource("nosetests1","myresource2","/data/testfile.csv",description="myresource2")

    metadata = ckan.action.package_show(id="nosetests1")
@@ -47,7 +53,7 @@ def test_add_resource(): 
    assert "myresource2" in resources



def test_edit_resource():

    cbowMount.edit_resource("nosetests1","myresource1",description="myresource1b")

    cbowMount.edit_resource("nosetests1","myresource1","/data/testfile.csv",description="myresource1b")

    pkg_metadata = ckan.action.package_show(id="nosetests1")

    rsc_metadata = [metadata if metadata['name']=="myresource1" else None for metadata in pkg_metadata['resources']][0]

    assert rsc_metadata['description'] == "myresource1b"
@@ -72,3 +78,4 @@ def teardown_module(): 
    #delete temporary file if it exists

    if os.path.isfile('/data/testfile.csv'):

       os.remove('/data/testfile.csv')