Commit b34579dc authored by Gao, Shang's avatar Gao, Shang
Browse files

adding data transfer back for yelp experiment

parent f52228df

crossbow/crossbowOlcf.py

+2 −2
Original line number Diff line number Diff line
@@ -35,13 +35,13 @@ class crossbowOlcf(object): 
    def scp(self):

        pass

        

    def get_globus_tokens(self,olcf_path='./'):

    def get_globus_tokens(self,crossbow_path='./'):



        #clone crossbow repo        

        ssh_stdin, ssh_stdout, ssh_stderr = \

            self.ssh.exec_command("cd %s; if [ ! -d crossbow ]; \

                                   then git clone https://code.ornl.gov/v33/crossbow.git;\

                                   else cd crossbow/; git pull; fi" % olcf_path)

                                   else cd crossbow/; git pull; fi" % crossbow_path)

        print ssh_stdout.readlines()

        

        time.sleep(1)

mnist_example/run_experiment.py

+3 −0
Original line number Diff line number Diff line
@@ -25,6 +25,9 @@ if 'mnist_test.csv' not in resources: 
#connect to OLCF before download starts

olcf = crossbowOlcf('rhea')



#download refresh token to OLCF

olcf.get_globus_tokens(olcf_path='./')



#download mnist dataset if it doesn't exist

olcf_endpoint = 'ef1a9560-7ca1-11e5-992c-22000b96db58'

path = '/~/'

yelp_example/naive_bayes.pbs

+2 −0
Original line number Diff line number Diff line
@@ -25,6 +25,8 @@ if [ ! -f embeddings.npy ] && [ ! -f data.pkl ]; then 
fi

python naive_bayes.py



python upload_results.py '../results/naive_bayes_preds.csv'



#cleanup

cd ${HOME}/

rm naive_bayes.pbs

yelp_example/scripts/upload_results.py

0 → 100644
+14 −0
Original line number Diff line number Diff line 
from crossbowGlobus import crossbowGlobus



#get csv file to upload

args = (sys.argv)

if len(args) != 2:

    raise Exception("Usage: python upload_results.py <.csv file>")

csv_file = args[1]



#connect to CKAN, upload csv

cbow = crossbowGlobus(api_key="eaabd7d9-3cb4-4014-85fe-73736e658472",

                      token_file='../crossbow/refresh-tokens.json')

                      

cbow.upload_resource('yelp',csv_file,source_endpoint=olcfatlas,description=None)