Commit 86341220 authored by Gao, Shang's avatar Gao, Shang
Browse files

updated pbs script for rhea environment

parent 09619ca4

yelp_example/load_dataset.py

+4 −3
Original line number Diff line number Diff line
@@ -2,7 +2,8 @@ from crossbowGlobus import crossbowGlobus 
import time



#connect to CKAN, make sure yelp dataset exists

cbow = crossbowGlobus(api_key="eaabd7d9-3cb4-4014-85fe-73736e658472")

cbow = crossbowGlobus(api_key="eaabd7d9-3cb4-4014-85fe-73736e658472",

                      token_file='../crossbow/refresh-tokens.json')

packages = cbow.list_packages()

if 'yelp' not in packages:

    raise Exception("yelp package is missing from CKAN")
@@ -12,8 +13,8 @@ if 'yelp_academic_dataset_review.json' not in resources: 
    raise Exception("yelp_academic_dataset_review.json is missing from yelp package")



#download yelp dataset

endpoint = '6a3ced3a-0d9a-11e7-bb38-22000b9a448b'

path = '/~/Desktop/crossbow/yelp_example'

endpoint = 'ef1a9560-7ca1-11e5-992c-22000b96db58'

path = '/~/crossbow/yelp_example'

dl_id = cbow.download_resource('yelp','yelp_academic_dataset_review.json',

                               dest_endpoint=endpoint,dest_path=path)

yelp_example/yelp.pbs

+7 −11
Original line number Diff line number Diff line 
#!/bin/bash

#PBS -A CSC237

#PBS -N SHANG_HAN

#PBS -l walltime=8:00:00

#PBS -N YELP_TEST

#PBS -l walltime=1:00:00

#PBS -l nodes=1

#PBS -lpartition=gpu

#PBS -lpartition=cpu



cd ${HOME}/



module unload PE-intel/14.0.4-1.8.4

module load PE-gnu/4.7.1

module load cudatoolkit



git clone 

rm -rf crossbow

git clone https://code.ornl.gov/v33/crossbow.git



export PATH=/ccs/home/iamshang/anaconda2/bin:/ccs/home/iamshang/anaconda2/lib/python2.7/site-packages:$PATH

export PYTHONPATH=/ccs/home/iamshang/crossbow/crossbow:$PYTHONPATH







THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32,dnn.enabled=False,'gcc.cxxflags=-march=core2' python ${HOME}/th_han.py
 
python ${HOME}/crossbow/yelp_example/load_dataset.py

python ${HOME}/crossbow/yelp_example/feature_extraction.py ${HOME}/crossbow/yelp_example/yelp_academic_dataset_review.json