MRjob
-----
# if not yet installed, install mrjob
python3 -m venv mypython
mypython/bin/pip install mrjob
mypython/bin/python
FILE=./data/fruits.txt

# run a job locally on $FILE
mypython/bin/python mr_wordcount.py $FILE

# run a job on the Hadoop cluster on $FILE
module load Hadoop
export STREAMING_JAR=/opt/apps/software/Hadoop/2.6.0-cdh5.8.0-native/share/hadoop/tools/lib/hadoop-streaming-2.6.0-cdh5.8.0.jar
mypython/bin/python mr_wordcount.py --hadoop-streaming-jar $STREAMING_JAR -r hadoop $FILE