Saturday, January 10, 2015

build hadoop on traditional HPC

several useful tutorials:
1. http://www.glennklockwood.com/di/hadoop-deploy.php

Configuration:

DFS_NAME_DIR
DFS_DATA_DIR
MAPRED_LOCAL_DIR

config_subs[MASTER_NODE]="$MASTER_NODE"
config_subs[MAPRED_LOCAL_DIR]="$MH_SCRATCH_DIR/mapred_scratch"
config_subs[HADOOP_TMP_DIR]="$MH_SCRATCH_DIR/tmp"
config_subs[DFS_NAME_DIR]="$MH_SCRATCH_DIR/namenode_data"
config_subs[DFS_DATA_DIR]="$MH_SCRATCH_DIR/hdfs_data"
config_subs[DFS_REPLICATION]="$MH_DFS_REPLICATION"
config_subs[DFS_BLOCK_SIZE]="$MH_DFS_BLOCK_SIZE"
config_subs[MAPRED_TASKTRACKER_MAP_TASKS_MAXIMUM]="$MH_MAP_TASKS_MAXIMUM"
config_subs[MAPRED_TASKTRACKER_REDUCE_TASKS_MAXIMUM]="$MH_REDUCE_TASKS_MAXIMUM"
config_subs[MAPRED_MAP_TASKS]="$MH_MAP_TASKS"
config_subs[MAPRED_REDUCE_TASKS]="$MH_REDUCE_TASKS"
config_subs[HADOOP_LOG_DIR]="$MH_SCRATCH_DIR/logs"
config_subs[HADOOP_PID_DIR]="$MH_SCRATCH_DIR/pids"