org.apache.pig.tools.pigstats.mapreduce
Class MRPigStatsUtil

java.lang.Object
  extended by org.apache.pig.tools.pigstats.PigStatsUtil
      extended by org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil

public class MRPigStatsUtil
extends PigStatsUtil

A utility class for Pig Statistics


Field Summary
static String FS_COUNTER_GROUP
           
static String TASK_COUNTER_GROUP
           
 
Fields inherited from class org.apache.pig.tools.pigstats.PigStatsUtil
HDFS_BYTES_READ, HDFS_BYTES_WRITTEN, MAP_INPUT_RECORDS, MAP_OUTPUT_RECORDS, MULTI_INPUTS_COUNTER_GROUP, MULTI_INPUTS_RECORD_COUNTER, MULTI_STORE_COUNTER_GROUP, MULTI_STORE_RECORD_COUNTER, REDUCE_INPUT_RECORDS, REDUCE_OUTPUT_RECORDS
 
Constructor Summary
MRPigStatsUtil()
           
 
Method Summary
static void accumulateStats(org.apache.hadoop.mapred.jobcontrol.JobControl jc)
          Updates the statistics after a patch of jobs is done
static JobStats addJobStats(org.apache.hadoop.mapred.jobcontrol.Job job)
          Add stats for a new Job, which doesn't yet need to be completed.
static MRJobStats addNativeJobStats(PigStats ps, NativeMapReduceOper mr, boolean success)
           
static MRJobStats addNativeJobStats(PigStats ps, NativeMapReduceOper mr, boolean success, Exception e)
           
static void displayStatistics()
          Logs the statistics in the Pig log file at INFO level
static long getMultiStoreCount(org.apache.hadoop.mapred.jobcontrol.Job job, org.apache.hadoop.mapred.JobClient jobClient, String counterName)
          Returns the count for the given counter name in the counter group 'MultiStoreCounters'
static void setBackendException(org.apache.hadoop.mapred.jobcontrol.Job job, Exception e)
           
static void startCollection(PigContext pc, org.apache.hadoop.mapred.JobClient client, JobControlCompiler jcc, MROperPlan plan)
          Starts collecting statistics for the given MR plan
static void stopCollection(boolean display)
          Stops collecting statistics for a MR plan
static void updateJobMroMap(Map<org.apache.hadoop.mapred.jobcontrol.Job,MapReduceOper> jobMroMap)
          Updates the PigStats.JobGraph of the PigStats.
 
Methods inherited from class org.apache.pig.tools.pigstats.PigStatsUtil
getEmptyPigStats, getMultiInputsCounterName, getMultiStoreCounterName, getPigStats, isTempFile, setErrorCode, setErrorMessage, setErrorThrowable, setStatsMap
 
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
 

Field Detail

TASK_COUNTER_GROUP

public static final String TASK_COUNTER_GROUP
See Also:
Constant Field Values

FS_COUNTER_GROUP

public static final String FS_COUNTER_GROUP
Constructor Detail

MRPigStatsUtil

public MRPigStatsUtil()
Method Detail

getMultiStoreCount

public static long getMultiStoreCount(org.apache.hadoop.mapred.jobcontrol.Job job,
                                      org.apache.hadoop.mapred.JobClient jobClient,
                                      String counterName)
Returns the count for the given counter name in the counter group 'MultiStoreCounters'

Parameters:
job - the MR job
jobClient - the Hadoop job client
counterName - the counter name
Returns:
the count of the given counter name

startCollection

public static void startCollection(PigContext pc,
                                   org.apache.hadoop.mapred.JobClient client,
                                   JobControlCompiler jcc,
                                   MROperPlan plan)
Starts collecting statistics for the given MR plan

Parameters:
pc - the Pig context
client - the Hadoop job client
jcc - the job compiler
plan - the MR plan

stopCollection

public static void stopCollection(boolean display)
Stops collecting statistics for a MR plan

Parameters:
display - if true, log collected statistics in the Pig log file at INFO level

addJobStats

public static JobStats addJobStats(org.apache.hadoop.mapred.jobcontrol.Job job)
Add stats for a new Job, which doesn't yet need to be completed.

Parameters:
job - the job being run
Returns:
JobStats for the job

displayStatistics

public static void displayStatistics()
Logs the statistics in the Pig log file at INFO level


updateJobMroMap

public static void updateJobMroMap(Map<org.apache.hadoop.mapred.jobcontrol.Job,MapReduceOper> jobMroMap)
Updates the PigStats.JobGraph of the PigStats. The initial PigStats.JobGraph is created without job ids using MROperPlan, before any job is submitted for execution. The PigStats.JobGraph then is updated with job ids after jobs are executed.

Parameters:
jobMroMap - the map that maps Jobs to MapReduceOpers

accumulateStats

public static void accumulateStats(org.apache.hadoop.mapred.jobcontrol.JobControl jc)
Updates the statistics after a patch of jobs is done

Parameters:
jc - the job control

setBackendException

@InterfaceAudience.Private
public static void setBackendException(org.apache.hadoop.mapred.jobcontrol.Job job,
                                                                 Exception e)

addNativeJobStats

public static MRJobStats addNativeJobStats(PigStats ps,
                                           NativeMapReduceOper mr,
                                           boolean success)

addNativeJobStats

public static MRJobStats addNativeJobStats(PigStats ps,
                                           NativeMapReduceOper mr,
                                           boolean success,
                                           Exception e)


Copyright © 2007-2012 The Apache Software Foundation