org.apache.pig.tools.pigstats
Class PigStatsUtil

java.lang.Object
  extended by org.apache.pig.tools.pigstats.PigStatsUtil

public abstract class PigStatsUtil
extends Object

A utility class for Pig Statistics


Field Summary
static String FS_COUNTER_GROUP
           
static String HDFS_BYTES_READ
           
static String HDFS_BYTES_WRITTEN
           
static String MAP_INPUT_RECORDS
           
static String MAP_OUTPUT_RECORDS
           
static String MULTI_INPUTS_COUNTER_GROUP
           
static String MULTI_INPUTS_RECORD_COUNTER
           
static String MULTI_STORE_COUNTER_GROUP
           
static String MULTI_STORE_RECORD_COUNTER
           
static String REDUCE_INPUT_RECORDS
           
static String REDUCE_OUTPUT_RECORDS
           
static String TASK_COUNTER_GROUP
           
 
Constructor Summary
PigStatsUtil()
           
 
Method Summary
static void accumulateStats(org.apache.hadoop.mapred.jobcontrol.JobControl jc)
          Updates the statistics after a patch of jobs is done
static JobStats addNativeJobStats(PigStats ps, NativeMapReduceOper mr, boolean success)
           
static JobStats addNativeJobStats(PigStats ps, NativeMapReduceOper mr, boolean success, Exception e)
           
static void displayStatistics()
          Logs the statistics in the Pig log file at INFO level
static PigStats getEmptyPigStats()
          Returns an empty PigStats object
static String getMultiInputsCounterName(String fname, int index)
          Returns the counter name for the given input file name
static long getMultiStoreCount(org.apache.hadoop.mapred.jobcontrol.Job job, org.apache.hadoop.mapred.JobClient jobClient, String counterName)
          Returns the count for the given counter name in the counter group 'MultiStoreCounters'
static String getMultiStoreCounterName(POStore store)
          Returns the counter name for the given POStore
static PigStats getPigStats(int code)
          Returns the PigStats with the given return code
static boolean isTempFile(String fileName)
           
static void setBackendException(org.apache.hadoop.mapred.jobcontrol.Job job, Exception e)
           
static void setErrorCode(int code)
           
static void setErrorMessage(String msg)
           
static void setStatsMap(Map<String,List<PigStats>> statsMap)
           
static void startCollection(PigContext pc, org.apache.hadoop.mapred.JobClient client, JobControlCompiler jcc, MROperPlan plan)
          Starts collecting statistics for the given MR plan
static void stopCollection(boolean display)
          Stops collecting statistics for a MR plan
static void updateJobMroMap(Map<org.apache.hadoop.mapred.jobcontrol.Job,MapReduceOper> jobMroMap)
          Updates the PigStats.JobGraph of the PigStats.
 
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
 

Field Detail

MULTI_STORE_RECORD_COUNTER

public static final String MULTI_STORE_RECORD_COUNTER
See Also:
Constant Field Values

MULTI_STORE_COUNTER_GROUP

public static final String MULTI_STORE_COUNTER_GROUP
See Also:
Constant Field Values

TASK_COUNTER_GROUP

public static final String TASK_COUNTER_GROUP
See Also:
Constant Field Values

FS_COUNTER_GROUP

public static final String FS_COUNTER_GROUP

MAP_INPUT_RECORDS

public static final String MAP_INPUT_RECORDS
See Also:
Constant Field Values

MAP_OUTPUT_RECORDS

public static final String MAP_OUTPUT_RECORDS
See Also:
Constant Field Values

REDUCE_INPUT_RECORDS

public static final String REDUCE_INPUT_RECORDS
See Also:
Constant Field Values

REDUCE_OUTPUT_RECORDS

public static final String REDUCE_OUTPUT_RECORDS
See Also:
Constant Field Values

HDFS_BYTES_WRITTEN

public static final String HDFS_BYTES_WRITTEN
See Also:
Constant Field Values

HDFS_BYTES_READ

public static final String HDFS_BYTES_READ
See Also:
Constant Field Values

MULTI_INPUTS_RECORD_COUNTER

public static final String MULTI_INPUTS_RECORD_COUNTER
See Also:
Constant Field Values

MULTI_INPUTS_COUNTER_GROUP

public static final String MULTI_INPUTS_COUNTER_GROUP
See Also:
Constant Field Values
Constructor Detail

PigStatsUtil

public PigStatsUtil()
Method Detail

getMultiStoreCount

public static long getMultiStoreCount(org.apache.hadoop.mapred.jobcontrol.Job job,
                                      org.apache.hadoop.mapred.JobClient jobClient,
                                      String counterName)
Returns the count for the given counter name in the counter group 'MultiStoreCounters'

Parameters:
job - the MR job
jobClient - the Hadoop job client
counterName - the counter name
Returns:
the count of the given counter name

getMultiStoreCounterName

public static String getMultiStoreCounterName(POStore store)
Returns the counter name for the given POStore

Parameters:
store - the POStore
Returns:
the counter name

getMultiInputsCounterName

public static String getMultiInputsCounterName(String fname,
                                               int index)
Returns the counter name for the given input file name

Parameters:
fname - the input file name
Returns:
the counter name

startCollection

public static void startCollection(PigContext pc,
                                   org.apache.hadoop.mapred.JobClient client,
                                   JobControlCompiler jcc,
                                   MROperPlan plan)
Starts collecting statistics for the given MR plan

Parameters:
pc - the Pig context
client - the Hadoop job client
jcc - the job compiler
plan - the MR plan

stopCollection

public static void stopCollection(boolean display)
Stops collecting statistics for a MR plan

Parameters:
display - if true, log collected statistics in the Pig log file at INFO level

getEmptyPigStats

public static PigStats getEmptyPigStats()
Returns an empty PigStats object

Returns:
an empty PigStats object

getPigStats

public static PigStats getPigStats(int code)
Returns the PigStats with the given return code

Parameters:
code - the return code
Returns:
the PigStats with the given return code

displayStatistics

public static void displayStatistics()
Logs the statistics in the Pig log file at INFO level


updateJobMroMap

public static void updateJobMroMap(Map<org.apache.hadoop.mapred.jobcontrol.Job,MapReduceOper> jobMroMap)
Updates the PigStats.JobGraph of the PigStats. The initial PigStats.JobGraph is created without job ids using MROperPlan, before any job is submitted for execution. The PigStats.JobGraph then is updated with job ids after jobs are executed.

Parameters:
jobMroMap - the map that maps Jobs to MapReduceOpers

accumulateStats

public static void accumulateStats(org.apache.hadoop.mapred.jobcontrol.JobControl jc)
Updates the statistics after a patch of jobs is done

Parameters:
jc - the job control

setErrorMessage

public static void setErrorMessage(String msg)

setErrorCode

public static void setErrorCode(int code)

setBackendException

public static void setBackendException(org.apache.hadoop.mapred.jobcontrol.Job job,
                                       Exception e)

isTempFile

public static boolean isTempFile(String fileName)

addNativeJobStats

public static JobStats addNativeJobStats(PigStats ps,
                                         NativeMapReduceOper mr,
                                         boolean success)

setStatsMap

public static void setStatsMap(Map<String,List<PigStats>> statsMap)

addNativeJobStats

public static JobStats addNativeJobStats(PigStats ps,
                                         NativeMapReduceOper mr,
                                         boolean success,
                                         Exception e)


Copyright © 2007-2012 The Apache Software Foundation