org.apache.pig.tools.pigstats.spark

Class SparkStatsUtil

java.lang.Object

org.apache.pig.tools.pigstats.spark.SparkStatsUtil

public class SparkStatsUtil
extends Object

Field Summary

Fields

Modifier and Type	Field and Description
static String	SPARK_INPUT_COUNTER_GROUP
static String	SPARK_INPUT_RECORD_COUNTER
static String	SPARK_STORE_COUNTER_GROUP
static String	SPARK_STORE_RECORD_COUNTER

Constructor Summary

Constructors

Constructor and Description
SparkStatsUtil()

Method Summary

Modifier and Type	Method and Description
static void	addFailedNativeJobStats(PigStats ps, NativeSparkOperator nativeSparkOperator, Exception e)
static void	addFailJobStats(String jobID, POStore poStore, SparkOperator sparkOperator, SparkPigStats sparkPigStats, Exception e)
static void	addNativeJobStats(PigStats ps, NativeSparkOperator nativeSparkOperator)
static String	getCounterName(POLoad load)
static String	getCounterName(POStore store)
static long	getRecordCount(POLoad load)
static long	getRecordCount(POStore store)
static boolean	isJobSuccess(int jobID, JavaSparkContext sparkContext)
static void	waitForJobAddStats(int jobID, POStore poStore, SparkOperator sparkOperator, JobMetricsListener jobMetricsListener, JavaSparkContext sparkContext, SparkPigStats sparkPigStats)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

SPARK_STORE_COUNTER_GROUP

public static final String SPARK_STORE_COUNTER_GROUP

See Also:

Constant Field Values

SPARK_STORE_RECORD_COUNTER

public static final String SPARK_STORE_RECORD_COUNTER

See Also:

Constant Field Values

SPARK_INPUT_COUNTER_GROUP

public static final String SPARK_INPUT_COUNTER_GROUP

See Also:

Constant Field Values

SPARK_INPUT_RECORD_COUNTER

public static final String SPARK_INPUT_RECORD_COUNTER

See Also:

Constant Field Values

Constructor Detail

SparkStatsUtil

public SparkStatsUtil()

Method Detail

waitForJobAddStats

public static void waitForJobAddStats(int jobID,
                                      POStore poStore,
                                      SparkOperator sparkOperator,
                                      JobMetricsListener jobMetricsListener,
                                      JavaSparkContext sparkContext,
                                      SparkPigStats sparkPigStats)
                               throws InterruptedException

Throws:

InterruptedException

addFailJobStats

public static void addFailJobStats(String jobID,
                                   POStore poStore,
                                   SparkOperator sparkOperator,
                                   SparkPigStats sparkPigStats,
                                   Exception e)

getCounterName

public static String getCounterName(POStore store)

getCounterName

public static String getCounterName(POLoad load)

getRecordCount

public static long getRecordCount(POStore store)

getRecordCount

public static long getRecordCount(POLoad load)

isJobSuccess

public static boolean isJobSuccess(int jobID,
                                   JavaSparkContext sparkContext)

addNativeJobStats

public static void addNativeJobStats(PigStats ps,
                                     NativeSparkOperator nativeSparkOperator)

addFailedNativeJobStats

public static void addFailedNativeJobStats(PigStats ps,
                                           NativeSparkOperator nativeSparkOperator,
                                           Exception e)