org.apache.pig.backend.hadoop.executionengine.spark

Class SparkUtil

java.lang.Object

org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil

public class SparkUtil
extends Object

Constructor Summary

Constructors

Constructor and Description
SparkUtil()

Method Summary

Modifier and Type	Method and Description
static void	assertPredecessorSize(List<<any>> predecessors, PhysicalOperator physicalOperator, int size)
static void	assertPredecessorSizeGreaterThan(List<<any>> predecessors, PhysicalOperator physicalOperator, int size)
static void	createIndexerSparkNode(SparkOperator baseSparkOp, String scope, NodeIdGenerator nig)
static <T> <any>	getManifest(Class<T> clazz)
static Partitioner	getPartitioner(String customPartitioner, int parallelism)
static <K,V> <any>	getProduct2Manifest()
static <K,V> <any>	getTuple2Manifest()
static org.apache.hadoop.mapred.JobConf	newJobConf(PigContext pigContext, PhysicalPlan physicalPlan, SparkEngineConf sparkEngineConf)
static <T> <any>	toScalaSeq(List<T> list)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

SparkUtil

public SparkUtil()

Method Detail

getManifest

public static <T> <any> getManifest(Class<T> clazz)

getTuple2Manifest

public static <K,V> <any> getTuple2Manifest()

getProduct2Manifest

public static <K,V> <any> getProduct2Manifest()

newJobConf

public static org.apache.hadoop.mapred.JobConf newJobConf(PigContext pigContext,
                                                          PhysicalPlan physicalPlan,
                                                          SparkEngineConf sparkEngineConf)
                                                   throws IOException

Throws:

IOException

toScalaSeq

public static <T> <any> toScalaSeq(List<T> list)

assertPredecessorSize

public static void assertPredecessorSize(List<<any>> predecessors,
                                         PhysicalOperator physicalOperator,
                                         int size)

assertPredecessorSizeGreaterThan

public static void assertPredecessorSizeGreaterThan(List<<any>> predecessors,
                                                    PhysicalOperator physicalOperator,
                                                    int size)

getPartitioner

public static Partitioner getPartitioner(String customPartitioner,
                                         int parallelism)

createIndexerSparkNode

public static void createIndexerSparkNode(SparkOperator baseSparkOp,
                                          String scope,
                                          NodeIdGenerator nig)
                                   throws PlanException,
                                          ExecException

Throws:

PlanException

ExecException