org.apache.pig.backend.hadoop.executionengine.spark

Class SparkUtil

java.lang.Object

org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil

public class SparkUtil
extends java.lang.Object

Constructor Summary

Constructors

Constructor and Description
SparkUtil()

Method Summary

Modifier and Type	Method and Description
static void	assertPredecessorSize(java.util.List<<any>> predecessors, PhysicalOperator physicalOperator, int size)
static void	assertPredecessorSizeGreaterThan(java.util.List<<any>> predecessors, PhysicalOperator physicalOperator, int size)
static void	createIndexerSparkNode(SparkOperator baseSparkOp, java.lang.String scope, NodeIdGenerator nig)
static <T,R> FlatMapFunction	flatMapFunction(FlatMapFunctionAdapter<T,R> function)
static <T> <any>	getManifest(java.lang.Class<T> clazz)
static Partitioner	getPartitioner(java.lang.String customPartitioner, int parallelism)
static <K,V> <any>	getProduct2Manifest()
static <K,V> <any>	getTuple2Manifest()
static org.apache.hadoop.mapred.JobConf	newJobConf(PigContext pigContext, PhysicalPlan physicalPlan, SparkEngineConf sparkEngineConf)
static <T,K,V> <any>	pairFlatMapFunction(PairFlatMapFunctionAdapter<T,K,V> function)
static <T> <any>	toScalaSeq(java.util.List<T> list)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

SparkUtil

public SparkUtil()

Method Detail

getManifest

public static <T> <any> getManifest(java.lang.Class<T> clazz)

getTuple2Manifest

public static <K,V> <any> getTuple2Manifest()

getProduct2Manifest

public static <K,V> <any> getProduct2Manifest()

newJobConf

public static org.apache.hadoop.mapred.JobConf newJobConf(PigContext pigContext,
                                                          PhysicalPlan physicalPlan,
                                                          SparkEngineConf sparkEngineConf)
                                                   throws java.io.IOException

Throws:

java.io.IOException

toScalaSeq

public static <T> <any> toScalaSeq(java.util.List<T> list)

assertPredecessorSize

public static void assertPredecessorSize(java.util.List<<any>> predecessors,
                                         PhysicalOperator physicalOperator,
                                         int size)

assertPredecessorSizeGreaterThan

public static void assertPredecessorSizeGreaterThan(java.util.List<<any>> predecessors,
                                                    PhysicalOperator physicalOperator,
                                                    int size)

getPartitioner

public static Partitioner getPartitioner(java.lang.String customPartitioner,
                                         int parallelism)

createIndexerSparkNode

public static void createIndexerSparkNode(SparkOperator baseSparkOp,
                                          java.lang.String scope,
                                          NodeIdGenerator nig)
                                   throws PlanException,
                                          ExecException

Throws:

PlanException

ExecException

flatMapFunction

public static <T,R> FlatMapFunction flatMapFunction(FlatMapFunctionAdapter<T,R> function)

pairFlatMapFunction

public static <T,K,V> <any> pairFlatMapFunction(PairFlatMapFunctionAdapter<T,K,V> function)