org.apache.pig.backend.hadoop.executionengine.spark.plan

Class SparkOperator

java.lang.Object

org.apache.pig.impl.plan.Operator<SparkOpPlanVisitor>

org.apache.pig.backend.hadoop.executionengine.spark.plan.SparkOperator

All Implemented Interfaces:

Serializable, Cloneable, Comparable<Operator>

Direct Known Subclasses:

NativeSparkOperator

public class SparkOperator
extends Operator<SparkOpPlanVisitor>

An operator model for a Spark job. Acts as a host to the plans that will execute in spark.

See Also:

Serialized Form

Field Summary

Fields

Modifier and Type	Field and Description
String	customPartitioner
PhysicalPlan	physicalPlan
int	requestedParallelism
Set<PhysicalOperator>	scalars
Set<String>	UDFs

Fields inherited from class org.apache.pig.impl.plan.Operator

mKey

Constructor Summary

Constructors

Constructor and Description
SparkOperator(OperatorKey k)

Method Summary

Modifier and Type	Method and Description
void	add(PhysicalOperator physicalOper)
void	addCrossKey(String key)
void	addMultiQueryOptimizeConnectionItem(OperatorKey to, OperatorKey from)
boolean	combineSmallSplits()
void	copyFeatures(SparkOperator copyFrom, List<org.apache.pig.backend.hadoop.executionengine.spark.plan.SparkOperator.OPER_FEATURE> excludeFeatures)
List<String>	getCrossKeys()
MultiMap<OperatorKey,OperatorKey>	getMultiQueryOptimizeConnectionItem()
int	getRequestedParallelism()
String	getSkewedJoinPartitionFile()
boolean	isCogroup()
boolean	isGlobalSort()
boolean	isGroupBy()
boolean	isIndexer()
boolean	isLimit()
boolean	isLimitAfterSort()
boolean	isNative()
boolean	isRegularJoin()
boolean	isSampler()
boolean	isSkewedJoin()
boolean	isSplitter()
boolean	isUnion()
void	markCogroup()
void	markGroupBy()
void	markIndexer()
void	markLimit()
void	markLimitAfterSort()
void	markNative()
void	markRegularJoin()
void	markSampler()
void	markUnion()
String	name()
protected void	noCombineSmallSplits()
void	setGlobalSort(boolean globalSort)
void	setRequestedParallelism(int requestedParallelism)
void	setRequestedParallelismByReference(SparkOperator oper)
void	setSkewedJoinPartitionFile(String file)
void	setSplitter(boolean spl)
boolean	supportsMultipleInputs() Indicates whether this operator supports multiple inputs.
boolean	supportsMultipleOutputs() Indicates whether this operator supports multiple outputs.
protected void	useTypedComparator(boolean useTypedComparator)
protected boolean	usingTypedComparator()
void	visit(SparkOpPlanVisitor v) Visit this node with the provided visitor.

Methods inherited from class org.apache.pig.impl.plan.Operator

clone, compareTo, equals, getOperatorKey, getProjectionMap, hashCode, regenerateProjectionMap, rewire, toString, unsetProjectionMap

Methods inherited from class java.lang.Object

finalize, getClass, notify, notifyAll, wait, wait, wait

Field Detail

physicalPlan

public PhysicalPlan physicalPlan

UDFs

public Set<String> UDFs

customPartitioner

public String customPartitioner

scalars

public Set<PhysicalOperator> scalars

requestedParallelism

public int requestedParallelism

Constructor Detail

SparkOperator

public SparkOperator(OperatorKey k)

Method Detail

supportsMultipleInputs

public boolean supportsMultipleInputs()

Description copied from class: Operator

Indicates whether this operator supports multiple inputs.

Specified by:

supportsMultipleInputs in class Operator<SparkOpPlanVisitor>

Returns:

true if it does, otherwise false.

supportsMultipleOutputs

public boolean supportsMultipleOutputs()

Description copied from class: Operator

Indicates whether this operator supports multiple outputs.

Specified by:

supportsMultipleOutputs in class Operator<SparkOpPlanVisitor>

Returns:

true if it does, otherwise false.

name

public String name()

Specified by:

name in class Operator<SparkOpPlanVisitor>

add

public void add(PhysicalOperator physicalOper)

visit

public void visit(SparkOpPlanVisitor v)
           throws VisitorException

Description copied from class: Operator

Visit this node with the provided visitor. This should only be called by the visitor class itself, never directly.

Specified by:

visit in class Operator<SparkOpPlanVisitor>

Parameters:

v - Visitor to visit with.

Throws:

VisitorException - if the visitor has a problem.

addCrossKey

public void addCrossKey(String key)

getCrossKeys

public List<String> getCrossKeys()

isGroupBy

public boolean isGroupBy()

markGroupBy

public void markGroupBy()

isCogroup

public boolean isCogroup()

markCogroup

public void markCogroup()

isRegularJoin

public boolean isRegularJoin()

markRegularJoin

public void markRegularJoin()

getRequestedParallelism

public int getRequestedParallelism()

setSplitter

public void setSplitter(boolean spl)

isSplitter

public boolean isSplitter()

isSampler

public boolean isSampler()

markSampler

public void markSampler()

setSkewedJoinPartitionFile

public void setSkewedJoinPartitionFile(String file)

getSkewedJoinPartitionFile

public String getSkewedJoinPartitionFile()

usingTypedComparator

protected boolean usingTypedComparator()

useTypedComparator

protected void useTypedComparator(boolean useTypedComparator)

noCombineSmallSplits

protected void noCombineSmallSplits()

combineSmallSplits

public boolean combineSmallSplits()

isIndexer

public boolean isIndexer()

markIndexer

public void markIndexer()

isUnion

public boolean isUnion()

markUnion

public void markUnion()

isNative

public boolean isNative()

markNative

public void markNative()

isLimit

public boolean isLimit()

markLimit

public void markLimit()

isLimitAfterSort

public boolean isLimitAfterSort()

markLimitAfterSort

public void markLimitAfterSort()

copyFeatures

public void copyFeatures(SparkOperator copyFrom,
                         List<org.apache.pig.backend.hadoop.executionengine.spark.plan.SparkOperator.OPER_FEATURE> excludeFeatures)

isSkewedJoin

public boolean isSkewedJoin()

setRequestedParallelism

public void setRequestedParallelism(int requestedParallelism)

setRequestedParallelismByReference

public void setRequestedParallelismByReference(SparkOperator oper)

addMultiQueryOptimizeConnectionItem

public void addMultiQueryOptimizeConnectionItem(OperatorKey to,
                                                OperatorKey from)

getMultiQueryOptimizeConnectionItem

public MultiMap<OperatorKey,OperatorKey> getMultiQueryOptimizeConnectionItem()

setGlobalSort

public void setGlobalSort(boolean globalSort)

isGlobalSort

public boolean isGlobalSort()