org.apache.pig.backend.hadoop.executionengine.spark.plan

Class SparkCompiler

java.lang.Object

org.apache.pig.impl.plan.PlanVisitor<PhysicalOperator,PhysicalPlan>

org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor

org.apache.pig.backend.hadoop.executionengine.spark.plan.SparkCompiler

public class SparkCompiler
extends PhyPlanVisitor

The compiler that compiles a given physical physicalPlan into a DAG of Spark operators

Field Summary

Fields inherited from class org.apache.pig.impl.plan.PlanVisitor

mCurrentWalker, mPlan

Constructor Summary

Constructors

Constructor and Description
SparkCompiler(PhysicalPlan physicalPlan, PigContext pigContext)

Method Summary

Modifier and Type	Method and Description
void	compile()
void	connectSoftLink()
SparkOperPlan	getSparkPlan()
void	visitCollectedGroup(POCollectedGroup op)
void	visitCounter(POCounter op)
void	visitCross(POCross op)
void	visitDistinct(PODistinct op)
void	visitFilter(POFilter op)
void	visitFRJoin(POFRJoin op)
void	visitGlobalRearrange(POGlobalRearrange op)
void	visitLimit(POLimit op)
void	visitLoad(POLoad op)
void	visitLocalRearrange(POLocalRearrange op)
void	visitMergeCoGroup(POMergeCogroup poCoGrp)
void	visitMergeJoin(POMergeJoin joinOp)
void	visitNative(PONative op)
void	visitPackage(POPackage op)
void	visitPOForEach(POForEach op)
void	visitRank(PORank op)
void	visitSkewedJoin(POSkewedJoin op) currently use regular join to replace skewedJoin Skewed join currently works with two-table inner join.
void	visitSort(POSort op)
void	visitSplit(POSplit op)
void	visitStore(POStore op)
void	visitStream(POStream op)
void	visitUnion(POUnion op)

Methods inherited from class org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor

visit, visitAdd, visitAnd, visitBinCond, visitBroadcastSpark, visitCast, visitComparisonFunc, visitConstant, visitDemux, visitDivide, visitEqualTo, visitGreaterThan, visitGTOrEqual, visitIsNull, visitLessThan, visitLTOrEqual, visitMapLookUp, visitMod, visitMultiply, visitNegative, visitNot, visitNotEqualTo, visitOr, visitPartialAgg, visitPartitionRearrange, visitPoissonSample, visitPOOptimizedForEach, visitPreCombinerLocalRearrange, visitProject, visitRegexp, visitReservoirSample, visitSubtract, visitUserFunc

Methods inherited from class org.apache.pig.impl.plan.PlanVisitor

getPlan, popWalker, pushWalker, visit

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

SparkCompiler

public SparkCompiler(PhysicalPlan physicalPlan,
                     PigContext pigContext)

Method Detail

compile

public void compile()
             throws IOException,
                    PlanException,
                    VisitorException

Throws:

IOException

PlanException

VisitorException

getSparkPlan

public SparkOperPlan getSparkPlan()

connectSoftLink

public void connectSoftLink()
                     throws PlanException,
                            IOException

Throws:

PlanException

IOException

visitSplit

public void visitSplit(POSplit op)
                throws VisitorException

Overrides:

visitSplit in class PhyPlanVisitor

Throws:

VisitorException

visitDistinct

public void visitDistinct(PODistinct op)
                   throws VisitorException

Overrides:

visitDistinct in class PhyPlanVisitor

Throws:

VisitorException

visitLoad

public void visitLoad(POLoad op)
               throws VisitorException

Overrides:

visitLoad in class PhyPlanVisitor

Throws:

VisitorException

visitNative

public void visitNative(PONative op)
                 throws VisitorException

Overrides:

visitNative in class PhyPlanVisitor

Throws:

VisitorException

visitStore

public void visitStore(POStore op)
                throws VisitorException

Overrides:

visitStore in class PhyPlanVisitor

Throws:

VisitorException

visitFilter

public void visitFilter(POFilter op)
                 throws VisitorException

Overrides:

visitFilter in class PhyPlanVisitor

Throws:

VisitorException

visitCross

public void visitCross(POCross op)
                throws VisitorException

Overrides:

visitCross in class PhyPlanVisitor

Throws:

VisitorException

visitStream

public void visitStream(POStream op)
                 throws VisitorException

Overrides:

visitStream in class PhyPlanVisitor

Throws:

VisitorException

visitSort

public void visitSort(POSort op)
               throws VisitorException

Overrides:

visitSort in class PhyPlanVisitor

Throws:

VisitorException

visitLimit

public void visitLimit(POLimit op)
                throws VisitorException

Overrides:

visitLimit in class PhyPlanVisitor

Throws:

VisitorException

visitLocalRearrange

public void visitLocalRearrange(POLocalRearrange op)
                         throws VisitorException

Overrides:

visitLocalRearrange in class PhyPlanVisitor

Throws:

VisitorException

visitCollectedGroup

public void visitCollectedGroup(POCollectedGroup op)
                         throws VisitorException

Overrides:

visitCollectedGroup in class PhyPlanVisitor

Throws:

VisitorException

visitPOForEach

public void visitPOForEach(POForEach op)
                    throws VisitorException

Overrides:

visitPOForEach in class PhyPlanVisitor

Throws:

VisitorException

visitCounter

public void visitCounter(POCounter op)
                  throws VisitorException

Overrides:

visitCounter in class PhyPlanVisitor

Throws:

VisitorException

visitRank

public void visitRank(PORank op)
               throws VisitorException

Overrides:

visitRank in class PhyPlanVisitor

Throws:

VisitorException

visitGlobalRearrange

public void visitGlobalRearrange(POGlobalRearrange op)
                          throws VisitorException

Overrides:

visitGlobalRearrange in class PhyPlanVisitor

Throws:

VisitorException

visitPackage

public void visitPackage(POPackage op)
                  throws VisitorException

Overrides:

visitPackage in class PhyPlanVisitor

Throws:

VisitorException

visitUnion

public void visitUnion(POUnion op)
                throws VisitorException

Overrides:

visitUnion in class PhyPlanVisitor

Throws:

VisitorException

visitSkewedJoin

public void visitSkewedJoin(POSkewedJoin op)
                     throws VisitorException

currently use regular join to replace skewedJoin Skewed join currently works with two-table inner join. More info about pig SkewedJoin, See https://wiki.apache.org/pig/PigSkewedJoinSpec

Overrides:

visitSkewedJoin in class PhyPlanVisitor

Parameters:

op -

Throws:

VisitorException

visitFRJoin

public void visitFRJoin(POFRJoin op)
                 throws VisitorException

Overrides:

visitFRJoin in class PhyPlanVisitor

Throws:

VisitorException

visitMergeJoin

public void visitMergeJoin(POMergeJoin joinOp)
                    throws VisitorException

Overrides:

visitMergeJoin in class PhyPlanVisitor

Throws:

VisitorException

visitMergeCoGroup

public void visitMergeCoGroup(POMergeCogroup poCoGrp)
                       throws VisitorException

Overrides:

visitMergeCoGroup in class PhyPlanVisitor

Throws:

VisitorException