org.apache.pig.impl

Class PigContext

java.lang.Object

org.apache.pig.impl.PigContext

All Implemented Interfaces:

Serializable

public class PigContext
extends Object
implements Serializable

See Also:

Serialized Form

Field Summary

Fields

Modifier and Type	Field and Description
int	defaultParallel
List<URL>	extraJars
boolean	inDumpSchema
boolean	inExplain
boolean	inIllustrator
static String	JOB_NAME
static String	JOB_NAME_PREFIX
static String	JOB_PRIORITY
Map<String,org.antlr.runtime.tree.Tree>	macros
static String	PIG_CMD_ARGS_REMAINDERS
Vector<String>	predeployedJars
List<String>	scriptFiles Deprecated.
Map<String,String>	scriptingUDFs
List<String>	scriptJars

Constructor Summary

Constructors

Constructor and Description
PigContext()
PigContext(org.apache.hadoop.conf.Configuration conf)
PigContext(ExecType execType, org.apache.hadoop.conf.Configuration conf)
PigContext(ExecType execType, Properties properties)
PigContext(Properties properties)

Method Summary

Modifier and Type	Method and Description
void	addJar(String path)
void	addJar(URL resource, String originalPath)
void	addPathToSkip(String path) Add a path to be skipped while automatically shipping binaries for streaming.
void	addScriptFile(String path) calls: addScriptFile(path, new File(path)), ensuring that a given path is added to the jar at most once.
void	addScriptFile(String name, String path) this method adds script files that must be added to the shipped jar named differently from their local fs path.
void	addScriptJar(String path)
void	connect()
void	copy(String src, String dst, boolean localDst)
ClassLoader	createCl(String jarFile) Creates a Classloader based on the passed jarFile and any extra jar files.
ExecutableManager	createExecutableManager() Create a new ExecutableManager depending on the ExecType.
String	doParamSubstitution(BufferedReader reader)
String	doParamSubstitution(BufferedReader reader, List<String> params, List<String> paramFiles)
String	doParamSubstitution(InputStream in, List<String> params, List<String> paramFiles)
BufferedReader	doParamSubstitutionOutputToFile(BufferedReader reader, String outputFilePath)
BufferedReader	doParamSubstitutionOutputToFile(BufferedReader reader, String outputFilePath, List<String> params, List<String> paramFiles)
Class	getClassForAlias(String alias)
static ClassLoader	getClassLoader()
StreamingCommand	getCommandForAlias(String alias) Get the StreamingCommand for the given alias.
Properties	getConf() Deprecated. use getProperties() instead
org.apache.log4j.Level	getDefaultLogLevel()
int	getDefaultParallel()
DataStorage	getDfs()
byte	getErrorSource() Check the execution mode and return the appropriate error source
ExecType	getExecType() Returns the type of execution currently in effect.
ExecutionEngine	getExecutionEngine()
DataStorage	getFs()
FuncSpec	getFuncSpecFromAlias(String alias)
String	getLastAlias()
DataStorage	getLfs()
Properties	getLog4jProperties()
static ArrayList<String>	getPackageImportList()
List<String>	getParamFiles()
List<String>	getParams()
Map<String,String>	getParamVal()
List<String>	getPathsToSkip() Get paths which are to skipped while automatically shipping binaries for streaming.
PreprocessorContext	getPreprocessorContext()
Properties	getProperties() Provides configuration information.
Map<String,File>	getScriptFiles() script files as name/file pairs to be added to the job jar
boolean	hasJar(String path)
static void	initializeImportList(String importListCommandLineProperties)
Object	instantiateFuncFromAlias(String alias)
static Object	instantiateFuncFromSpec(FuncSpec funcSpec)
static Object	instantiateFuncFromSpec(String funcSpec)
static <T> T	instantiateObjectFromParams(org.apache.hadoop.conf.Configuration conf, String classParamKey, String argParamKey, Class<T> clazz) A common Pig pattern for initializing objects via system properties is to support passing something like this on the command line: -Dpig.notification.listener=MyClass -Dpig.notification.listener.arg=myConstructorStringArg This method will properly initialize the class with the args, if they exist.
void	markJarAsPredeployed(String path) Adds the specified path to the predeployed jars list.
void	registerFunction(String function, FuncSpec functionSpec) Defines an alias for the given function spec.
void	registerStreamCmd(String alias, StreamingCommand command) Defines an alias for the given streaming command.
void	rename(String oldName, String newName)
static Class	resolveClassName(String name)
static void	setClassLoader(ClassLoader cl)
void	setDefaultLogLevel(org.apache.log4j.Level l)
void	setExecType(ExecType execType)
void	setJobtrackerLocation(String newLocation)
void	setLastAlias(String value)
void	setLog4jProperties(Properties p)
static void	setPackageImportList(ArrayList<String> list)
void	setParamFiles(List<String> paramFiles)
void	setParams(List<String> params)
static void	staticDataCleanup()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

JOB_NAME

public static final String JOB_NAME

See Also:

Constant Field Values

JOB_NAME_PREFIX

public static final String JOB_NAME_PREFIX

See Also:

Constant Field Values

JOB_PRIORITY

public static final String JOB_PRIORITY

See Also:

Constant Field Values

PIG_CMD_ARGS_REMAINDERS

public static final String PIG_CMD_ARGS_REMAINDERS

See Also:

Constant Field Values

extraJars

public transient List<URL> extraJars

scriptJars

public transient List<String> scriptJars

predeployedJars

public transient Vector<String> predeployedJars

scriptFiles

@Deprecated
public List<String> scriptFiles

Deprecated.

scriptingUDFs

public transient Map<String,String> scriptingUDFs

macros

public transient Map<String,org.antlr.runtime.tree.Tree> macros

defaultParallel

public int defaultParallel

inExplain

public boolean inExplain

inDumpSchema

public boolean inDumpSchema

inIllustrator

public boolean inIllustrator

Constructor Detail

PigContext

public PigContext()

PigContext

public PigContext(org.apache.hadoop.conf.Configuration conf)
           throws PigException

Throws:

PigException

PigContext

public PigContext(Properties properties)
           throws PigException

Throws:

PigException

PigContext

public PigContext(ExecType execType,
                  org.apache.hadoop.conf.Configuration conf)

PigContext

public PigContext(ExecType execType,
                  Properties properties)

Method Detail

staticDataCleanup

public static void staticDataCleanup()

getParams

public List<String> getParams()

setParams

public void setParams(List<String> params)

getParamFiles

public List<String> getParamFiles()

setParamFiles

public void setParamFiles(List<String> paramFiles)

getPreprocessorContext

public PreprocessorContext getPreprocessorContext()

getParamVal

public Map<String,String> getParamVal()
                               throws IOException

Throws:

IOException

initializeImportList

public static void initializeImportList(String importListCommandLineProperties)

connect

public void connect()
             throws ExecException

Throws:

ExecException

setJobtrackerLocation

public void setJobtrackerLocation(String newLocation)

addScriptFile

public void addScriptFile(String path)

calls: addScriptFile(path, new File(path)), ensuring that a given path is added to the jar at most once.

Parameters:

path -

addScriptFile

public void addScriptFile(String name,
                          String path)

this method adds script files that must be added to the shipped jar named differently from their local fs path.

Parameters:

name - name in the jar

path - path on the local fs

addScriptJar

public void addScriptJar(String path)

addJar

public void addJar(String path)
            throws MalformedURLException

Throws:

MalformedURLException

addJar

public void addJar(URL resource,
                   String originalPath)
            throws MalformedURLException

Throws:

MalformedURLException

hasJar

public boolean hasJar(String path)

markJarAsPredeployed

public void markJarAsPredeployed(String path)

Adds the specified path to the predeployed jars list. These jars will never be included in generated job jar.

This can be called for jars that are pre-installed on the Hadoop cluster to reduce the size of the job jar.

doParamSubstitution

public String doParamSubstitution(InputStream in,
                                  List<String> params,
                                  List<String> paramFiles)
                           throws IOException

Throws:

IOException

doParamSubstitution

public String doParamSubstitution(BufferedReader reader,
                                  List<String> params,
                                  List<String> paramFiles)
                           throws IOException

Throws:

IOException

doParamSubstitution

public String doParamSubstitution(BufferedReader reader)
                           throws IOException

Throws:

IOException

doParamSubstitutionOutputToFile

public BufferedReader doParamSubstitutionOutputToFile(BufferedReader reader,
                                                      String outputFilePath,
                                                      List<String> params,
                                                      List<String> paramFiles)
                                               throws IOException

Throws:

IOException

doParamSubstitutionOutputToFile

public BufferedReader doParamSubstitutionOutputToFile(BufferedReader reader,
                                                      String outputFilePath)
                                               throws IOException

Throws:

IOException

getScriptFiles

public Map<String,File> getScriptFiles()

script files as name/file pairs to be added to the job jar

Returns:

name/file pairs

rename

public void rename(String oldName,
                   String newName)
            throws IOException

Throws:

IOException

copy

public void copy(String src,
                 String dst,
                 boolean localDst)
          throws IOException

Throws:

IOException

getExecutionEngine

public ExecutionEngine getExecutionEngine()

getDfs

public DataStorage getDfs()

getLfs

public DataStorage getLfs()

getFs

public DataStorage getFs()

getProperties

public Properties getProperties()

Provides configuration information.

Returns:

- information about the configuration used to connect to execution engine

getConf

@Deprecated
public Properties getConf()

Deprecated. use getProperties() instead

getLastAlias

public String getLastAlias()

setLastAlias

public void setLastAlias(String value)

registerFunction

public void registerFunction(String function,
                             FuncSpec functionSpec)

Defines an alias for the given function spec. This is useful for functions that require arguments to the constructor.

Parameters:

function - - the new function alias to define.

functionSpec - - the FuncSpec object representing the name of the function class and any arguments to constructor.

registerStreamCmd

public void registerStreamCmd(String alias,
                              StreamingCommand command)

Defines an alias for the given streaming command. This is useful for complicated streaming command specs.

Parameters:

alias - - the new command alias to define.

command - - the command

getExecType

public ExecType getExecType()

Returns the type of execution currently in effect.

Returns:

current execution type

createCl

public ClassLoader createCl(String jarFile)
                     throws MalformedURLException

Creates a Classloader based on the passed jarFile and any extra jar files.

Parameters:

jarFile - the jar file to be part of the newly created Classloader. This jar file plus any jars in the extraJars list will constitute the classpath.

Returns:

the new Classloader.

Throws:

MalformedURLException

resolveClassName

public static Class resolveClassName(String name)
                              throws IOException

Throws:

IOException

instantiateObjectFromParams

public static <T> T instantiateObjectFromParams(org.apache.hadoop.conf.Configuration conf,
                                                String classParamKey,
                                                String argParamKey,
                                                Class<T> clazz)
                                         throws ExecException

A common Pig pattern for initializing objects via system properties is to support passing something like this on the command line: -Dpig.notification.listener=MyClass -Dpig.notification.listener.arg=myConstructorStringArg This method will properly initialize the class with the args, if they exist.

Parameters:

conf -

classParamKey - the property used to identify the class

argParamKey - the property used to identify the class args

clazz - The class that is expected

Returns:

T

Throws:

ExecException

instantiateFuncFromSpec

public static Object instantiateFuncFromSpec(FuncSpec funcSpec)

instantiateFuncFromSpec

public static Object instantiateFuncFromSpec(String funcSpec)

getClassForAlias

public Class getClassForAlias(String alias)
                       throws IOException

Throws:

IOException

instantiateFuncFromAlias

public Object instantiateFuncFromAlias(String alias)
                                throws IOException

Throws:

IOException

getCommandForAlias

public StreamingCommand getCommandForAlias(String alias)

Get the StreamingCommand for the given alias.

Parameters:

alias - the alias for the StreamingCommand

Returns:

StreamingCommand for the alias

setExecType

public void setExecType(ExecType execType)

createExecutableManager

public ExecutableManager createExecutableManager()
                                          throws ExecException

Create a new ExecutableManager depending on the ExecType.

Returns:

a new ExecutableManager depending on the ExecType

Throws:

ExecException

getFuncSpecFromAlias

public FuncSpec getFuncSpecFromAlias(String alias)

addPathToSkip

public void addPathToSkip(String path)

Add a path to be skipped while automatically shipping binaries for streaming.

Parameters:

path - path to be skipped

getPathsToSkip

public List<String> getPathsToSkip()

Get paths which are to skipped while automatically shipping binaries for streaming.

Returns:

paths which are to skipped while automatically shipping binaries for streaming

getErrorSource

public byte getErrorSource()

Check the execution mode and return the appropriate error source

Returns:

error source

getPackageImportList

public static ArrayList<String> getPackageImportList()

setPackageImportList

public static void setPackageImportList(ArrayList<String> list)

setLog4jProperties

public void setLog4jProperties(Properties p)

getLog4jProperties

public Properties getLog4jProperties()

getDefaultLogLevel

public org.apache.log4j.Level getDefaultLogLevel()

setDefaultLogLevel

public void setDefaultLogLevel(org.apache.log4j.Level l)

getDefaultParallel

public int getDefaultParallel()

getClassLoader

public static ClassLoader getClassLoader()

setClassLoader

public static void setClassLoader(ClassLoader cl)