public abstract class AbstractAccumuloStorage extends LoadFunc implements StoreFuncInterface
Modifier and Type | Field and Description |
---|---|
protected static String |
ASTERISK |
protected org.apache.accumulo.core.security.Authorizations |
authorizations |
protected LoadStoreCaster |
caster |
protected static char |
COLON |
protected List<Column> |
columns |
protected String |
columnSeparator |
protected static char |
COMMA |
protected org.apache.commons.cli.CommandLine |
commandLine |
protected String |
contextSignature |
protected String |
end |
protected boolean |
ignoreWhitespace |
protected String |
inst |
protected long |
maxLatency |
protected long |
maxMutationBufferSize |
protected int |
maxWriteThreads |
protected String |
password |
protected ResourceSchema |
schema |
protected String |
start |
protected AccumuloStorageOptions |
storageOptions |
protected String |
table |
protected org.apache.hadoop.io.Text |
tableName |
protected String |
user |
protected String |
zookeepers |
Constructor and Description |
---|
AbstractAccumuloStorage(String columns,
String args) |
Modifier and Type | Method and Description |
---|---|
void |
checkSchema(ResourceSchema s)
Set the schema for data to be stored.
|
void |
cleanupOnFailure(String failure,
org.apache.hadoop.mapreduce.Job job)
This method will be called by Pig if the job which contains this store
fails.
|
void |
cleanupOnSuccess(String location,
org.apache.hadoop.mapreduce.Job job)
This method will be called by Pig if the job which contains this store
is successful, and some cleanup of intermediate resources is required.
|
protected void |
clearUnset(org.apache.hadoop.conf.Configuration conf,
Map<String,String> entriesToUnset)
Replaces the given entries in the configuration by clearing the
Configuration and re-adding the elements that aren't in the Map of
entries to unset
|
protected void |
configureInputFormat(org.apache.hadoop.mapreduce.Job job)
Method to allow specific implementations to add more elements to the Job
for reading data from Accumulo.
|
protected void |
configureOutputFormat(org.apache.hadoop.mapreduce.Job job)
Method to allow specific implementations to add more elements to the Job
for writing data to Accumulo.
|
protected void |
extractArgs(org.apache.commons.cli.CommandLine cli,
AccumuloStorageOptions opts)
Extract arguments passed into the constructor to avoid the URI
|
protected org.apache.commons.cli.CommandLine |
getCommandLine() |
protected Map<String,String> |
getEntries(org.apache.hadoop.conf.Configuration conf,
String prefix)
Extract elements from the Configuration whose keys match the given prefix
|
org.apache.hadoop.mapreduce.InputFormat |
getInputFormat()
This will be called during planning on the front end.
|
protected Map<String,String> |
getInputFormatEntries(org.apache.hadoop.conf.Configuration conf) |
LoadCaster |
getLoadCaster()
This will be called on both the front end and the back
end during execution.
|
protected abstract Collection<org.apache.accumulo.core.data.Mutation> |
getMutations(Tuple tuple) |
Tuple |
getNext()
Retrieves the next tuple to be processed.
|
org.apache.hadoop.mapreduce.OutputFormat |
getOutputFormat()
Return the OutputFormat associated with StoreFuncInterface.
|
protected Map<String,String> |
getOutputFormatEntries(org.apache.hadoop.conf.Configuration conf) |
protected abstract Tuple |
getTuple(org.apache.accumulo.core.data.Key key,
org.apache.accumulo.core.data.Value value) |
protected Properties |
getUDFProperties()
Returns UDFProperties based on
contextSignature . |
protected org.apache.hadoop.mapreduce.RecordWriter<org.apache.hadoop.io.Text,org.apache.accumulo.core.data.Mutation> |
getWriter() |
protected void |
loadDependentJars(org.apache.hadoop.conf.Configuration conf)
Ensure that Accumulo's dependent jars are added to the Configuration to
alleviate the need for clients to REGISTER dependency jars.
|
protected org.apache.accumulo.core.util.Pair<org.apache.hadoop.io.Text,org.apache.hadoop.io.Text> |
makePair(String first,
String second) |
protected org.apache.hadoop.io.Text |
objectToText(Object o,
ResourceSchema.ResourceFieldSchema fieldSchema) |
protected byte[] |
objToBytes(Object o,
byte type) |
protected org.apache.hadoop.io.Text |
objToText(Object o,
byte type) |
void |
prepareToRead(org.apache.hadoop.mapreduce.RecordReader reader,
PigSplit split)
Initializes LoadFunc for reading data.
|
void |
prepareToWrite(org.apache.hadoop.mapreduce.RecordWriter writer)
Initialize StoreFuncInterface to write data.
|
void |
putNext(Tuple tuple)
Write a tuple to the data store.
|
String |
relativeToAbsolutePath(String location,
org.apache.hadoop.fs.Path curDir)
This method is called by the Pig runtime in the front end to convert the
input location to an absolute path if the location is relative.
|
String |
relToAbsPathForStoreLocation(String location,
org.apache.hadoop.fs.Path curDir)
This method is called by the Pig runtime in the front end to convert the
output location to an absolute path if the location is relative.
|
protected byte |
schemaToType(Object o,
int i,
ResourceSchema.ResourceFieldSchema[] fieldSchemas) |
protected byte |
schemaToType(Object o,
ResourceSchema.ResourceFieldSchema fieldSchema) |
void |
setLocation(String location,
org.apache.hadoop.mapreduce.Job job)
Communicate to the loader the location of the object(s) being loaded.
|
void |
setStoreFuncUDFContextSignature(String signature)
This method will be called by Pig both in the front end and back end to
pass a unique signature to the
StoreFuncInterface which it can use to store
information in the UDFContext which it needs to store between
various method invocations in the front end and back end. |
void |
setStoreLocation(String location,
org.apache.hadoop.mapreduce.Job job)
Communicate to the storer the location where the data needs to be stored.
|
void |
setUDFContextSignature(String signature)
This method will be called by Pig both in the front end and back end to
pass a unique signature to the
LoadFunc . |
protected void |
simpleUnset(org.apache.hadoop.conf.Configuration conf,
Map<String,String> entriesToUnset)
Unsets elements in the Configuration using the unset method
|
protected byte[] |
tupleToBytes(Tuple tuple,
int i,
ResourceSchema.ResourceFieldSchema[] fieldSchemas) |
protected org.apache.hadoop.io.Text |
tupleToText(Tuple tuple,
int i,
ResourceSchema.ResourceFieldSchema[] fieldSchemas) |
protected void |
unsetEntriesFromConfiguration(org.apache.hadoop.conf.Configuration conf,
Map<String,String> entriesToUnset)
Removes the given values from the configuration, accounting for changes
in the Configuration API given the version of Hadoop being used.
|
getAbsolutePath, getCacheFiles, getPathStrings, getShipFiles, join, warn
protected static final char COLON
protected static final char COMMA
protected static final String ASTERISK
protected final AccumuloStorageOptions storageOptions
protected final org.apache.commons.cli.CommandLine commandLine
protected String inst
protected String zookeepers
protected String user
protected String password
protected String table
protected org.apache.hadoop.io.Text tableName
protected org.apache.accumulo.core.security.Authorizations authorizations
protected String start
protected String end
protected int maxWriteThreads
protected long maxMutationBufferSize
protected long maxLatency
protected String columnSeparator
protected boolean ignoreWhitespace
protected LoadStoreCaster caster
protected ResourceSchema schema
protected String contextSignature
public AbstractAccumuloStorage(String columns, String args) throws org.apache.commons.cli.ParseException, IOException
org.apache.commons.cli.ParseException
IOException
protected void extractArgs(org.apache.commons.cli.CommandLine cli, AccumuloStorageOptions opts) throws IOException
cli
- opts
- IOException
protected org.apache.commons.cli.CommandLine getCommandLine()
protected Map<String,String> getInputFormatEntries(org.apache.hadoop.conf.Configuration conf)
protected Map<String,String> getOutputFormatEntries(org.apache.hadoop.conf.Configuration conf)
protected void unsetEntriesFromConfiguration(org.apache.hadoop.conf.Configuration conf, Map<String,String> entriesToUnset)
conf
- entriesToUnset
- protected void simpleUnset(org.apache.hadoop.conf.Configuration conf, Map<String,String> entriesToUnset)
conf
- entriesToUnset
- protected void clearUnset(org.apache.hadoop.conf.Configuration conf, Map<String,String> entriesToUnset)
conf
- entriesToUnset
- public Tuple getNext() throws IOException
LoadFunc
getNext
in class LoadFunc
IOException
- if there is an exception while retrieving the next
tupleprotected abstract Tuple getTuple(org.apache.accumulo.core.data.Key key, org.apache.accumulo.core.data.Value value) throws IOException
IOException
public org.apache.hadoop.mapreduce.InputFormat getInputFormat()
LoadFunc
getInputFormat
in class LoadFunc
public void prepareToRead(org.apache.hadoop.mapreduce.RecordReader reader, PigSplit split)
LoadFunc
prepareToRead
in class LoadFunc
reader
- RecordReader
to be used by this instance of the LoadFuncsplit
- The input PigSplit
to processprotected org.apache.hadoop.mapreduce.RecordWriter<org.apache.hadoop.io.Text,org.apache.accumulo.core.data.Mutation> getWriter()
protected Map<String,String> getEntries(org.apache.hadoop.conf.Configuration conf, String prefix)
conf
- prefix
- public void setLocation(String location, org.apache.hadoop.mapreduce.Job job) throws IOException
LoadFunc
LoadFunc.relativeToAbsolutePath(String, Path)
. Implementations
should use this method to communicate the location (and any other information)
to its underlying InputFormat through the Job object.
This method will be called in the frontend and backend multiple times. Implementations
should bear in mind that this method is called multiple times and should
ensure there are no inconsistent side effects due to the multiple calls.setLocation
in class LoadFunc
location
- Location as returned by
LoadFunc.relativeToAbsolutePath(String, Path)
job
- the Job
object
store or retrieve earlier stored information from the UDFContext
IOException
- if the location is not valid.protected org.apache.accumulo.core.util.Pair<org.apache.hadoop.io.Text,org.apache.hadoop.io.Text> makePair(String first, String second)
protected void loadDependentJars(org.apache.hadoop.conf.Configuration conf) throws IOException
job
- The Mapreduce Job objectIOException
protected void configureInputFormat(org.apache.hadoop.mapreduce.Job job)
job
- protected void configureOutputFormat(org.apache.hadoop.mapreduce.Job job)
job
- public String relativeToAbsolutePath(String location, org.apache.hadoop.fs.Path curDir) throws IOException
LoadFunc
relativeToAbsolutePath
in class LoadFunc
location
- location as provided in the "load" statement of the scriptcurDir
- the current working direction based on any "cd" statements
in the script before the "load" statement. If there are no "cd" statements
in the script, this would be the home directory -
/user/
IOException
- if the conversion is not possiblepublic void setUDFContextSignature(String signature)
LoadFunc
LoadFunc
. The signature can be used
to store into the UDFContext
any information which the
LoadFunc
needs to store between various method invocations in the
front end and back end. A use case is to store LoadPushDown.RequiredFieldList
passed to it in LoadPushDown.pushProjection(RequiredFieldList)
for
use in the back end before returning tuples in LoadFunc.getNext()
.
This method will be call before other methods in LoadFunc
setUDFContextSignature
in class LoadFunc
signature
- a unique signature to identify this LoadFuncpublic void setStoreFuncUDFContextSignature(String signature)
StoreFuncInterface
StoreFuncInterface
which it can use to store
information in the UDFContext
which it needs to store between
various method invocations in the front end and back end. This is necessary
because in a Pig Latin script with multiple stores, the different
instances of store functions need to be able to find their (and only their)
data in the UDFContext object.setStoreFuncUDFContextSignature
in interface StoreFuncInterface
signature
- a unique signature to identify this StoreFuncInterfaceprotected Properties getUDFProperties()
contextSignature
.public String relToAbsPathForStoreLocation(String location, org.apache.hadoop.fs.Path curDir) throws IOException
StoreFuncInterface
LoadFunc.getAbsolutePath(java.lang.String, org.apache.hadoop.fs.Path)
provides a default
implementation for hdfs and hadoop local file system and it can be used
to implement this method.relToAbsPathForStoreLocation
in interface StoreFuncInterface
location
- location as provided in the "store" statement of the scriptcurDir
- the current working direction based on any "cd" statements
in the script before the "store" statement. If there are no "cd" statements
in the script, this would be the home directory -
/user/
IOException
- if the conversion is not possiblepublic void setStoreLocation(String location, org.apache.hadoop.mapreduce.Job job) throws IOException
StoreFuncInterface
StoreFuncInterface
here is the
return value of StoreFuncInterface.relToAbsPathForStoreLocation(String, Path)
This method will be called in the frontend and backend multiple times. Implementations
should bear in mind that this method is called multiple times and should
ensure there are no inconsistent side effects due to the multiple calls.
StoreFuncInterface.checkSchema(ResourceSchema)
will be called before any call to
StoreFuncInterface.setStoreLocation(String, Job)
.setStoreLocation
in interface StoreFuncInterface
location
- Location returned by
StoreFuncInterface.relToAbsPathForStoreLocation(String, Path)
job
- The Job
objectIOException
- if the location is not valid.public org.apache.hadoop.mapreduce.OutputFormat getOutputFormat()
StoreFuncInterface
getOutputFormat
in interface StoreFuncInterface
OutputFormat
associated with StoreFuncInterfacepublic void prepareToWrite(org.apache.hadoop.mapreduce.RecordWriter writer)
StoreFuncInterface
prepareToWrite
in interface StoreFuncInterface
writer
- RecordWriter to use.protected abstract Collection<org.apache.accumulo.core.data.Mutation> getMutations(Tuple tuple) throws ExecException, IOException
ExecException
IOException
public void putNext(Tuple tuple) throws ExecException, IOException
StoreFuncInterface
putNext
in interface StoreFuncInterface
tuple
- the tuple to store.IOException
- if an exception occurs during the writeExecException
public void cleanupOnFailure(String failure, org.apache.hadoop.mapreduce.Job job)
StoreFuncInterface
cleanupOnFailure
in interface StoreFuncInterface
failure
- Location returned by
StoreFuncInterface.relToAbsPathForStoreLocation(String, Path)
job
- The Job
object - this should be used only to obtain
cluster properties through JobContextImpl.getConfiguration()
and not to set/query
any runtime job information.public void cleanupOnSuccess(String location, org.apache.hadoop.mapreduce.Job job)
StoreFuncInterface
cleanupOnSuccess
in interface StoreFuncInterface
location
- Location returned by
StoreFuncInterface.relToAbsPathForStoreLocation(String, Path)
job
- The Job
object - this should be used only to obtain
cluster properties through JobContextImpl.getConfiguration()
and not to set/query
any runtime job information.public void checkSchema(ResourceSchema s) throws IOException
StoreFuncInterface
checkSchema
in interface StoreFuncInterface
s
- to be checkedIOException
- if this schema is not acceptable. It should include
a detailed error message indicating what is wrong with the schema.protected org.apache.hadoop.io.Text tupleToText(Tuple tuple, int i, ResourceSchema.ResourceFieldSchema[] fieldSchemas) throws IOException
IOException
protected org.apache.hadoop.io.Text objectToText(Object o, ResourceSchema.ResourceFieldSchema fieldSchema) throws IOException
IOException
protected byte schemaToType(Object o, ResourceSchema.ResourceFieldSchema fieldSchema)
protected byte schemaToType(Object o, int i, ResourceSchema.ResourceFieldSchema[] fieldSchemas)
protected byte[] tupleToBytes(Tuple tuple, int i, ResourceSchema.ResourceFieldSchema[] fieldSchemas) throws IOException
IOException
protected org.apache.hadoop.io.Text objToText(Object o, byte type) throws IOException
IOException
protected byte[] objToBytes(Object o, byte type) throws IOException
IOException
public LoadCaster getLoadCaster() throws IOException
LoadFunc
getLoadCaster
in class LoadFunc
LoadCaster
associated with this loader. Returning null
indicates that casts from byte array are not supported for this loader.
constructionIOException
- if there is an exception during LoadCasterCopyright © 2007-2017 The Apache Software Foundation