# $SnapHashLicense:
#
# SnapLogic - Open source data services
#
# Copyright (C) 2009, SnapLogic, Inc. All rights reserved.
#
# See http://www.snaplogic.org for more information about
# the SnapLogic project.
#
# This program is free software, distributed under the terms of
# the GNU General Public License Version 2. See the LEGAL file
# at the top of the source tree.
#
# "SnapLogic" is a trademark of SnapLogic, Inc.
#
#
# $
# $Id: component_api.py 10183 2009-12-16 23:58:55Z grisha $
"""
This module provides a Python based interface for creating a Component.
Developers of components on the Python based Component Container must
derive from the ComponentAPI class and override some of its methods
to implement functionality thzat is specific to their component.
"""
import urlparse
import re
from snaplogic.common.snap_exceptions import *
from snaplogic import cc
import snaplogic.snapi_base.keys as keys
import snaplogic.cc.prop as prop
from snaplogic.snapi_base.resdef import validate_limits,UNLIMITED
from snaplogic.snapi_base import resdef
from snaplogic.common.snap_exceptions import *
from snaplogic.common import snapstream,headers
from snaplogic.common import prop_err
from snaplogic.common import component_prop_defs
from snaplogic import snapi
def has_param(value):
"""
Returns True if the the property value specified has an unsubstituted parameter,
@param value: Property value.
@type value: any supported property value.
@return: True, If it has an unsubstituted param, False otherwise.
@rtype: bool
"""
return resdef_module.has_param(value)
class ComponentAPI(object):
"""
This class defines the Component API.
Component developers creating new Python based components must derive from
this class. In the derived class, the execute() method must be overriden
to implement the data processing logic of the component. Apart from this key
method, the developer can also override the following methods to make their
component easier to use.
create_resource_template(): This method can be implemented to provide clients
that are creating resource definitions with a template definition which defines
properties, views, parmeters and other values that must always be present in the
resource definition for that component.
suggest_resource_values(): This method can be implemented to provide clients
that are creating resource definitions with a helper function which fills in
suggested values for the resource definition, based on the resource values already
entered by the client.
validate(): This method can be implemented to allow the component to validate
properties, views and other values that are very specific to that component.
validate_config(): This method can be implemented to validate the contents of the
configuration file (if any) for the component.
"""
CAPABILITY_INPUT_VIEW_LOWER_LIMIT = keys.CAPABILITY_INPUT_VIEW_LOWER_LIMIT
CAPABILITY_INPUT_VIEW_UPPER_LIMIT = keys.CAPABILITY_INPUT_VIEW_UPPER_LIMIT
CAPABILITY_INPUT_VIEW_ALLOW_BINARY = keys.CAPABILITY_INPUT_VIEW_ALLOW_BINARY
CAPABILITY_OUTPUT_VIEW_LOWER_LIMIT = keys.CAPABILITY_OUTPUT_VIEW_LOWER_LIMIT
CAPABILITY_OUTPUT_VIEW_UPPER_LIMIT = keys.CAPABILITY_OUTPUT_VIEW_UPPER_LIMIT
CAPABILITY_OUTPUT_VIEW_ALLOW_BINARY = keys.CAPABILITY_OUTPUT_VIEW_ALLOW_BINARY
CAPABILITY_ALLOW_PASS_THROUGH = keys.CAPABILITY_ALLOW_PASS_THROUGH
_cap_keys = [
CAPABILITY_INPUT_VIEW_LOWER_LIMIT,
CAPABILITY_INPUT_VIEW_UPPER_LIMIT,
CAPABILITY_INPUT_VIEW_ALLOW_BINARY,
CAPABILITY_OUTPUT_VIEW_LOWER_LIMIT,
CAPABILITY_OUTPUT_VIEW_UPPER_LIMIT,
CAPABILITY_OUTPUT_VIEW_ALLOW_BINARY,
CAPABILITY_ALLOW_PASS_THROUGH
]
UNLIMITED_VIEWS = UNLIMITED
def __init__(self):
"""
Do nothing.
We really don't do much in the init method, as a component derived from it may neglect to call
the parent class init.
"""
pass
def _init_comp(self, config_dict, resdef, resource_name=None, rid=None, resdef_authoring_phase=True,
invoker=None, resource_ref=None):
"""
This method is for the internal use of CC. It initializes the component object.
@param comp_name: Name of the component class.
@type comp_name: str
@param config_dict: The information from the config of this component (if any).
@type config_dict: dict
@param resdef: The resource definition for this component.
@type resdef: L{ResDef}
@param resource_name: Name given to the component resource in a pipeline.
@type resource_name: str
@param rid: Runtime id of the resource, for this particular invocation of component.
@type rid: str
@param resdef_authoring_phase: If set to True, the component is being created for resdef authoring purposes
and not execution.
@type resdef_authoring_phase: bool
@param invoker: Username provided in the HTTP header named 'invoker'.
@type invoker: str
@param resource_ref: The reference dict provided by pipeline manager at runtime. The map is keyed by
reference name. The value is an object that contains the reference value (a resource URI) and
params that should be substituted into the resdef fetched from that URI.
@type resource_ref: dict
"""
# Make a copy of cc config section to make it available to this component
self.env = cc.config.get_section('cc').copy()
self.config_dictionary = config_dict
"""Configuration dictionary for the component (if any)."""
self.got_stopped = False
"""This flag is set to True if the component has been asked to stop executing and exit."""
# Resource definition of this component.
self._resdef = resdef
# Property definition is only allowed from some resdef authoring functions like define_resource_template()
# and suggest_resource_values()
self._allow_definition_changes = resdef_authoring_phase
self._invoker_username = invoker
# This dictionary maps input stream to its input view object. This mapping is primarily
# used for quick access to view object from the stream. This is primarily used by the
# select() function.
self._stream_to_input_view = {}
# The params passed to the component.
self.parameters = {}
# Dictionary of resource references.
self.resource_ref = {}
self.log, self.elog, ignore_alog = cc.logger.make_specific_loggers(resdef.get_component_name(), rid,
resource_name, self._invoker_username)
# Absorb the capabilties set by the component author.
if self._resdef is not None:
if hasattr(self, "capabilities"):
if not isinstance(self.capabilities, dict):
raise SnapObjTypeError("The Component attribute \"capabilities\" must be a dict")
for k in self.capabilities:
if k not in ComponentAPI._cap_keys:
raise SnapObjNotFoundError("\"%s\" is not a valid capability name" % k)
resdef.dict[keys.CAPABILITIES][k] = self.capabilities[k]
validate_limits(resdef.dict[keys.CAPABILITIES][self.CAPABILITY_INPUT_VIEW_LOWER_LIMIT],
resdef.dict[keys.CAPABILITIES][self.CAPABILITY_INPUT_VIEW_UPPER_LIMIT])
validate_limits(resdef.dict[keys.CAPABILITIES][self.CAPABILITY_OUTPUT_VIEW_LOWER_LIMIT],
resdef.dict[keys.CAPABILITIES][self.CAPABILITY_OUTPUT_VIEW_UPPER_LIMIT])
self.__process_resource_ref(resource_ref)
def __process_resource_ref(self, resource_ref):
"""
Sets up referred resdefs for the component to access.
This method processes the runtime information provided by the pipeline manager,
which includes the URI of the resource being referred to and any parameters
that need to be substituted into the resdef of that resource. With the URI
provided, this method fetches the resdef and then if there are params to be
substituted, it substitutes those params. Finally, it creates a dict, with
reference name as key and the resdef as value.
@param resource_ref: The runtime information provided by the pipeline manager.
@type resource_ref: dict
"""
if resource_ref is None:
resource_ref = {}
for ref_name in self._resdef.list_resource_ref_names():
if ref_name in resource_ref:
uri = resource_ref[ref_name][keys.URI]
params = resource_ref[ref_name][keys.PARAMS]
else:
uri = self._resdef.get_resource_ref(ref_name)
params = None
if uri:
ref_resdef = self.get_resource_object(uri)
if params:
prop.substitute_params_in_resdef(ref_resdef, params)
self.resource_ref[ref_name] = ref_resdef
else:
self.resource_ref[ref_name] = None
def select_input_view(self, input_view_list, timeout = None):
"""
Wait on one or more input views and return list of views that have data to be consumed.
For record mode views, the view is returned, if it has a record available for reading.
For binary mode views, the view is returned if it has data of the size specified by
binary block size.
@param input_view_list: The list of input view objects to be selected on.
@type input_view_list: List of L{InputView}
@param timeout: The timeout (in seconds) for the wait. This defaults to infinite.
@type timeout: int
@return: List of input views with data.
@rtype: List of L{InputView}
"""
ret_streams = snapstream.select_streams([inp.snap_stream for inp in input_view_list], timeout)
return [self._stream_to_input_view[s] for s in ret_streams]
def process_input_views(self, method_dict, start_with = None):
"""
Register processing methods with input views and block until all data on those views are processed.
This is a utility method which can be used to wait on one or more of the input views and
have user defined methods invoked when data is available on those views. This utility method
takes a dictionary, where the key is the input view object and the value is the call-back method
that must be called when data is available on that input view. The function specified must have
the signature:
foo_bar_processing_method(data, input_view_object, list_of_active_input_views)
Where the three params will be as follows:
data - the data read from the input view.
input_view_object - the input view object from which the data was read.
list_of_active_input_views - the subset of the inputs views initially specified in the dictionary,
that still haven't finished providing data.
If this call-back function is a member of the component class, then it can have "self" in its
signature:
foo_bar_processing_method(self, data, input_view_object, list_of_active_input_views)
The call-back method is not required to return anything. Optionally, it can return a subset of
the input views specified in list_of_active_input_views. This tells the utility method that
the next data must be read from this subset of views. This is also the main reason why we provide
the list_of_active_input_views param: so that the call-back author can choose a subset from it.
@param method_dict: The key of the dictionary is the input view object and value is the
method that should process the data from that view.
@type method_dict: dict
@param start_with: This can be set to list of one or more input view objects that must be read in the
first iteration.
@type start_with: list
"""
# Make a tuple out of this list, so component author cannot modify it.
open_input_views = tuple([inp_view for inp_view in method_dict])
if start_with is not None:
test_list = start_with
else:
test_list = open_input_views
while(len(test_list)):
selected_list = self.select_input_view(test_list)
inp = selected_list[0]
if inp.is_binary:
data = inp.read_binary()
else:
data = inp.read_record()
if data is None:
# The input view is done. Recreate the open input views tuple, without this view.
open_input_views = tuple([i for i in open_input_views if i != inp] )
inp._close()
ret_list = method_dict[inp](data, inp, open_input_views)
if ret_list:
# The user has provided a preference
test_list = ret_list
else:
test_list = open_input_views
field_ref_re = re.compile('\$[{]([\w]+)[}]')
"""For finding input field name references in the expressions in the form ${Field001}"""
def get_referenced_fields(self, expr, uniq_and_sorted=False):
"""
Retrieve field names referenced (via L{field_ref_re}) in the provided expression.
If uniq_and_sorted is specified as True, the
field names in retrieved list will be unique and alphanumerically
sorted (that is, given "${one} ${two} ${one}", ['one', 'two'] will be returned).
@param expr: Expression from which to retrieve references to field names
@type expr: string
@param uniq_and_sorted: if True, the result will consist only of unique names, sorted in
alphanumeric order. If False, the result will be the names
in order of appearance, multiple times if they appear so in expr.
@type uniq_and_sorted: bool
@return: list of field
@rtype: list
"""
retval = self.field_ref_re.findall(expr)
if uniq_and_sorted:
# Make them unique
retval = set(retval)
# Make the list again so we can sort
retval = list(retval)
retval.sort()
return retval
def replace_referenced_fields(self, expr, replacements):
"""
Returns expr where referenced view field names (via L{field_ref_re})
are replaced with the values from the replacement dictionary (where the
keys are the names). For example, if expr provided is "${foo} ${bar}",
and the replacements is { 'foo' : 'John', 'bar' : 'Smith' }, "John Smith"
will be returned. If a value for the referenced field in not specified
in the replacements dictionary, this field is not substituted.
@param expr: String which contains references to view fields
@type expr: string
@param replacements: What to replace (keyed by field name)
@type replacements: dict or callable
@return: The original string with referenced fields replaced.
@rtype: string
"""
def repl_callable(match):
field_name = match.group(1)
if replacements.has_key(field_name):
repl_str = replacements[field_name]
else:
repl_str = "${%s}" % field_name
return repl_str
result = ComponentAPI.field_ref_re.sub(repl_callable, expr)
return result
def add_record_input_view_def(self, name, fields, description_str, modifiable = True):
"""
Creates an input view in the record mode. This kind of view reads data in a record format.
The view defines the fields that the record is expected to have. The view may support
a feature called pass_through (if the pass_through param is set to True). This feature
allows the view to accept fields in the input record that have not been formally defined
in the view.
@param name: Input view name.
@type name: str
@param fields: A sequence of fields, where each field is defined by the sequence
(field name, field type, field description string). The field name must be a
unique string name in the context of that view. The field type is also a string
which specifies one of the supported data types for that field. The field
description is a string which describes the field.
@type fields: sequence
@param description_str: Description for the entire view.
@type description_str: str
@param modifiable: If set to False, then only the server and component can modify this view and not
the client. The default value is True (meaning it is modifiable by client).
@type modifiable: bool
"""
if not self._allow_definition_changes:
raise SnapObjPermissionError("Resource definition changes is not permitted at this stage")
self._resdef.add_record_input_view(name, fields, description_str, modifiable)
def add_binary_input_view_def(self, name, view_content_types, description_str, modifiable = True):
"""
This creates an input view, which reads data in as a binary stream. The Component Container
makes no attempt to parse the data being sent by the remote end. It provides it "as is" for
the component code to process. A view in this mode is normally not needed. However, there are
situations in which this mode is needed, for example when dealing with image data, structuring
the data into fields does not make sense. Since the expected content type of this binary stream is
only known to the component code, the list of expected view_content_types must be provided while
creating this view.
@param name: Input view name.
@type name: str
@param view_content_types: Specifies a list of content types the input view can accept.
For example:
('image/jpeg',)
Or, a more extensive list can be specified as follows:
('image/jpeg;q=0.6', 'image/png;q=0.6', 'image/gif;q=0.5', 'image/*;q=0.1')
The q represents the quality param of the content type as defined for Accept header in RFC 2616.
It basically indicates the preference for a particular content type and takes the value from
0.0 to 1.0.
@type view_content_types: sequence
@param description_str: Description for the entire view.
@type description_str: str
@param modifiable: If set to False, then only the server and component can modify this view and not
the client. The default value is True (meaning it is modifiable by client).
@type modifiable: bool
"""
if not self._allow_definition_changes:
raise SnapObjPermissionError("Resource definition changes is not permitted at this stage")
self._resdef.add_binary_input_view(name, view_content_types, description_str, modifiable)
def get_input_view_def(self, name):
"""
Returns definition of the specified input view name.
@param name: Name of the input view
@type name: str
@return: A dictionary describing the view. The dictionary has the following entries:
is_record - If this is True, then the view is in record and the dictionary has
the following other entries:
fields - The list of fields that was specified for add_record_input_view_def()
description_str - Description string for the view.
modifiable - Boolean value indicating if the view is modifiable.
If is_record is set to False, then the view is in binary mode and the dictionary
has the following set of entries:
view_content_types - Basically the list of view_content_types as specified for
the method add_binary_input_view_def().
description_str - Description string for the view.
modifiable - Boolean value indicating if the view is modifiable.
@rtype: dict
"""
return self._resdef.get_input_view(name)
def remove_input_view_def(self, name):
"""
Removes specified view definition from the resource.
@param name: Name of the view to remove
@type name: str
"""
if not self._allow_definition_changes:
raise SnapObjPermissionError("Resource definition changes are not permitted at this stage")
self._resdef.remove_input_view(name)
def list_input_view_names(self):
"""
Return a list of names of all input views.
@return: List of input view names.
@rtype: list of str
"""
return self._resdef.list_input_view_names()
#
# Methods for manipulating and retrieving information related to output views.
#
def add_record_output_view_def(self, name, fields, description_str, modifiable = True):
"""
Creates a record mode output view. This type of output view has data written to it in the
record format. The view defines the list of typed fields that the records must have.
@param name: Output view name.
@type name: str
@param fields: A sequence of fields, where each field is defined by the sequence
(field name, field type, field description string). The field name must be a
unique string name in the context of that view. The field type is also a string
which specifies one of the supported data types for that field. The field
description is a string which describes the field.
@type fields: sequence
@param description_str: Description string for the entire view.
@type description_str: str
@param modifiable: If set to False, then only the server and component can modify this view and not
the client. The default value is True (meaning it is modifiable by client).
@type modifiable: bool
"""
if not self._allow_definition_changes:
raise SnapObjPermissionError("Resource definition changes are not permitted at this stage")
self._resdef.add_record_output_view(name, fields, description_str, modifiable)
def add_binary_output_view_def(self, name, view_content_types, description_str, modifiable = True):
"""
Add binary mode output view.
The output view is defined in "binary mode". In binary mode, the output view has no concept of fields,
the component code writes a binary strings to the output view. As with binary input view, the output
view must be reated with the list of possible content types specified.
@param name: Input view name.
@type name: str
@param view_content_types: TSpecifies a list of content types the output view can create.
For example:
('image/jpeg',)
Or, a more extensive list can be specified as follows:
('image/jpeg;q=0.6', 'image/png;q=0.6', 'image/gif;q=0.5', 'image/*;q=0.1')
The q represents the quality param of the content type as defined for Accept header in RFC 2616.
It basically indicates the preference for a particular content type and takes the value from
0.0 to 1.0.
@type view_content_types: sequence
@param description_str: Description string for the entire view.
@type description_str: str
@param modifiable: If set to False, then only the server and component can modify this view and not
the client. The default value is True (meaning it is modifiable by client).
@type modifiable: bool
"""
if not self._allow_definition_changes:
raise SnapObjPermissionError("Resource definition changes are not permitted at this stage")
self._resdef.add_binary_output_view(name, view_content_types, description_str, modifiable)
def get_output_view_def(self, name):
"""
Return definition of the specified view name.
@param name: Name of the output view
@type name: str
@return: A dictionary describing the view. The dictionary has the following entries:
is_record - If this is True, then the view is in record mode and the dictionary has the
following other entries:
fields - The list of fields that was specified for add_record_output_view_def()
description_str - Description string for the view.
If is_record is set to False, then the view is in binary mode and the dictionary has
the following set of other entries:
view_content_types - Basically the list of view_content_types as specified for the
method add_binary_output_view_def().
description_str - Description string for the view.
@rtype: dict
"""
return self._resdef.get_output_view(name)
def remove_output_view_def(self, name):
"""
Remove specified output view from the resource.
@param name: Name of the output view to remove
@type name: str
"""
if not self._allow_definition_changes:
raise SnapObjPermissionError("Resource definition changes are not permitted at this stage")
self._resdef.remove_output_view(name)
def list_output_view_names(self):
"""
Return a list of names of all output views.
@return: List of output view names.
@rtype: list of str
"""
return self._resdef.list_output_view_names()
#
# Methods related to pass through
#
def set_output_view_pass_through(self, output_view_name, input_view_names, modifiable = True):
"""
Set output view pass-through settings.
This method is used to specify the list of input views that will pass through the fields
to a specified output view.
@param output_view_name: Name of the output view that is getting the pass through fields.
@type output_view_name: str
@param input_view_names: The list of input views that will pass through fields to the output
view. The input views must be created as pass through input views.
@type input_view_names: list of str
@param modifiable: If set to False, then only the server and component can modify this setting
and not the client. The default value is True (meaning it is modifiable by client).
@type modifiable: bool
"""
if not self._allow_definition_changes:
raise SnapObjPermissionError("Resource definition changes are not permitted at this stage")
self._resdef.set_output_view_pass_through(output_view_name, input_view_names, modifiable)
def unset_output_view_pass_through(self, output_view_name):
"""
Remove pass through setting on an output view.
@param output_view_name: Name of the output view that is having its pass through setting removed.
@type output_view_name: str
"""
if not self._allow_definition_changes:
raise SnapObjPermissionError("Resource definition changes are not permitted at this stage")
self._resdef.unset_output_view_pass_through(output_view_name)
def get_output_view_pass_through(self, output_view_name):
"""
Returns the list of input views that are passing through fields to the output view.
@param output_view_name: Name of the output view whose pass through settings are being requested.
@type output_view_name: str
@return: dictionary containing pass-through setting for that output view-
{'input_views': (List of input view names that are passing through fields to the output view),
'modifable': True/False modifable flag }
@rtype: dict
"""
return self._resdef.get_output_view_pass_through(output_view_name)
def list_pass_through_output_views(self):
"""
Returns a list of pass through output views
@return: List of output view names that have pass-through settings.
@rtype: list of str
"""
return self._resdef.list_pass_through_output_views()
#
# Methods for defining and manipulating parameters of the resource.
#
def add_parameter_def(self, param_name, default_value = None, modifiable = True):
"""
Add a parameter to the resource.
Parameters can be optional or required. If a default value is specified here, then the
psarameter is optional.
@param param_name: Name of the parameter to be defined
@type param_name: str
@param default_value: If the parameter is optional, then a default value
must be specified here. If it is not an optional parameter, then
this value must be None.
@type default_value: str
@param modifiable: If set to False, then only the server and component can modify this
param definition and not the client. The default value is True (meaning it is
modifiable by client).
@type modifiable: bool
"""
if not self._allow_definition_changes:
raise SnapObjPermissionError("Resource definition changes are not permitted at this stage")
self._resdef.define_param(param_name, default_value, modifiable)
def remove_parameter_def(self, param_name):
"""
Remove definition of specified param.
@param param_name: Name of the parameter.
@type param_name: string
"""
if not self._allow_definition_changes:
raise SnapObjPermissionError("Resource definition changes are not permitted at this stage")
self._resdef.remove_param_def(param_name)
def get_parameter_def(self, param_name):
"""
Get parameter definition.
@param param_name: Name of the parameter.
@type param_name: string
@return: Returns a 2-tupe. The first value is the default value of the param or None and
second is the value of modifiable flag.
@rtype: tuple
"""
return self._resdef.get_param_def(param_name)
def _set_parameters(self, d):
"""
Set parameters in the component.
@param d: Dictionary of params and their values
@type d: dict
"""
self.parameters = dict(d)
def get_parameters(self):
"""
Get parameter dictionary.
@return: The param dictionary containing param name as key and param value as the value.
@rtype: dict
"""
return dict(self.parameters)
def add_resource_ref_def(self, ref_name, label, categories_allowed=None, required=True):
"""
Add a resource ref to the resource.
Sometimes, a component needs access to a certain resource definition in order to run.
For example, a DBRead component needs access to a connection resource definition in order
to run. A resource reference is the mechanism by which a component can declare such a dependency.
The component defines the reference with a name (reference name). When a resource is created based on
the component, the resource author should fill in an appropriate resource URI as a value to this
reference. The definition also specifies whether the reference is required or optional (meaning,
whether the value "must" be provided or "may" be provided while creating a resource). The definition
can also specify the categories to which the resource must belong in order to be an appropriate value
for the reference. For example, lets assume that DB connection resources are all defined under the
category "connection.db". MySQL connections would have category "connection.db.mysql", Oracle would
have "connection.db.oracle", etc. Now, DBRead can define a resource reference as follows:
add_resource_ref_def("db_connection", "Database Connection Reference", ["connection.db"], True)
Note how the definition specifies "connection.db" as the general category of resources that can be
assigned to this reference.
Another thing to note is that when a resource based on DBRead is dragged into a pipeline, its
resource reference value can be overwritten within the context of the pipeline by the pipeline author.
It can not only be set to another resource URI, but also to the instance name of another resource
within that pipeline. When another resource within the pipeline is specified, the SnapLogic platform
not only fetches the resource for the component, but also substitutes whatever params have been
specified in that resource, before providing it to the DBRead component.
@param ref_name: Name of the parameter to be defined
@type ref_name: str
@param label: This is a UI friendly label to display resource reference.
@type label: str
@param categories_allowed: List of categories to allow resource reference from.
@type categories_allowed: list
@param required: Indicates whether the reference must have a value or not.
@type required: bool.
"""
if not self._allow_definition_changes:
raise SnapObjPermissionError("Resource definition changes are not permitted at this stage")
self._resdef._define_resource_ref(ref_name, label, categories_allowed, required)
def get_resource_ref(self, name):
ret = self._resdef.get_resource_ref(name)
return ret
def set_resource_ref(self, ref_name, ref_value):
self._resdef.set_resource_ref(ref_name, ref_value)
def get_referenced_resdef(self, ref_name):
return self.resource_ref[ref_name]
def set_categories(self, category_list, modifiable=True):
"""
Set categories list (thereby overwriting all existing entries).
@param category_list: List of categories.
@type category_list: list
@param modifiable: If set to False, then only the server and component can change this
categories setting. The default value is True (meaning it is modifiable by client).
@type modifiable: bool
"""
self._resdef._set_categories_modifiable(modifiable)
self._resdef.set_resource_categories(category_list)
def set_property_def(self, property_name, property_def):
"""
Set the definition of a component specific property in the resource definition.
@param property_name: Name of the property
@type property_name: str
@param property_def: Simple or complex property definition.
@type property_def: L{prop.SimpleProp}or L{prop.ListProp} or L{prop.DictProp}
"""
if not self._allow_definition_changes:
raise SnapObjPermissionError("Resource definition changes are not permitted at this stage")
if type(property_def) not in (prop.SimpleProp, prop.ListProp, prop.DictProp):
raise SnapObjTypeError("\"%s\" is not a valid object for property definition" % type(property_def))
prop_dict = property_def._to_resdef()
self._resdef._set_property_def(property_name, prop_dict)
self._resdef.set_property_value(property_name, None)
def del_property_def(self, property_name):
"""
Deletes the specified property definition and its value from the resdef
@param property_name: Name of the property to be removed
@type property_name: str
"""
if not self._allow_definition_changes:
raise SnapObjPermissionError("Resource definition changes are not permitted at this stage")
self._resdef._del_property_def(property_name)
def get_property_def(self, property_name):
"""
Get the current definition of the property.
@param property_name: Name of the property.
@type property_name: str
@return: Dictionary containing -
{keys.PROPERTY_DEFINITION: definition of the simple or complex property}
@rtype: dict
"""
ret = self._resdef.get_property_def(property_name)
ret[keys.PROPERTY_DEFINITION] = prop.create_property_from_resdef(ret[keys.PROPERTY_DEFINITION])
return ret
def get_property_value(self, property_name):
"""
Return current value of property.
@param property_name: Name of the property.
@type property_name: str
@return: Value of simple or complex property.
@rtype: simple types or list or dict.
"""
if self._allow_definition_changes and len(self.parameters) > 0:
# There are params to be substituted. For methods like suggest_resource_values()
# we do not substitute params up-front, because that would modify the
# resdef with the substituted value and the substituted resdef would be returned
# to the client (a bad thing). So, we do the substituition on call, right here
# and never touch the actual resdef.
# We get a deep-copied value here.
prop_val = self._resdef.get_property_value(property_name)
prop_def = self.get_property_def(property_name)[keys.PROPERTY_DEFINITION]
# Substitute and return
return prop.substitute_params_in_property(prop_val, self.parameters, prop_def)
else:
# In execute mode, the params have already been substituted up-front.
return self._resdef.get_property_value(property_name)
def set_property_value(self, property_name, value):
"""
Set property value.
@param property_name: Name of the property.
@type property_name: str
@param value: New value of the simple or complex property.
@type value: Simple types or list or dict.
"""
self._resdef.set_property_value(property_name, value)
#
# The following methods and attributes should be overridden in the derived class
#
component_description = ""
"""A brief description of the component. Should be set in the derived component class."""
component_label = ""
"""Display friendly label for the component. Should be set in the derived component class."""
component_doc_uri = None
"""Specify a URI at which documentation for this component can be found. Set to None, if not such URI exists."""
def create_resource_template(self):
"""
Create a resource template for the component.
Most Components have resource definitions that have a certain pattern or template. For example,
- Component specific properties like "filename"
- Specific upper limit or lower limit or both on number of input/output views.
- Hardcoded view names and their structures.
- Hardcoded property values, etc.
In order to provide the client who is designing a resource based on the component, with such a
template this method has been provided. The component author should use this method to define
component specific properties and fill in expected values for properties and views.
"""
def suggest_resource_values(self, resdef_error):
"""
Suggest the values of the resource definition.
This is an optional method, that can be overridden to provide the component with the capability
to suggest values for the resource definition, based on the values already provided by the
resource author. If the component is unable to make suggestions, due to errors in the resource
definition, then this method can use the resdef_error object to specify what those errors are.
@param resdef_error: This object provides a convenient interface for setting error messages
on invalid values in the resource definition.
@type resdef_error: L{prop_err.ComponentResourceErr}
"""
def validate(self, resdef_error):
"""
Validate the resource definition for the component.
This is an optional method, that can be overridden to provide the component with the capability
to validate the contents of the resource definition. The errors found should be set using the
set_message() methods provided by the resdef_error object.
@param resdef_error: This object provides a convenient interface for setting error messages
on invalid values in the resource definition.
@type resdef_error: L{prop_err.ComponentResourceErr}
"""
def execute(self, input_views, output_views):
"""
Execute the component code.
This is a required method. It must be overridden by all valid components to implement the
data processing logic of the component.
"""
raise SnapComponentError("The execute method must be overridden")
def validate_config_file(self):
"""
Validates the component config file.
The Component Container (CC) will parse the contents of the config file into a dictionary at startup
or reconfiguration time and then call this method to validate the contents of the dictionary. The
dictionary is made available as "config_dictionary" attribute of this class.
"""
# Set these optional methods to False, so we know when they get overridden.
create_resource_template = False
suggest_resource_values = False
validate = False
validate_config_file = False
def __process_local_resource(self, uri=None):
"""
Modify URI and create header entries needed for making a local resource request.
@param uri: URI of the local server.
"""
# Overwrite the server portion of the URI with local server's URI.
if uri is not None:
inp_uri = urlparse.urlparse(uri)
server_uri = urlparse.urlparse(cc.server_uri)
uri = urlparse.urlunparse((server_uri[0], server_uri[1], inp_uri[2], inp_uri[3], inp_uri[4], inp_uri[5]))
custom_hdr = {}
# Never send None value in header, it gets stringified to the string "None"
if cc.cc_token is not None:
custom_hdr[headers.CC_TOKEN_HEADER] = cc.cc_token
if self._invoker_username is not None:
custom_hdr[headers.INVOKER_HEADER] = self._invoker_username
return (uri, custom_hdr)
def get_invoker_username(self):
"""
Return the SnapLogic username used (if any) in the credentials for invoking the component method.
@return: Username (if any).
@rtype: str
"""
return self._invoker_username
def get_local_resource_object(self, uri):
"""
Fetch resource object from the local main server.
Unlike snapi.get_resource_object(), this call goes only to the local main
server to fetch the resource object. The resdef is fetched with the
username provided in the HTTP PREPARE request.
@param uri: URI of the resource to be retrieved.
@type uri: str.
@return: resdef object retrieved.
@rtype: L{ResDef}
"""
# Overwrite the server portion of the URI with local server's URI.
(uri, custom_hdr) = self.__process_local_resource(uri)
return resdef_module.read_resdef(uri, None, custom_hdr, True)
def create_local_resource_object(self, comp_name):
"""
Fetch component resource template from the local main server.
Unlike snapi.create_resource_object(), this call goes only to the local main
server to fetch the resource object. The resdef template is fetched with the
username provided during the execution of the resource.
@param comp_name: Name of the component
@type comp_name: str
@return: resdef object created.
@rtype: L{ResDef}
"""
# Overwrite the server portion of the URI with local server's URI.
(uri, custom_hdr) = self.__process_local_resource()
return resdef_module.get_resdef_template(cc.server_uri, comp_name, None, custom_hdr, True)
def get_resource_object(self, uri, cred=None):
"""
Retrieve resource definition object from specified location.
If no credentials are specified and the URI points to the local server, then the credentials
used to execute this component will be used to retrieve the resource.
@param uri: URI of the resource to be retrieved.
@type uri: str.
@param cred: Credentials specified as (username, password)
@type cred: 2-tuple.
@return: resdef object created.
@rtype: L{ResDef}
"""
if cred is None and cc.is_my_server_uri(uri):
return self.get_local_resource_object(uri)
else:
return snapi.get_resource_object(uri, cred)
def create_resource_object(self, comp_name, server_uri=None, cred=None):
"""
Create a SnapLogic resource definition object for a specified component name.
If no server uri is specified, then the local server URI is used. If credentials
are not specified and we are accessing the local server, then the credentials
used to execute this component are used.
@param comp_name: Name of the component for which resource must be created.
@type comp_name: str
@param server_uri: Server from which to access the component.
@type server_uri: str
@param cred: Credentials specified as (username, password)
@type cred: 2-tuple.
@return: resdef object created.
@rtype: L{ResDef}
"""
if server_uri is None or cc.is_my_server_uri(server_uri):
if cred is None:
return self.create_local_resource_object(comp_name)
else:
return snapi.create_resource_object(cc.server_uri, comp_name, cred)
else:
return snapi.create_resource_object(server_uri, comp_name, cred)
def exec_resource(self, uri, inputs=None, outputs=None, params=None, cred=None, name=None):
"""
Execute resource.
@param uri: The URI of the resource to be executed.
@type uri: str
@param inputs: Dictionary with names of the input views as keys. The value can be requested content type or
can be None, if default content_type is fine.
@type inputs: dict
@param outputs: Dictionary with names of the output views as keys. The value can be requested content type or
can be None, if default content_type is fine.
@type outputs: dict
@param params: Dictionary with param name as key and param value as value.
@type params: dict
@param cred: A 2-tuple containing (username, password)
@type cred: tuple
@param name: User defined name for the resource. Could be any string the caller of this
method would like to use to identify the pipeline being invoked. This name does not have to
be unique.
@type name: str
@return: Runtime handle of the executing resource.
@rtype: L{Handle}
"""
resdef = self.get_resource_object(uri, cred)
if cred is None and cc.is_my_server_uri(uri):
# Overwrite the server portion of the URI with local server's URI.
(uri, custom_hdr) = self.__process_local_resource(uri)
return exec_interface.exec_resource(name, uri, resdef, inputs, outputs, params, None, custom_hdr)
else:
return snapi.exec_resource(uri, inputs, outputs, params, cred, name)
|