# $SnapHashLicense:
#
# SnapLogic - Open source data services
#
# Copyright (C) 2009, SnapLogic, Inc. All rights reserved.
#
# See http://www.snaplogic.org for more information about
# the SnapLogic project.
#
# This program is free software, distributed under the terms of
# the GNU General Public License Version 2. See the LEGAL file
# at the top of the source tree.
#
# "SnapLogic" is a trademark of SnapLogic, Inc.
#
#
# $
# $Id: cc_info.py 10330 2009-12-24 22:13:38Z grisha $
"""
This module handles general management information about the component container.
Some of its services will include:
- providing the list of components in the container
- aggregating and returning log information
- Reporting uptime and container level statistics
- The process identity of the CC. Could be http://hostname:8088 for Paste and something longer for Apache mode.
"""
import copy
import prop
from snaplogic import rp
from snaplogic import cc
from snaplogic.common.snap_exceptions import *
import snaplogic.common.snap_http_lib as snap_http_lib
from snaplogic.common import uri_prefix,uri_utils
from snaplogic.common import runtime_table
from snaplogic.common import prop_err
from snaplogic.common.prop_err import create_err_obj
import snaplogic.snapi_base.keys as keys
from snaplogic.snapi_base.resdef import ResDef
from snaplogic.cc import registration
from snaplogic.common import snap_log
def list_resource_runtimes(http_req):
"""This function returns the resource runtime entries that CC has."""
ret_dict = runtime_table.list_runtime_entries()
return (http_req.OK, ret_dict)
def _set_method_availability(comp_name, comp, capability_dict):
if comp.create_resource_template == False:
capability_dict[keys.CAPABILITY_CREATE_RESOURCE_TEMPLATE] = None
else:
capability_dict[keys.CAPABILITY_CREATE_RESOURCE_TEMPLATE] = \
snap_http_lib.concat_paths(uri_prefix.COMPONENT_RESOURCE_TEMPLATE, comp_name)
if comp.suggest_resource_values == False:
capability_dict[keys.CAPABILITY_SUGGEST_RESOURCE_VALUES] = None
else:
capability_dict[keys.CAPABILITY_SUGGEST_RESOURCE_VALUES] = \
snap_http_lib.concat_paths(uri_prefix.COMPONENT_SUGGEST_RESOURCE_VALUES, comp_name)
if comp.validate == False:
capability_dict[keys.CAPABILITY_VALIDATE] = None
else:
capability_dict[keys.CAPABILITY_VALIDATE] = \
snap_http_lib.concat_paths(uri_prefix.COMPONENT_VALIDATE_RESOURCE, comp_name)
# Advertise the upgrade URI
capability_dict[keys.CAPABILITY_UPGRADE] = \
snap_http_lib.concat_paths(uri_prefix.COMPONENT_UPGRADE_RESOURCE, comp_name)
def list_components(http_req):
"""
Return a brief summary of all components in the CC.
The information returned is in the following format:
{ 'snaplogic.components.CsvRead: {
'capabilities': {
'create_resource_template': http://,
'validate': True,
'suggest_resource_values': True
},
'description': 'Read CSV format files',
'component_version': '1.0',
'label': 'CSV Reader'
},
'snaplogic.components.CsvWrite: {
'capabilities': {
'create_resource_template': True,
'validate': False,
'suggest_resource_values': False
},
'description': 'Write CSV format files',
'component_version': '1.1',
'label': 'CSV Writer'
}
}
"""
ret_dict = {}
comps = registration.get_registered_components()
for comp_name in comps:
comp_dict = {}
# Save all the unique capabilities of the component into the dict.
if hasattr(comps[comp_name], "capabilities"):
comp_dict[keys.CAPABILITIES] = copy.deepcopy(comps[comp_name].capabilities)
else:
comp_dict[keys.CAPABILITIES] = {}
_set_method_availability(comp_name, comps[comp_name], comp_dict[keys.CAPABILITIES])
comp_dict[keys.DESCRIPTION] = comps[comp_name].component_description
comp_dict[keys.LABEL] = comps[comp_name].component_label
comp_dict[keys.COMPONENT_DOC_URI] = comps[comp_name].component_doc_uri
# This is a hand-coded special case to give us a component-level URI to GET an
# overview. In essence, this returns the same information as if you would POST
# a {} to the CAPABILITY_CREATE_RESOURCE_TEMPLATE URI. It's just easier to use
# and more readily visible (due to clearer label) to the client. This is not
# terribly elegant, though... FIXIT?
comp_dict[keys.OVERVIEW] = comp_dict[keys.CAPABILITIES][keys.CAPABILITY_CREATE_RESOURCE_TEMPLATE]
# If component has a version make sure to include it
if hasattr(comps[comp_name], keys.COMPONENT_VERSION):
comp_dict[keys.COMPONENT_VERSION] = comps[comp_name].component_version
else:
# Otherwise, set version to None
comp_dict[keys.COMPONENT_VERSION] = None
ret_dict[comp_name] = comp_dict
return (http_req.OK, ret_dict)
def create_resource_template(http_req):
"""
Create resource template, if the component has that capability.
@param http_req: The http request object that triggered this request.
@type http_req: L{HttpRequest}
@return: In case of success: return (http_req.OK, template resdef).
In case of errors : return (http_req.NOT_FOUND, "error string")
@rtype: 2 tuple
"""
return _get_or_create_resource_template(http_req, False)
def get_resource_template(http_req):
"""
Get resource template, if the component has that capability.
@param http_req: The http request object that triggered this request.
@type http_req: L{HttpRequest}
@return: In case of success: return (http_req.OK, template resdef).
In case of errors : return (http_req.NOT_FOUND, "error string")
@rtype: 2 tuple
"""
return _get_or_create_resource_template(http_req, True)
def _get_or_create_resource_template(http_req, empty_request=False):
"""
Get or create resource template, if the component has that capability.
@param http_req: The http request object that triggered this request.
@type http_req: L{HttpRequest}
@param empty_request: If set, we will POST an empty dictionary, which
results in a complete, from-scratch definition of
the resdef being returned. This is useful when a
GET request to the relevant URI is issued, and the
handler of the GET request just quickly needs the
resdef dictionary.
@type empty_request: bool
@return: In case of success: return (http_req.OK, template resdef).
In case of errors : return (http_req.NOT_FOUND, "error string")
@rtype: 2 tuple
"""
if empty_request:
# Someone didn't specify a template, they just want to get a brand new one
# from scratch. So, we just set the template to the empty dictionary and
# proceed as usual.
resdef_dict = {}
else:
# Someone provided us with a first draft of the template. So we read it an
# then proceed as usual.
http_req.make_input_rp()
try:
resdef_dict = http_req.input.next()
except StopIteration:
return (http_req.BAD_REQUEST, "Resource definition missing")
try:
comp_name = uri_utils.extract_comp_name_from_uri(http_req.path)
except Exception, e:
cc.elog(e, "Create resource template method got invalid URI %s" % http_req.path)
return (http_req.NOT_FOUND, {keys.ERROR: "The URI provided is not valid for creating a resource template"} )
comp_class = registration.get_component_class(comp_name)
if comp_class is None:
return (http_req.NOT_FOUND, "Component %s not found in this CC" % comp_name)
config_dict = registration.get_component_config(comp_name)
# Instantiate the component class.
c = comp_class()
if comp_class.create_resource_template == False:
return (http_req.NOT_FOUND, "Component %s does not ability to create resource template" % comp_name)
resdef_dict[keys.COMPONENT_NAME] = comp_name
# If component has a version number add it to the resdef
if hasattr(c, keys.COMPONENT_VERSION):
resdef_dict[keys.COMPONENT_VERSION] = c.component_version
else:
# Otherwise, set version to None
resdef_dict[keys.COMPONENT_VERSION] = None
if hasattr(c, "component_description"):
resdef_dict[keys.DESCRIPTION] = c.component_description
if hasattr(c, "component_doc_uri"):
resdef_dict[keys.COMPONENT_DOC_URI] = c.component_doc_uri
c._init_comp(config_dict, ResDef(resdef_dict), invoker=http_req.invoker)
try:
c.create_resource_template()
except Exception, e:
cc.elog(e, "Component specific create_resource_template method failed unexpectedly for %s" % comp_name)
return (http_req.INTERNAL_SERVER_ERROR,
"Component specific create_resource_template method failed unexpectedly for %s" % comp_name)
_set_method_availability(comp_name, comp_class, c._resdef.dict[keys.CAPABILITIES])
return (http_req.OK, c._resdef.dict)
def suggest_resource_values(http_req):
"""
Suggest resource values, if the component has that capability.
@param http_req: The http request object that triggered this request.
@type http_req: L{HttpRequest}
@return: In case of success: return (http_req.OK, updated resdef).
In case of resdef errors : return (http_req.BAD_REQUEST, error dictionary)
In case of other errors : return (http_req.NOT_FOUND, "error string")
@rtype: 2 tuple
"""
http_req.make_input_rp()
try:
resdef_dict = http_req.input.next()
except StopIteration:
return (http_req.BAD_REQUEST, "Resource definition missing")
try:
param_dict = http_req.input.next()
except StopIteration:
param_dict = {}
try:
comp_name = uri_utils.extract_comp_name_from_uri(http_req.path)
except Exception, e:
cc.elog(e, "Suggest resource values method got invalid URI %s" % http_req.path)
return (http_req.NOT_FOUND, "The URI provided for suggest resource values does not exist" )
comp_class = registration.get_component_class(comp_name)
if comp_class is None:
return (http_req.NOT_FOUND, "Component %s not found in this CC" % comp_name)
if comp_class.suggest_resource_values == False:
return (http_req.NOT_FOUND, "Component %s does not have ability to suggest resource values" % comp_name)
config_dict = registration.get_component_config(comp_name)
c = comp_class()
res_def = ResDef(resdef_dict)
param_dict = res_def._update_param_values_with_defaults(param_dict)
c._init_comp(config_dict, res_def, invoker=http_req.invoker)
c._set_parameters(param_dict)
# We specify partial_data to be True, since at the time of calling sugggest_resource_values, all the resource
# values have not be provided and rigorous validation should be avoided.
err_obj = prop_err.ComponentResourceErr(resdef_dict, partial_data = True)
# If the resdef sent was found to be of invalid structure, then send the errors back right away.
err = err_obj._to_resdef()
if err is not None:
return (http_req.BAD_REQUEST, err)
try:
c.suggest_resource_values(err_obj)
except Exception, e:
cc.elog(e, "Component specific suggest_resource_values method failed unexpectedly for %s" % comp_name)
return (http_req.INTERNAL_SERVER_ERROR,
"Component specific suggest_resource_values method failed unexpectedly for %s" % comp_name)
err = err_obj._to_resdef()
if err is not None:
return (http_req.BAD_REQUEST, err)
return (http_req.OK, c._resdef.dict)
def upgrade_resource(http_req):
"""
Upgrade resource, if the component has that capability.
@param http_req: The http request object that triggered this request.
@type http_req: L{HttpRequest}
@return: In case of success: return (http_req.OK, updated resdef).
If component is not found : return (http_req.NOT_FOUND, "error string")
In case of other errors : return (http_req.INTERNAL_ERROR, "error string")
@rtype: 2 tuple
"""
http_req.make_input_rp()
try:
resdef = http_req.input.next()
except StopIteration:
return (http_req.BAD_REQUEST, "Resource definition missing")
comp_name = resdef[keys.COMPONENT_NAME]
comp_class = registration.get_component_class(comp_name)
if comp_class is None:
return (http_req.NOT_FOUND, "Component %s not found in this CC" % comp_name)
config_dict = registration.get_component_config(comp_name)
c = comp_class()
c._init_comp(config_dict, ResDef(resdef), invoker=http_req.invoker)
# Check version of the resdef that we're upgrading ("from_version")
if keys.COMPONENT_VERSION in resdef and resdef[keys.COMPONENT_VERSION] is not None:
from_version = resdef[keys.COMPONENT_VERSION]
else:
# If resdef has no version set resdef version to 1.0
resdef[keys.COMPONENT_VERSION] = from_version = "1.0"
# Check component version to which we're upgrading ("to_version")
if hasattr(c, keys.COMPONENT_VERSION) and c.component_version is not None:
to_version = c.component_version
else:
# Component has no version, assume 1.0
to_version = "1.0"
# Split version into major.minor
(from_major, from_minor) = from_version.split('.')
(to_major, to_minor) = to_version.split('.')
# We cannot upgrade across major versions, e.g. from 1.5 to 2.3
# so if that's the case report an error
if from_major != to_major:
return (http_req.BAD_REQUEST, "Cannot upgrade if major version differs: %s from %s to %s" %
(comp_name, from_major, to_major))
# As part of the upgrade fix the component_doc_uri in the resdef
# to match the latest version
if hasattr(c, "component_doc_uri"):
c._resdef.dict[keys.COMPONENT_DOC_URI] = c.component_doc_uri
# Now for every version increment call the upgrade method in the component.
# This method mutates the resdef of the resource. Return the mutated resdef.
#
# E.g. if:
# from_version to_version what's called
# 1.0 1.1 upgrade_1_0_to_1_1
# 1.0 1.2 upgrade_1_0_to_1_1, upgrade_1_1_to_1_2
# 1.0 1.3 upgrade_1_0_to_1_1, upgrade_1_1_to_1_2, upgrade_1_2_to_1_2
# 1.0 2.0 not possible (major number different)
major = from_major
for i in range(int(from_minor), int(to_minor)):
# What's the name of the method for performing the incremental upgrade
method_name = "upgrade_" + str(major) + "_" + str(i) + "_to_" + str(major) + "_" + str(i + 1)
try:
# Does the component have the method?
if hasattr(c, method_name):
method = c.__getattribute__(method_name)
cc.log(snap_log.LEVEL_DEBUG,
"Upgrading component %s: %s" %
(comp_name, method_name))
method()
# If the method went through successfully, set the new version
c._resdef.dict[keys.COMPONENT_VERSION] = major + "." + str(i + 1)
else:
# Component doesn't have the method to perform this upgrade
error = "%s cannot be upgraded: missing method %s" % (comp_name, method_name)
cc.log(snap_log.LEVEL_ERR, error)
return (http_req.INTERNAL_SERVER_ERROR, error)
except Exception, e:
error = "%s %s failed: %s" % (comp_name, method_name, e)
cc.elog(e, error)
return (http_req.INTERNAL_SERVER_ERROR, error)
cc.log(snap_log.LEVEL_DEBUG,
"Successfully upgraded %s from version %s to version %s" %
(comp_name, from_version, to_version))
return (http_req.OK, c._resdef.dict)
def validate_resdef(http_req):
"""
Validate resource definition, if the component has that capability.
@param http_req: The http request object that triggered this request.
@type http_req: L{HttpRequest}
@return: In case of success: return (http_req.OK, None).
In case of resdef validation errors : return (http_req.BAD_REQUEST, error dictionary)
In case of other errors : return (http_req.NOT_FOUND, "error string")
@rtype: 2 tuple
"""
http_req.make_input_rp()
try:
resdef_dict = http_req.input.next()
except StopIteration:
return (http_req.BAD_REQUEST, "Resource definition missing")
try:
comp_name = uri_utils.extract_comp_name_from_uri(http_req.path)
except Exception, e:
cc.elog(e, "Validate resource definition method got invalid URI %s" % http_req.path)
return (http_req.NOT_FOUND, "The URI provided for validating resource definition does not exist" )
comp_class = registration.get_component_class(comp_name)
if comp_class is None:
return (http_req.NOT_FOUND, "Component %s not found in this CC" % comp_name)
if comp_class.validate == False:
return (http_req.NOT_FOUND, "Component %s does not have ability to validate" % comp_name)
c = comp_class()
config_dict = registration.get_component_config(comp_name)
c._init_comp(config_dict, ResDef(resdef_dict), invoker=http_req.invoker)
err_obj = prop_err.ComponentResourceErr(resdef_dict)
# If the resdef sent was found to be of invalid structure, then send the errors back right away.
err = err_obj._to_resdef()
if err is not None:
return (http_req.BAD_REQUEST, err)
# Validate that component and resdef versions match
if hasattr(c, keys.COMPONENT_VERSION) and c.component_version is not None:
component_version = c.component_version
else:
# Component has no version, assume 1.0
component_version = "1.0"
if keys.COMPONENT_VERSION in resdef_dict and resdef_dict[keys.COMPONENT_VERSION] is not None:
resource_version = resdef_dict[keys.COMPONENT_VERSION]
else:
# Resdef has no version, assume 1.0
resource_version = '1.0'
if resource_version != component_version:
# Version mismatch is a validation error
err_obj.set_message("Component version %s doesn't match resource version %s." %
(component_version, resource_version))
# If there is a version mismatch don't attempt to call component's validate method:
# it's likely it will fail: there may be properties that changed etc.
else:
try:
c.validate(err_obj)
except Exception, e:
cc.elog(e, "Component specific validate method failed unexpectedly for %s" % comp_name)
return (http_req.INTERNAL_SERVER_ERROR,
"Component specific validate method failed unexpectedly for %s" % comp_name)
err_dict = err_obj._to_resdef()
if err_dict is not None:
# We have errors
return (http_req.BAD_REQUEST, err_dict)
return (http_req.OK, None)
|