#!/usr/bin/env python
# $SnapHashLicense:
#
# SnapLogic - Open source data services
#
# Copyright (C) 2008, SnapLogic, Inc. All rights reserved.
#
# See http://www.snaplogic.org for more information about
# the SnapLogic project.
#
# This program is free software, distributed under the terms of
# the GNU General Public License Version 2. See the LEGAL file
# at the top of the source tree.
#
# "SnapLogic" is a trademark of SnapLogic, Inc.
#
#
# $
# $Id: registration.py 5682 2008-12-05 18:14:13Z grisha $
"""
This is a place holder for component registration. Should be replaced by the
actual implementation.
"""
import sys
import os
from stat import *
from snaplogic.common.snap_exceptions import *
from snaplogic.common import snap_log
from snaplogic import cc
from snaplogic.cc.component_api import ComponentAPI
from snaplogic.common.config import snap_config
# Do not add components manually here. They will automatically be picked up now.
component_map = {}
# Global value marking what component directory currently looking in.
current_component_dir = None
def discover_configured_components():
global current_component_dir
cc_section = cc.config.get_section('cc')
try:
dirs = cc_section['component_dirs']
if type(dirs) is not list:
dirs = [dirs]
except KeyError:
dirs = []
for dirname in dirs:
cc.log(snap_log.LEVEL_INFO, "Discovering components under directory %s" % dirname)
current_component_dir = dirname
discover_component_packages(dirname)
if not component_map:
cc.log(snap_log.LEVEL_ERR, "No components loaded.")
def discover_component_packages(dirname):
try:
contents = os.listdir(dirname)
except OSError, e:
cc.elog(e, "Error searching for components in directory '%s': %s." % (dirname, e))
return
sys.path.append(dirname)
for filename in contents:
if filename != 'config':
package = _load_module_file(dirname, filename)
if package is not None:
_load_package_components(package)
def _load_package_components(package):
if hasattr(package, 'get_component_list'):
try:
component_list = package.get_component_list()
except Exception, e:
cc.elog(e)
return
elif hasattr(package, '__path__'):
# This is indeed a package (not a module). Can try to autodiscover components in the
# package's path.
component_list = discover_package_components(package)
if not component_list:
return
else:
return
# Run through each component and verify it's OK.
for comp_class in component_list:
if _check_api_version(comp_class):
comp_name = comp_class.__module__
if not hasattr(comp_class, "component_version"):
cc.log(snap_log.LEVEL_WARN, "Component %s doesn't have component_version field. "
"Component author should add component_version for resource upgrades "
"to work properly." % comp_name)
cc_conf = cc.config.get_section('cc')
if 'component_conf_dir' in cc_conf and cc_conf['component_conf_dir']:
config_dir = cc_conf['component_conf_dir']
else:
config_dir = os.path.join(current_component_dir, "config")
config_file = os.path.join(config_dir, comp_name + ".conf")
if os.path.exists(config_file):
cc.log(snap_log.LEVEL_INFO, "Loading component configuration from " + config_file)
try:
cc.config.add_new_config(comp_name, config_file)
comp_class.config_dictionary = cc.config.get_section(comp_name)
comp_class().validate_config_file()
except SnapComponentConfigError, e:
cc.log(snap_log.LEVEL_ERR,
"Error loading component config file '%s': %s." % (config_file, str(e)))
cc.log(snap_log.LEVEL_ERR, "Ignoring invalid config file for component %s." % comp_name)
comp_class.config_dictionary = {}
except Exception, e:
cc.elog(e, "Error loading component config file '%s': %s" % (config_file, str(e)))
comp_class.config_dictionary = {}
else:
cc.log(snap_log.LEVEL_DEBUG, "Component configuration file not found in " + config_file)
component_map[comp_name] = comp_class
cc.log(snap_log.LEVEL_INFO, "Registered component %s" % comp_name)
def _check_api_version(comp_class):
try:
comp_name = comp_class.__module__
except Exception, e:
cc.elog(e, "Invalid component class value (%s)." % repr(comp_class))
return False
try:
(major, minor) = comp_class.api_version.split('.')
except AttributeError:
cc.log(snap_log.LEVEL_ERR,
"Error loading component '%s': component class missing api_version attribute." % comp_name)
return False
except ValueError, e:
cc.elog(e, "Error loading component '%s': incorrectly formatted api_version string." % comp_name)
return False
# XXX TODO: the version check should really be part of the ComponentAPI.
if major != '1':
cc.log(snap_log.LEVEL_ERR,
"Error loading component '%s': incompatible ComponentAPI version." % comp_name)
return False
return True
def _load_module_file(path, filename, package=None):
if filename == '__init__.py':
return None
filepath = os.path.join(path, filename)
if os.path.isdir(filepath):
if filename.find('.') != -1:
return None
elif '__init__.py' not in os.listdir(filepath):
# This is not a python package. Just ignore it.
return None
module_name = filename
elif os.path.isfile(filepath):
(module_name, module_ext) = os.path.splitext(filename)
if module_ext != '.py':
return None
else:
# Ignoring this entry since it's not a file or directory.
return None
try:
if package is None:
return __import__(module_name)
else:
full_name = "%s.%s" % (package.__name__, module_name)
return _load_module(full_name)
except Exception, e:
cc.elog(e, "Error importing component module '%s': %s." % (filepath, e))
return None
def _load_module(name, fromlist=[]):
parts = name.split('.')
module = __import__(name, fromlist=fromlist)
# __import__ returns the top-level package in a dotted name qualifier. Descend into the package
# hiearchy for the actual module desired.
for p in parts[1:]:
module = getattr(module, p)
return module
def discover_package_components(package, ignore_list=[]):
"""
Discover components in a package.
Searches for components in the top-level directory of the python package given. Does not attempt to consult
get_component_list() method, and therefore is useful as a default implementation of a component package's
get_component_list() method. The package's path is determined by package's __path__ attribute.
The search algorithm begins by listing the contents of the directory given by dirname. Each file ending with '.py'
and python package directory are imported. If the module/package contains a class by the same name as the
file/directory, the class is examined. If the class has an api_version attribute and the version indicated
is supported by this software, the class is considered a component and will be returned in the list of components
loaded.
Any file or directory listed in ignore_list is skipped at the initial directory listing search phase.
@param package: The python package to search for component modules. Can be either a package object or
a package qualifier string (i.e. "snaplogic.components").
@type package: package
@param ignore_list: A list of file or directory names to ignore in the discovery process.
@type ignore_list: list
@return: A list of component classes discovered by this function.
@rtype: list
@raise SnapValueError: There was a problem importing the package name given by package (when string).
"""
if type(package) in [str, unicode]:
package_name = package
try:
package = _load_module(package_name)
except Exception, e:
raise SnapException.chain(e,
SnapValueError("Error importing component package '%s': %s" % (package_name,
str(e))))
components = []
for path in package.__path__:
try:
contents = os.listdir(path)
except OSError, e:
cc.elog(e, "Error listing contents of directory '%s': %s." % (path, e))
return
for filename in contents:
if filename not in ignore_list and filename != 'config':
module = _load_module_file(path, filename, package)
if module is not None:
comp = _get_component_from_module(module)
if comp is not None:
components.append(comp)
return components
def _get_component_from_module(module):
class_name = module.__name__.split('.')[-1]
if not hasattr(module, class_name):
# Per ticket #1385, logging on modules in the same directory as components that are not also components
# can make it unnecessarily difficult or awkward developing component packages. Therefore this
# error is disabled now.
return None
try:
comp_class = getattr(module, class_name)
if _check_api_version(comp_class):
return comp_class
except Exception, e:
cc.elog(e,
"Error importing component module '%s': %s." % (module.__name__, e))
return None
def get_component_class(name):
"""
Get a component class by its component name.
@param name: Name of the component. For example snaplogic.component.CsvRead
@type name: str
@return: The python class that is derived from ComponentAPI and implements the component.
@rtype: Derived from L{ComponentAPI}
"""
if name not in component_map:
raise SnapObjNotFoundError("Component %s not found" % name)
return component_map[name]
def get_component_config(name):
"""
The config file contents for the component as a dictionary.
This dictionary is generated by the config parser. The config file for a component
is located at <components dir>/config/<name of the component>.conf. For example:
c:\snaplogic\components_dir\config\snaplogic.components.CsvRead.conf is the config
file for the CsvRead component.
If no config file is found, then empty dictionary is returned.
@param name: Name of the component. For example snaplogic.component.CsvRead
@type name: str
@return: Config dictionary.
@rtype: dict
"""
retval = {}
cc_conf = cc.config.get_section('cc')
for cc_conf_key in cc_conf.keys():
if cc_conf_key.startswith(snap_config.COMPONENT_CONFIG_PREFIX):
comp_key = cc_conf_key[len(snap_config.COMPONENT_CONFIG_PREFIX):]
retval[comp_key] = cc_conf[cc_conf_key]
if name in cc.config.list_sections():
comp_conf = cc.config.get_section(name)
retval.update(comp_conf)
return retval
def list_component_names():
return component_map.keys()
def get_registered_components():
return dict(component_map)
|