#
# SnapLogic - Open source data services
#
# Copyright (C) 2009, SnapLogic, Inc. All rights reserved.
#
# See http://www.snaplogic.org for more information about
# the SnapLogic project.
#
# This program is free software, distributed under the terms of
# the GNU General Public License Version 2. See the LEGAL file
# at the top of the source tree.
#
# "SnapLogic" is a trademark of SnapLogic, Inc.
#
# $
#
# $Id: snap_config.py 10320 2009-12-23 23:39:58Z dmitri $
"""
The config object.
This is a singleton, which gets instantiated once by each process.
A module within each process may call get_instance() - which is a
module level method here - to get a reference to the once instance.
If other modules need the config object while they are being
imported/loaded then the importer better make sure the config
object already exists before that other module is imported.
There is also a parsing function for config files included here.
"""
import os
from snaplogic.common.snap_exceptions import *
from snaplogic.common import expand_variables,snap_params
from configobj import ConfigObj
# Some computed attributes are made available on the module level.
enforce_singleton = True
"""For testing, can be set to False."""
_instance = None
# Those values are used by the creator of the SnapConfig object to
# indicate which default values they would like to have.
MAIN_SERVER_DEFAULT = "MAIN_SERVER"
CC_DEFAULT = "CC"
MANAGEMENT_DEFAULT = "MANAGEMENT_SERVER"
COMPONENT_CONFIG_PREFIX = "default_component_"
from snaplogic.common.snap_log import LEVEL_ERR,LEVEL_INFO
def get_instance():
"""
Return the single instance of the config object in this process.
If no instance exists yet then it returns None.
"""
return _instance
def parse_bool(bool_str):
"""
Parses a boolean flag string from a config file.
Take a boolean string value for a config entry and convert it to its native python bool value.
Using this method to parse all boolean values from the config file will allow a consistent feel
in the file.
@param bool_str: A string representation of a bool value as read from config file.
@type bool_str: str
@return: The boolean value represented by the string.
@rtype: bool
@raise SnapValueError: The bool_str parameter does not represent a recognized boolean value.
"""
bool_str = bool_str.lower()
if bool_str == 'yes':
return True
elif bool_str == 'no':
return False
else:
raise SnapValueError("Invalid boolean value: " + bool_str)
class SnapConfig(object):
"""
The config object for the Snap server.
The config object is a singleton. The data contained in the
object model reflects the hierarchies of the INI file format.
Thus, you can get handles to section specific dictionaries.
"""
# This dictionary specifies the mandatory parameters in each section.
# Note that these are not the mandatory parameters that need to be
# specified in the config file. Instead, these are ALL the parameters
# that the system in one way or the other will rely on. For many of
# them we have default values (see below), so they don't need to be
# set specifically in the config file. However, by just giving a
# complete list here, we can merely check later on whether this parameter
# is present in the section or not. So, if we ever take things out of
# the default value dictionary they are automatically cought if someone
# doesn't set them in the config file.
_mandatory_params = dict(main = [
"log_dir", "log_level", "max_log_size", "log_backup_count", "disable_logging",
"state_dir", "server_address",
"server_proxy_uri", "server_hostname", "server_port", "polling_interval",
"explorer_uri", "static_dir", "license_file", "pipe_to_http_uri_prefix",
"console_output", "auth_file_config", "auth_file_passwords",
"credentials_store", "runtime_entry_timeout",
"admin_mode" # See comment for 'admin_mode' further below
],
mgmt_server = [
"static_dir", "hostname", "port", "favicon_path"
],
component_container = [],
repository = [
"type", "path", "host", "db", "user", "password"
],
data_cache = [
"cache_dir", "cache_timeout", "cache_size",
"high_water_mark", "low_water_mark"
],
# The 'per_cc" section is a bit special. It is not present at the
# top level of the config, but is a sub-section of the CC section.
# It is not checked on the main server at all, but instead only
# by the CC process.
# There is some code further down which deals with this special
# case.
per_cc = [
"log_dir", "log_level", "max_log_size", "log_backup_count", "disable_logging",
"console_output", "component_dirs",
"cc_address", "cc_port", "cc_hostname", "cc_proxy_uri",
"cc_no_proxy", "cc_http_proxy",
"cc_http_proxy_port", "cc_http_proxy_realm", "cc_http_proxy_host",
"cc_http_proxy_username", "cc_http_proxy_password", "cc_ftp_proxy", "cc_ftp_proxy_port",
"type_check_records", "optimize_streams", "max_stream_buffer_size",
"component_conf_dir", COMPONENT_CONFIG_PREFIX + "root_directory",
"runtime_entry_timeout", snap_params.TRACE_DATA_CONFIG
])
_optional_params = dict(
# This is a special section, which is not present at the top
# level of the config file. Instead, it is a possible (optional)
# subsection of the notification section. There is some code
# further below, which handles this special case.
notification_email = [
"smtp_server", "smtp_use_tls", "smtp_login", "smtp_password",
"from", "to", "subject_prefix", "allow_user_destination"
],
notification_file_write = [
"filename", "root_directory", "allow_user_destination"
],
)
# Some of the mandatory parameters can have hard coded default values. Those
# are listed in this dictionary here. Other parameters cannot have default
# values and require them to be set in the config file.
_default_config_main = {
"main" : {
"log_dir" : "$SNAP_HOME/logs",
"log_level" : LEVEL_INFO,
"disable_logging" : None,
"max_log_size" : None,
"log_backup_count" : None,
"state_dir" : "state",
"server_proxy_uri" : "",
"server_address" : "0.0.0.0",
# Default for server name doesn't make sense, since the server
# name is a complex thing to find out. The installer does that
# and therefore is the one that writes the server name into the
# config file. However, if the user specifies a server_proxy_uri
# then the server_hostname parameter is not needed. That is why we
# do a special check for that case in the server code itself,
# while the config object here is just going to allow to keep
# the server_hostname undefined.
"server_hostname" : "",
"server_port" : "8088",
"polling_interval" : "60",
"runtime_entry_timeout" : "300",
# Explorer URI depends on the server name and therefore, we cannot
# give a default for it either.
#"explorer_uri" : "http://127.0.0.1:8081/designer/explorer.html",
"static_dir" : "static",
"license_file" : "$SNAP_HOME/../config/license.txt",
"pipe_to_http_uri_prefix" : "/feed",
"console_output" : "no",
# There are no default values for the auth files (password and access).
# The server will not start without them, so the admin is forced to provide
# and configure those.
"credentials_store" : "",
"admin_mode" : False # Admin mode is not set in the config file,
# but with a command line switch for the server.
# Since it's part of the configuration, though,
# this is a good place to put it.
},
"mgmt_server" : {
"static_dir" : "static",
"hostname" : "0.0.0.0",
"port" : "8081",
"favicon_path" : ""
},
"component_container" : {}, # There are no defaults for the 'component_container' section,
# since it just contains CC definitions, which only the admin
# can know.
"data_cache" : {
"cache_dir" : "$SNAP_HOME/cache_dir",
"cache_timeout" : "300",
"cache_size" : "10MB",
"high_water_mark" : "90",
"low_water_mark" : "60"
},
"repository" : {
"type" : "sqlite",
"path" : "repository/repository.db",
"host" : None,
"db" : None,
"user" : None,
"password" : None
}
}
# All the parameters that should be standardized to lower-case
_make_lowercase = [ "cc_proxy_uri", "cc_address", "cc_hostname", "server_proxy_uri", "server_hostname", "server_hostname",
"smtp_server", "smtp_use_ttl", "allow_user_destination" ]
"""
Add that to _default_config_main if we want defaults for the repository.
"""
# Default values for sub-section, which are ONLY put into place if the
# subsection has actually been defined. These default values are only
# written into the actual config, if those values have not been specified
# by the user. So, while normal default values are written into the config
# first and are then overwritten by whatever the user defines, here we have
# it the other way around.
_default_subsection_config = dict(notification_email = {
#"smtp_server" : "", The server is definitely needed
"smtp_use_tls" : "no",
"smtp_login" : "",
"smtp_password" : "",
#"from" : "", A from address needs to be defined
"to" : "",
"subject_prefix" : "SnapLogic email notification: ",
"allow_user_destination" : "yes",
},
notification_file_write = {
#"filename" : "", Filename and root directory...
#"root_directory" : "", ... definitely need to be provided.
"allow_user_destination" : "yes",
})
_default_config_cc = {
"cc" : {
"log_dir" : "$SNAP_HOME/logs",
"log_level" : LEVEL_INFO,
"disable_logging" : None,
"console_output" : "no",
"type_check_records" : "yes",
"cc_address" : "0.0.0.0",
"cc_proxy_uri" : "",
"cc_hostname" : "",
"cc_http_proxy" : "",
"cc_no_proxy" : "",
"cc_http_proxy_port" : "",
"cc_http_proxy_realm" : "",
"cc_http_proxy_host" : "",
"cc_http_proxy_username" : "",
"cc_http_proxy_password" : "",
"cc_ftp_proxy" : "",
"cc_ftp_proxy_port" : "",
"optimize_streams" : "yes",
"max_stream_buffer_size" : "1000000",
"component_conf_dir" : "",
"runtime_entry_timeout" : "300",
snap_params.TRACE_DATA_CONFIG : "",
COMPONENT_CONFIG_PREFIX + "root_directory" : "$SNAP_HOME" if os.environ.has_key('SNAP_HOME') else os.getcwd()
}
}
_default_management = {
"mgmt_server" : {
"static_dir" : "$SNAP_HOME/designer"
}
}
def __init__(self, fname=None, param_dict=None, default=None, common_dict={}, instance_only=False,
exclude_list=None):
"""
Initialize the config object.
Since this is a singleton, it will throw an exception if an instance
of the object already exists. The instance_only parameter can be used
to create an instance that is not assigned to the global parameter. If
this parameter is true, no singleton enforcement will occur.
If a filename is provided then it will be taken as the filename of an INI
file that is to be read and parsed. Alternative to a filename, the creator
of the object may also specify a ready-made dictionary (sections) of
dictionaries (parameters in the sections). Only one of those two parameters
may be specified at any given time.
If no filename AND no dictionary is provided then a couple of parameters are
initialized to default values.
The common_dict provides a means of process-global values not necessarily found in the
config file. The section becomes a standard place for any code within the process to look
for important values not available in the config file or not known until runtime. These values
are also expected to be used in the various processes, but the actual value may differ
depending on the process context.
Note also that some computed values are placed as module level attributes
into the config module.
@param fname: Optional name of an INI config file.
@type fname: string
@param param_dict: Dictionary with values for the config object. This must
be a dictionary of dictionary, to represent the sections
and their parameters.
@type param_dict: dictionary
@param default: Indicate the default settings you want. Two values are supported:
"MAIN_SERVER" and "CC". Each results in a different set of default
values. If 'None' is specified, no default values will be chosen.
@type default: string
@param common_dict: Common values used within the entire process.
@type common_dict: dictionary
@param instance_only: A flag indicating the new object should not be made the global instance or
perform singleton check.
@type instance_only: bool
@param exclude_list: List of sections or keys that shouldn't be expanded.
(by expansion we mean substitution of variables like SNAP_HOME).
E.g. SnapLogic server wants to pass the "component_container" section right
to the CC without expanding the variables: because CC may be
running on a different machine it may have a different value
for SNAP_HOME.
If a section is excluded all its subsections are excluded as well.
@type exclude_list: list
"""
global _instance, MAIN_SERVER_DEFAULT, CC_DEFAULT, MANAGEMENT_DEFAULT
# Make sure this remains a singleton
if not instance_only and _instance and enforce_singleton:
raise SnapObjExistsError("Attempting creation of duplicate config singleton.")
# Most values have sensible default settings, which we specify here.
# Later, as the user-specified config is processed, we will be overwriting
# those defaults one by one with what's specified.
if not default:
self._config_dict = {}
elif default == MAIN_SERVER_DEFAULT:
self._config_dict = SnapConfig._default_config_main
elif default == CC_DEFAULT:
self._config_dict = SnapConfig._default_config_cc
elif default == MANAGEMENT_DEFAULT:
self._config_dict = SnapConfig._default_management
else:
raise SnapValueError("Unknown config default identifier '%s'. Only understand '%s', '%s', and %s." %
(default, MAIN_SERVER_DEFAULT, CC_DEFAULT, MANAGEMENT_DEFAULT))
# If specified, read and parse the config file
if fname and not param_dict:
try:
new_config = ConfigObj(fname, {'file_error' : True})
except Exception, e:
raise SnapFormatError("Failed to parse config file %s. Exception: %s" % (fname, e))
elif param_dict and not fname:
new_config = param_dict
elif param_dict and fname:
raise SnapValueError("Config object cannot have file-name AND dictionary as parameters.")
else:
# No new configuration was specified (neither file nor dictionary)
new_config = {}
# Store the common values into the 'common' section.
if 'common' not in new_config:
new_config['common'] = common_dict
else:
raise SnapValueError("Specified config contains a 'common' section. Section reserved for internal use.")
# Now we 'merge' the new configuration with the existing default config
for section in new_config:
if not self._config_dict.has_key(section) or not isinstance(new_config[section], dict):
# If we don't have that section in the default yet, we take the new section as is.
# We also do the same if the 'section' is not a dictionary at all, but just a normal
# value. Rare, but can happen.
# We only make sure that the 'main' section is not overwritten by such a value.
if section == 'main' and self._config_dict.has_key(section):
raise SnapValueError("Section 'main' cannot be overwritten by a scalar value.")
self._config_dict[section] = new_config[section]
else:
# Most sections are dictionaries...
for element in new_config[section]:
# Here we could call section specific merge handlers one day...
self._config_dict[section][element] = new_config[section][element]
# Expand variables in the config parameters, except the ones which
# shouldn't be expanded (e.g. replace SNAP_HOME with the value of environment variable SNAP_HOME)
self._section_expand_vars(self._config_dict, exclude_list)
# A small helper function
def _section_check(section_name, dict_to_check, is_mandatory, mandatory_params=None, default_dict=None):
# This is a small helper function, which makes sure a section
# is present in the specified dictionary and that it is indeed
# of type dictionary.
# It also checks whether all the specified mandatory parameters
# are present in the section.
# It also checks whether there are any unknown parameters specified.
# Returns the section dictionary.
# If a default-dict is defined then the function tries to fill in any missing
# mandatory parameters from the default dictionary.
try:
section = dict_to_check[section_name]
except:
if is_mandatory:
raise SnapValueError("Mandatory section '%s' is missing from the server configuration." % section_name)
else:
return None
if not isinstance(section, dict):
raise SnapValueError("Mandatory section '%s' in server configuration has been overwritten with a scalar value." % section_name)
if mandatory_params:
for param in mandatory_params:
if param not in section:
if default_dict and param in default_dict:
section[param] = default_dict[param]
else:
raise SnapValueError("Mandatory parameter '%s' is missing from section '%s' in server configuration." % (param, section_name))
for param in section:
if isinstance(section[param], dict) and section_name == "component_container":
# The CC sections can be defined with some user defined name. No point
# checking that against the known parameters. We can detect them, because
# of the section we are in and because they are dictionaries, which makes
# it easy.
continue
if param not in mandatory_params:
raise SnapValueError("Unknown parameter '%s' in section '%s' of server configuration." % (param, section_name))
return section
if default == MAIN_SERVER_DEFAULT:
self._inherit_server_settings()
if "notification" in self._config_dict:
# Assemble a list of the known notification sub-sections (it's a bit special,
# so we need some special-case code for that.
known_notification_sections = []
for n in SnapConfig._optional_params:
if n.startswith("notification_"):
known_notification_sections.append(n.split("_",1)[1])
# Now check whether all the notification sections specified in
# the config file are actually known to us.
for s in self._config_dict["notification"]:
if s not in known_notification_sections:
raise SnapValueError("Unknown sub-section '%s' in section 'notification' of server configuration." % (s))
# Check that the required parameters are present in all the defined sections.
for section in SnapConfig._mandatory_params:
# Most sections that are present in the mandatory-params dictionary
# are also present at the top level of the server configuration.
# The 'per_cc' section is an exception. It contains the mandatory
# values for each CC definition within the CC section. The per-CC
# section is checked and filled with defaults by the CC process,
# so as we are processing the main section, we should not check this.
if section != "per_cc":
_section_check(section, self._config_dict, True, SnapConfig._mandatory_params[section])
for section in SnapConfig._optional_params:
if section.startswith("notification_") and "notification" in self._config_dict:
# The notification subsections also require some special handling
section_name, sub_section_name = section.split("_", 1)
if sub_section_name in self._config_dict[section_name]:
_section_check(sub_section_name, self._config_dict["notification"], False,
SnapConfig._optional_params[section], SnapConfig._default_subsection_config[section])
elif default == CC_DEFAULT:
# We are only here if we are executing as part of the CC process. So, we
# can specifically and easily check that we have all the required parameters.
cc_section = _section_check("cc", self._config_dict, True, SnapConfig._mandatory_params["per_cc"])
if not instance_only and enforce_singleton and _instance is None:
_instance = self
# Perform lower-case conversion on those elements that have been specified.
# We traverse the config and all its sub-sections.
def make_lower(d):
for p in d:
if isinstance(d[p], dict):
make_lower(d[p])
else:
if p in SnapConfig._make_lowercase:
d[p] = d[p].lower()
make_lower(self._config_dict)
def _inherit_server_settings(self):
"""
Inherit some of the server config options in all the CC sections.
This function must be called on by the main process.
"""
cc_sections = self._config_dict["component_container"]
for cc, cc_section in cc_sections.items():
for opt in ("max_log_size", "log_backup_count"):
if opt in cc_section:
SnapValueError("The option '%s' cannot be specified in the CC section" % opt)
# The only way this option is set, is here, by copying its value from the main
# section.
cc_section[opt] = self._config_dict["main"][opt]
def _section_expand_vars(self, dictionary, exclude_list = None):
"""
This function takes a dictionary of configuration parameters
that can either contain primitive values for keys, or other dictionaries.
We expand variables in all values going recursive if needed.
E.g. if we find
component_dirs = $SNAP_HOME
we may replace it with
component_dirs = /opt/snaplogic/2.0.5
if SNAP_HOME = /opt/snaplogic/2.0.5
@param dictionary: dictionary of configuration parameters and their values
@type dictionary: dict
@param exclude_list: list of keys/sections that we should ignore, e.g. [ "cc" ]
If a section is excluded all its subsections are excluded as well.
@type exclude_list: list
"""
for key in dictionary:
if exclude_list is not None and key in exclude_list:
# Values in this section shouldn't be expanded
continue
# Expand variables, e.g. SNAP_HOME
value = dictionary[key]
if isinstance(value, dict):
# This section contains sections so go recursive
self._section_expand_vars(value, exclude_list)
elif isinstance(value, str) or isinstance(value, unicode):
dictionary[key] = expand_variables(value)
elif isinstance(value, list):
# It's a list of values. Note we don't support recursion here
for index, item in enumerate(value):
value[index] = expand_variables(value[index])
def list_sections(self):
"""
Return a list with the names of all sections.
@return: List of names of all sections.
@rtype: list
"""
return self._config_dict.keys()
def get_section(self, sname):
"""
Return handle to the dictionary of the specified section.
The INI config file format supports sections. Each section is
represented as a dictionary. This function returns a handle
to the dictionary of the specified name.
@param sname: Name of a section in the config INI file.
@type sname: string
@return: Dictionary for that section.
@rtype: dictionary
"""
try:
return self._config_dict[sname]
except KeyError, e:
raise SnapException.chain(e, SnapObjNotFoundError("A config section with the name '%s' does not exist." % sname))
def add_to_section(self, sname, name_value_pair):
"""
Add a name value pair to an existing section.
@param sname: Name of section.
@type sname string
@param name_value_pair: A tuple containing a name/value pair.
Name has to be a string, value can be anything.
@type name_value_pair: tuple
"""
if not self._config_dict.has_key(sname):
raise SnapObjNotFoundError("A config section with the name '%s' does not exist." % sname)
section_dict = self._config_dict[sname]
try:
(name, value) = name_value_pair
except Exception, e:
raise SnapException.chain(e, SnapValueError("Not a valid name value pair."))
if section_dict.has_key(name) and enforce_singleton:
raise SnapValueError("There already is a key '%s' in the config dictionary." % name)
section_dict[name] = value
def add_new_config(self, sname, config_file):
"""
Add a new section based on contents of specified config file.
@param sname: Name of new section.
@type sname: string
@param config_file: Name of additional config file.
@type config_file: string
"""
if sname in self.list_sections():
raise SnapValueError("The section '%s' already exists in the config object." % sname)
try:
config_dict = ConfigObj(config_file, {'file_error' : True})
except Exception, e:
raise SnapFormatError("Failed to parse config file %s. Exception: %s" % (config_file, e))
self._config_dict[sname] = config_dict
|