confini

Parse and merge multiple ini files in python3
git clone git://git.defalsify.org/python-confini.git
Log | Files | Refs | README | LICENSE

config.py (12635B)


      1 # standard imports
      2 import logging
      3 import sys
      4 import os
      5 import tempfile
      6 import configparser
      7 import re
      8 
      9 # external imports
     10 import gnupg
     11 
     12 # local imports
     13 from confini.common import to_constant_name
     14 
     15 logg = logging.getLogger('confini')
     16 
     17 current_config = None
     18 
     19 
     20 def set_current(conf, description=''):
     21     global current_config
     22     logg.debug('setting current config ({})'.format(description))
     23     current_config = conf 
     24 
     25 
     26 class Config:
     27 
     28     default_censor_string = '***'
     29 
     30     def __init__(self, default_dir, env_prefix=None, override_dirs=[], skip_doc=False):
     31         self.parser = configparser.ConfigParser(strict=True)
     32         self.skip_doc = skip_doc
     33         self.doc = None
     34         self.required = {}
     35         self.censored = {}
     36         self.store = {}
     37         self.decrypt = []
     38         self.env_prefix = None
     39         self.src_dirs = {}
     40         self.__override_dirs = []
     41         self.__schema_dirs = []
     42         self.__processed = False
     43         self.dirs = []
     44 
     45         if env_prefix != None:
     46             logg.info('using prefix {} for environment variable override matches'.format(env_prefix))
     47             self.env_prefix = '{}_'.format(env_prefix)
     48 
     49         if isinstance(default_dir, str):
     50             default_dir = [default_dir]
     51         for v in default_dir:
     52             self.add_schema_dir(v)
     53 
     54         self.__target_tmpdir = None
     55 
     56         if isinstance(override_dirs, str):
     57             override_dirs = [override_dirs]
     58         elif override_dirs == None:
     59             override_dirs = []
     60         for d in override_dirs:
     61             self.add_override_dir(d)
     62 
     63 
     64     def __collect(self):
     65         self.collect_from_dirs(self.__schema_dirs)
     66         for d in self.__override_dirs:
     67             self.dirs.append(d)
     68 
     69 
     70     def set_env_prefix(self, v):
     71         self.env_prefix = v
     72 
     73 
     74     def add_override_dir(self, v):
     75         if not os.path.isdir(v):
     76             raise OSError('{} is not a directory'.format(v))
     77         self.__override_dirs.append(v)
     78 
     79 
     80     def add_schema_dir(self, v):
     81         if not os.path.isdir(v):
     82             raise OSError('{} is not a directory'.format(v))
     83         self.__schema_dirs.append(v)
     84 
     85 
     86     def __clean(self):
     87         if self.__target_tmpdir != None:
     88             logg.debug('cleaning collection tmpdir {}'.format(self.__target_tmpdir.name))
     89             self.__target_tmpdir.cleanup() 
     90 
     91 
     92     def collect_from_dirs(self, dirs):
     93         self.__target_tmpdir = tempfile.TemporaryDirectory()
     94         self.dirs = [self.__target_tmpdir.name]
     95         for i, d in enumerate(dirs):
     96             for filename_in in os.listdir(d):
     97                 filename_out = None
     98                 if filename_in == '.confini':
     99                     filename_out = filename_in
    100                 elif re.match(r'.+\.ini$', filename_in) == None:
    101                     continue
    102                 else:
    103                     filename_out = '{}_{}'.format(i, filename_in)
    104                 in_filepath = os.path.join(d, filename_in)
    105                 out_filepath = os.path.join(self.dirs[0], filename_out)
    106                 fr = open(in_filepath, 'rb')
    107                 fw = open(out_filepath, 'wb')
    108                 fw.write(fr.read())
    109                 fw.close()
    110                 fr.close()
    111                 logg.debug('base config {} will be processed as {}'.format(in_filepath, out_filepath))
    112 
    113 
    114     def add_decrypt(self, decrypter):
    115         self.decrypt.append(decrypter)
    116 
    117 
    118     def add(self, value, constant_name, exists_ok=False):
    119         value_stored = self.store.get(constant_name)
    120         if not self.is_as_none(value_stored):
    121             if not exists_ok:
    122                 raise AttributeError('config key {} already exists'.format(constant_name))
    123             else:
    124                 if value_stored != value:
    125                     logg.debug('updating key {}'.format(constant_name))
    126                 else:
    127                     return None
    128         self.store[constant_name] = value
    129         return value
    130 
    131 
    132     def censor(self, identifier, section=None):
    133         constant_name = ''
    134         if section != None:
    135             constant_name = to_constant_name(identifier, section)
    136         else:
    137             constant_name = identifier
    138         self.censored[constant_name] = True
    139 
    140 
    141     def require(self, directive, section):
    142         if self.required.get(section) == None:
    143             self.required[section] = []
    144         self.required[section].append(directive)
    145 
    146 
    147     def validate(self):
    148         for k in self.required.keys():
    149             for v in self.required[k]:
    150                 try:
    151                     _ = self.parser[k][v]
    152                 except:
    153                     return False
    154         return True
    155 
    156 
    157     def __sections_override(self, dct, dct_description, allow_empty=False):
    158         for s in self.parser.sections():
    159             for k in self.parser[s]:
    160                 cn = to_constant_name(k, s)
    161                 self.override(cn, self.parser[s][k], dct, dct_description, allow_empty=True)
    162 
    163 
    164     def dict_override(self, dct, dct_description, allow_empty=False):
    165         for k in dct.keys():
    166             try:
    167                 self.override(k, self.store[k], dct, dct_description, allow_empty=allow_empty)
    168             except KeyError:
    169                 logg.warning('override key {} have no match in config store'.format(k))
    170 
    171 
    172     def override(self, cn, v, dct, dct_description, allow_empty=False):
    173         cn_env = cn
    174         if self.env_prefix != None:
    175             cn_env = self.env_prefix + cn
    176         val = dct.get(cn_env)
    177         if val == None:
    178             val = self.store.get(cn, v)
    179         elif val == '' and not allow_empty:
    180             val = self.store.get(cn, v)
    181         else:
    182             logg.info('{} {} overrides {}'.format(dct_description, cn_env, cn))
    183         self.add(val, cn, exists_ok=True)
    184 
    185 
    186     def set_dir(self, k, d):
    187         logg.debug('set dir {} for key {}'.format(d, k))
    188         self.src_dirs[k] = d
    189 
    190 
    191     def __process_doc_(self, d):
    192         if self.skip_doc:
    193             return
    194         doc_fp = os.path.join(d, '.confini')
    195         if self.doc == None:
    196             from confini.doc import ConfigDoc
    197             self.doc = ConfigDoc()
    198         try:
    199             self.doc.process(doc_fp)
    200         except FileNotFoundError:
    201             pass
    202 
    203 
    204     def __collect_dir(self, out_dir):
    205         for i, d in enumerate(self.dirs):
    206             d = os.path.realpath(d)
    207             if i == 0:
    208                 d_label = 'default'
    209             else:
    210                 d_label = 'override #' + str(i)
    211             tmp_out_dir = os.path.join(out_dir, str(i))
    212             os.makedirs(tmp_out_dir)
    213             logg.debug('processing dir {} ({})'.format(d, d_label))
    214             tmp_out = open(os.path.join(tmp_out_dir, 'config.ini'), 'ab')
    215             for filename in os.listdir(d):
    216                 if re.match(r'.+\.ini$', filename) == None:
    217                     logg.debug('skipping file {}/{}'.format(d, filename))
    218                     continue
    219                 logg.debug('reading file {}/{}'.format(d, filename))
    220                 f = open(os.path.join(d, filename), 'rb')
    221                 while 1:
    222                     data = f.read()
    223                     if not data:
    224                         break
    225                     tmp_out.write(data)
    226                 f.close()
    227             tmp_out.close()
    228 
    229             self.__process_doc_(d)
    230 
    231 
    232     def __process_schema_dir(self, in_dir, allow_empty=False):
    233         d = os.listdir(in_dir)
    234         d.sort()
    235         c = 0
    236 
    237         # TODO: this will fail of sections/options are repeated. should first use individual parser instances to flatten to single file (perhaps in collect_from_dirs already)
    238         for i, tmp_config_dir in enumerate(d):
    239             tmp_config_dir = os.path.join(in_dir, tmp_config_dir)
    240             for tmp_file in os.listdir(os.path.join(tmp_config_dir)):
    241                 tmp_config_file_path = os.path.join(tmp_config_dir, tmp_file)
    242                 if c == 0:
    243                     logg.debug('apply default parser for config directory {}'.format(self.dirs[i]))
    244                     self.parser.read(tmp_config_file_path)
    245                     for s in self.parser.sections():
    246                         for so in self.parser.options(s):
    247                             k = to_constant_name(so, s)
    248                             v = self.parser.get(s, so)
    249                             logg.debug('default config set {}'.format(k))
    250                             self.add(v, k, exists_ok=True)
    251                             self.set_dir(k, self.dirs[i])
    252                 else:
    253                     logg.debug('apply override parser for config directory {}'.format(self.dirs[i]))
    254                     local_parser = configparser.ConfigParser(strict=True)
    255                     local_parser.read(tmp_config_file_path)
    256                     for s in local_parser.sections():
    257                         for so in local_parser.options(s):
    258                             k = to_constant_name(so, s)
    259                             if not self.have(k):
    260                                 raise KeyError('config overrides in {} defines key {} not present in default config {}'.format(self.dirs[i], k, self.dirs[0]))
    261                             v = local_parser.get(s, so)
    262                             logg.debug('checking {} {} {}'.format(k, s, so))
    263                             if allow_empty or not self.is_as_none(v):
    264                                 logg.debug('multi config file overrides {}'.format(k))
    265                                 self.add(v, k, exists_ok=True)
    266                                 self.set_dir(k, self.dirs[i])
    267             c += 1
    268 
    269 
    270     def process(self, set_as_current=False):
    271         """Concatenates all .ini files in the config directory attribute and parses them to memory
    272         """
    273         self.__collect()
    274 
    275         tmp_dir = tempfile.mkdtemp()
    276         logg.debug('using tmp processing dir {}'.format(tmp_dir))
    277       
    278         self.__collect_dir(tmp_dir)
    279 
    280         self.__process_schema_dir(tmp_dir, allow_empty=True)
    281 
    282         self.__sections_override(os.environ, 'environment variable', allow_empty=True)
    283 
    284         if set_as_current:
    285             set_current(self, description=self.dir)
    286 
    287         self.__clean()
    288 
    289 
    290     def _decrypt(self, k, v):
    291         if len(self.decrypt) == 0:
    292             return v
    293         for decrypter in self.decrypt:
    294             logg.debug('applying decrypt with {}'.format(str(decrypter)))
    295             (v, r) = decrypter.decrypt(k, v)
    296             if r:
    297                 return v
    298         return v
    299 
    300 
    301     def get(self, k, default=None):
    302         v = self.store[k]
    303         if v == None:
    304             if default != None:
    305                 logg.debug('returning default value for empty value {}'.format(k))
    306             return default
    307         if self.is_as_none(v): 
    308             if default != None:
    309                 logg.debug('returning default value for empty string value {}'.format(k))
    310                 return default
    311             else:
    312                 return None
    313 
    314         return self._decrypt(k, v)
    315 
    316 
    317     def remove(self, k, strict=True):
    318         removes = []
    319         if strict:
    320             removes = [k]
    321         else:
    322             l = len(k)
    323             re_s = r'^' + k
    324             for v in self.all():
    325                 if len(v) >= l and re.match(re_s, v):
    326                     removes.append(v)
    327         for v in removes:
    328             del self.store[v]
    329             logg.debug('removing key: {}'.format(v))
    330 
    331 
    332     def have(self, k):
    333         try:
    334             v = self.store[k]
    335             return True
    336         except KeyError:
    337             return False
    338 
    339 
    340     def all(self):
    341         return list(self.store.keys())
    342 
    343 
    344     def true(self, k):
    345         v = self.store.get(k)
    346         if type(v).__name__ == 'bool':
    347             return v
    348         d = self._decrypt(k, v) #, self.src_dirs.get(k))
    349         if d == None:
    350             return False
    351         if d.lower() not in ['true', 'false', '0', '1', 'on', 'off']:
    352             raise ValueError('{} not a boolean value'.format(k))
    353         return d.lower() in ['true', '1', 'on']
    354 
    355 
    356     def apply_censor(self, k):
    357         try:
    358             _ = self.censored[k]
    359             return self.default_censor_string
    360         except KeyError:
    361             return self.store[k]
    362 
    363 
    364     def __str__(self):
    365         ls = []
    366         for k in self.store.keys():
    367             v = self.apply_censor(k)
    368             ls.append('{}={}'.format(k, v))
    369 
    370         return '\n'.join(ls)
    371 
    372 
    373     def __repr__(self):
    374         return "<Config '{}'>".format(self.dir)
    375 
    376 
    377     @classmethod
    378     def is_as_none(cls, v):
    379         if isinstance(v, str) and v == '':
    380             return True
    381         if v == None:
    382             return True
    383 
    384 
    385 def config_from_environment():
    386     config_dir = config_dir_from_environment()
    387     c = Config(config_dir)
    388     c.process()
    389     return c
    390 
    391 
    392 def config_dir_from_environment():
    393     return os.environ.get('CONFINI_DIR')