config.py (12635B)
1 # standard imports 2 import logging 3 import sys 4 import os 5 import tempfile 6 import configparser 7 import re 8 9 # external imports 10 import gnupg 11 12 # local imports 13 from confini.common import to_constant_name 14 15 logg = logging.getLogger('confini') 16 17 current_config = None 18 19 20 def set_current(conf, description=''): 21 global current_config 22 logg.debug('setting current config ({})'.format(description)) 23 current_config = conf 24 25 26 class Config: 27 28 default_censor_string = '***' 29 30 def __init__(self, default_dir, env_prefix=None, override_dirs=[], skip_doc=False): 31 self.parser = configparser.ConfigParser(strict=True) 32 self.skip_doc = skip_doc 33 self.doc = None 34 self.required = {} 35 self.censored = {} 36 self.store = {} 37 self.decrypt = [] 38 self.env_prefix = None 39 self.src_dirs = {} 40 self.__override_dirs = [] 41 self.__schema_dirs = [] 42 self.__processed = False 43 self.dirs = [] 44 45 if env_prefix != None: 46 logg.info('using prefix {} for environment variable override matches'.format(env_prefix)) 47 self.env_prefix = '{}_'.format(env_prefix) 48 49 if isinstance(default_dir, str): 50 default_dir = [default_dir] 51 for v in default_dir: 52 self.add_schema_dir(v) 53 54 self.__target_tmpdir = None 55 56 if isinstance(override_dirs, str): 57 override_dirs = [override_dirs] 58 elif override_dirs == None: 59 override_dirs = [] 60 for d in override_dirs: 61 self.add_override_dir(d) 62 63 64 def __collect(self): 65 self.collect_from_dirs(self.__schema_dirs) 66 for d in self.__override_dirs: 67 self.dirs.append(d) 68 69 70 def set_env_prefix(self, v): 71 self.env_prefix = v 72 73 74 def add_override_dir(self, v): 75 if not os.path.isdir(v): 76 raise OSError('{} is not a directory'.format(v)) 77 self.__override_dirs.append(v) 78 79 80 def add_schema_dir(self, v): 81 if not os.path.isdir(v): 82 raise OSError('{} is not a directory'.format(v)) 83 self.__schema_dirs.append(v) 84 85 86 def __clean(self): 87 if self.__target_tmpdir != None: 88 logg.debug('cleaning collection tmpdir {}'.format(self.__target_tmpdir.name)) 89 self.__target_tmpdir.cleanup() 90 91 92 def collect_from_dirs(self, dirs): 93 self.__target_tmpdir = tempfile.TemporaryDirectory() 94 self.dirs = [self.__target_tmpdir.name] 95 for i, d in enumerate(dirs): 96 for filename_in in os.listdir(d): 97 filename_out = None 98 if filename_in == '.confini': 99 filename_out = filename_in 100 elif re.match(r'.+\.ini$', filename_in) == None: 101 continue 102 else: 103 filename_out = '{}_{}'.format(i, filename_in) 104 in_filepath = os.path.join(d, filename_in) 105 out_filepath = os.path.join(self.dirs[0], filename_out) 106 fr = open(in_filepath, 'rb') 107 fw = open(out_filepath, 'wb') 108 fw.write(fr.read()) 109 fw.close() 110 fr.close() 111 logg.debug('base config {} will be processed as {}'.format(in_filepath, out_filepath)) 112 113 114 def add_decrypt(self, decrypter): 115 self.decrypt.append(decrypter) 116 117 118 def add(self, value, constant_name, exists_ok=False): 119 value_stored = self.store.get(constant_name) 120 if not self.is_as_none(value_stored): 121 if not exists_ok: 122 raise AttributeError('config key {} already exists'.format(constant_name)) 123 else: 124 if value_stored != value: 125 logg.debug('updating key {}'.format(constant_name)) 126 else: 127 return None 128 self.store[constant_name] = value 129 return value 130 131 132 def censor(self, identifier, section=None): 133 constant_name = '' 134 if section != None: 135 constant_name = to_constant_name(identifier, section) 136 else: 137 constant_name = identifier 138 self.censored[constant_name] = True 139 140 141 def require(self, directive, section): 142 if self.required.get(section) == None: 143 self.required[section] = [] 144 self.required[section].append(directive) 145 146 147 def validate(self): 148 for k in self.required.keys(): 149 for v in self.required[k]: 150 try: 151 _ = self.parser[k][v] 152 except: 153 return False 154 return True 155 156 157 def __sections_override(self, dct, dct_description, allow_empty=False): 158 for s in self.parser.sections(): 159 for k in self.parser[s]: 160 cn = to_constant_name(k, s) 161 self.override(cn, self.parser[s][k], dct, dct_description, allow_empty=True) 162 163 164 def dict_override(self, dct, dct_description, allow_empty=False): 165 for k in dct.keys(): 166 try: 167 self.override(k, self.store[k], dct, dct_description, allow_empty=allow_empty) 168 except KeyError: 169 logg.warning('override key {} have no match in config store'.format(k)) 170 171 172 def override(self, cn, v, dct, dct_description, allow_empty=False): 173 cn_env = cn 174 if self.env_prefix != None: 175 cn_env = self.env_prefix + cn 176 val = dct.get(cn_env) 177 if val == None: 178 val = self.store.get(cn, v) 179 elif val == '' and not allow_empty: 180 val = self.store.get(cn, v) 181 else: 182 logg.info('{} {} overrides {}'.format(dct_description, cn_env, cn)) 183 self.add(val, cn, exists_ok=True) 184 185 186 def set_dir(self, k, d): 187 logg.debug('set dir {} for key {}'.format(d, k)) 188 self.src_dirs[k] = d 189 190 191 def __process_doc_(self, d): 192 if self.skip_doc: 193 return 194 doc_fp = os.path.join(d, '.confini') 195 if self.doc == None: 196 from confini.doc import ConfigDoc 197 self.doc = ConfigDoc() 198 try: 199 self.doc.process(doc_fp) 200 except FileNotFoundError: 201 pass 202 203 204 def __collect_dir(self, out_dir): 205 for i, d in enumerate(self.dirs): 206 d = os.path.realpath(d) 207 if i == 0: 208 d_label = 'default' 209 else: 210 d_label = 'override #' + str(i) 211 tmp_out_dir = os.path.join(out_dir, str(i)) 212 os.makedirs(tmp_out_dir) 213 logg.debug('processing dir {} ({})'.format(d, d_label)) 214 tmp_out = open(os.path.join(tmp_out_dir, 'config.ini'), 'ab') 215 for filename in os.listdir(d): 216 if re.match(r'.+\.ini$', filename) == None: 217 logg.debug('skipping file {}/{}'.format(d, filename)) 218 continue 219 logg.debug('reading file {}/{}'.format(d, filename)) 220 f = open(os.path.join(d, filename), 'rb') 221 while 1: 222 data = f.read() 223 if not data: 224 break 225 tmp_out.write(data) 226 f.close() 227 tmp_out.close() 228 229 self.__process_doc_(d) 230 231 232 def __process_schema_dir(self, in_dir, allow_empty=False): 233 d = os.listdir(in_dir) 234 d.sort() 235 c = 0 236 237 # TODO: this will fail of sections/options are repeated. should first use individual parser instances to flatten to single file (perhaps in collect_from_dirs already) 238 for i, tmp_config_dir in enumerate(d): 239 tmp_config_dir = os.path.join(in_dir, tmp_config_dir) 240 for tmp_file in os.listdir(os.path.join(tmp_config_dir)): 241 tmp_config_file_path = os.path.join(tmp_config_dir, tmp_file) 242 if c == 0: 243 logg.debug('apply default parser for config directory {}'.format(self.dirs[i])) 244 self.parser.read(tmp_config_file_path) 245 for s in self.parser.sections(): 246 for so in self.parser.options(s): 247 k = to_constant_name(so, s) 248 v = self.parser.get(s, so) 249 logg.debug('default config set {}'.format(k)) 250 self.add(v, k, exists_ok=True) 251 self.set_dir(k, self.dirs[i]) 252 else: 253 logg.debug('apply override parser for config directory {}'.format(self.dirs[i])) 254 local_parser = configparser.ConfigParser(strict=True) 255 local_parser.read(tmp_config_file_path) 256 for s in local_parser.sections(): 257 for so in local_parser.options(s): 258 k = to_constant_name(so, s) 259 if not self.have(k): 260 raise KeyError('config overrides in {} defines key {} not present in default config {}'.format(self.dirs[i], k, self.dirs[0])) 261 v = local_parser.get(s, so) 262 logg.debug('checking {} {} {}'.format(k, s, so)) 263 if allow_empty or not self.is_as_none(v): 264 logg.debug('multi config file overrides {}'.format(k)) 265 self.add(v, k, exists_ok=True) 266 self.set_dir(k, self.dirs[i]) 267 c += 1 268 269 270 def process(self, set_as_current=False): 271 """Concatenates all .ini files in the config directory attribute and parses them to memory 272 """ 273 self.__collect() 274 275 tmp_dir = tempfile.mkdtemp() 276 logg.debug('using tmp processing dir {}'.format(tmp_dir)) 277 278 self.__collect_dir(tmp_dir) 279 280 self.__process_schema_dir(tmp_dir, allow_empty=True) 281 282 self.__sections_override(os.environ, 'environment variable', allow_empty=True) 283 284 if set_as_current: 285 set_current(self, description=self.dir) 286 287 self.__clean() 288 289 290 def _decrypt(self, k, v): 291 if len(self.decrypt) == 0: 292 return v 293 for decrypter in self.decrypt: 294 logg.debug('applying decrypt with {}'.format(str(decrypter))) 295 (v, r) = decrypter.decrypt(k, v) 296 if r: 297 return v 298 return v 299 300 301 def get(self, k, default=None): 302 v = self.store[k] 303 if v == None: 304 if default != None: 305 logg.debug('returning default value for empty value {}'.format(k)) 306 return default 307 if self.is_as_none(v): 308 if default != None: 309 logg.debug('returning default value for empty string value {}'.format(k)) 310 return default 311 else: 312 return None 313 314 return self._decrypt(k, v) 315 316 317 def remove(self, k, strict=True): 318 removes = [] 319 if strict: 320 removes = [k] 321 else: 322 l = len(k) 323 re_s = r'^' + k 324 for v in self.all(): 325 if len(v) >= l and re.match(re_s, v): 326 removes.append(v) 327 for v in removes: 328 del self.store[v] 329 logg.debug('removing key: {}'.format(v)) 330 331 332 def have(self, k): 333 try: 334 v = self.store[k] 335 return True 336 except KeyError: 337 return False 338 339 340 def all(self): 341 return list(self.store.keys()) 342 343 344 def true(self, k): 345 v = self.store.get(k) 346 if type(v).__name__ == 'bool': 347 return v 348 d = self._decrypt(k, v) #, self.src_dirs.get(k)) 349 if d == None: 350 return False 351 if d.lower() not in ['true', 'false', '0', '1', 'on', 'off']: 352 raise ValueError('{} not a boolean value'.format(k)) 353 return d.lower() in ['true', '1', 'on'] 354 355 356 def apply_censor(self, k): 357 try: 358 _ = self.censored[k] 359 return self.default_censor_string 360 except KeyError: 361 return self.store[k] 362 363 364 def __str__(self): 365 ls = [] 366 for k in self.store.keys(): 367 v = self.apply_censor(k) 368 ls.append('{}={}'.format(k, v)) 369 370 return '\n'.join(ls) 371 372 373 def __repr__(self): 374 return "<Config '{}'>".format(self.dir) 375 376 377 @classmethod 378 def is_as_none(cls, v): 379 if isinstance(v, str) and v == '': 380 return True 381 if v == None: 382 return True 383 384 385 def config_from_environment(): 386 config_dir = config_dir_from_environment() 387 c = Config(config_dir) 388 c.process() 389 return c 390 391 392 def config_dir_from_environment(): 393 return os.environ.get('CONFINI_DIR')