leveldir

Multi-level directory structure data stores in python3
git clone git://git.defalsify.org/python-leveldir.git
Log | Files | Refs | LICENSE

commit b928f0caec021f1809271de87ddffa5b3d88f73e
parent c15a09261ce9fbb2f8881a8ef2d74eba97f7e71e
Author: lash <dev@holbrook.no>
Date:   Wed, 26 Jan 2022 11:45:39 +0000

Activate directory prepare on existing dir, settable hex formatter

Diffstat:
MCHANGELOG | 5+++++
AWAIVER | 17+++++++++++++++++
AWARRANTY | 5+++++
Mleveldir/base.py | 28++++++++++++++++++++--------
Mleveldir/hex.py | 39+++++++++++++++++++++++++--------------
Mleveldir/numeric.py | 1+
Msetup.cfg | 2+-
Mtests/test_hexdir.py | 17+++++++++++++++++
8 files changed, 91 insertions(+), 23 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG @@ -1,3 +1,8 @@ +- 0.1.1 + * Add subdirectory entry mode for hexdir + * Add custom hex formatting for path generation +- 0.1.0 + * License change to WTFPL - 0.0.2 * make subdir values cumulative in numdir - 0.0.1 diff --git a/WAIVER b/WAIVER @@ -0,0 +1,17 @@ +# Copyright waiver for the python package "leveldir" + +I dedicate any and all copyright interest in this software to the +public domain. I make this dedication for the benefit of the public at +large and to the detriment of my heirs and successors. I intend this +dedication to be an overt act of relinquishment in perpetuity of all +present and future rights to this software under copyright law. + +To the best of my knowledge and belief, my contributions are either +originally authored by me or are derived from prior works which I have +verified are also in the public domain and are not subject to claims +of copyright by other parties. + +To the best of my knowledge and belief, no individual, business, +organization, government, or other entity has any copyright interest +in my contributions, and I affirm that I will not make contributions +that are otherwise encumbered. diff --git a/WARRANTY b/WARRANTY @@ -0,0 +1,5 @@ +This program is free software. It comes without any warranty, to + * the extent permitted by applicable law. You can redistribute it + * and/or modify it under the terms of the Do What The Fuck You Want + * To Public License, Version 2, as published by Sam Hocevar. See + * http://www.wtfpl.net/ for more details. diff --git a/leveldir/base.py b/leveldir/base.py @@ -10,12 +10,20 @@ class LevelDir: self.levels = levels self.entry_length = entry_length fi = None + self.__prepare_directory(self.path) + + self.__verify_directory() + + self.master_file = os.path.join(self.path, 'master') + + + def have(self, k): + fp = self.to_filepath(k) try: - fi = os.stat(self.path) - self.__verify_directory() + os.stat(fp) except FileNotFoundError: - LevelDir.__prepare_directory(self.path) - self.master_file = os.path.join(self.path, 'master') + return False + return True def __verify_directory(self): @@ -37,9 +45,13 @@ class LevelDir: return r - @staticmethod - def __prepare_directory(path): + @classmethod + def __prepare_directory(cls, path): os.makedirs(path, exist_ok=True) state_file = os.path.join(path, 'master') - f = open(state_file, 'w') - f.close() + try: + os.stat(state_file) + except FileNotFoundError: + f = open(state_file, 'w') + f.close() + return state_file diff --git a/leveldir/hex.py b/leveldir/hex.py @@ -11,17 +11,22 @@ from .base import LevelDir logg = logging.getLogger(__name__) +def default_formatter(hx): + return hx.upper() + + class HexDir(LevelDir): - def __init__(self, root_path, key_length, levels=2, prefix_length=0): + def __init__(self, root_path, key_length, levels=2, prefix_length=0, formatter=default_formatter): super(HexDir, self).__init__(root_path, levels, key_length + prefix_length) #self.path = root_path self.key_length = key_length self.prefix_length = prefix_length self.__levels = levels + 2 + self.formatter = formatter - def add(self, key, content, prefix=b''): + def __check(self, key, content, prefix): l = len(key) if l != self.key_length: raise ValueError('expected key length {}, got {}'.format(self.key_length, l)) @@ -32,9 +37,16 @@ class HexDir(LevelDir): raise ValueError('content must be bytes, got {}'.format(type(content).__name__)) if prefix != None and not isinstance(prefix, bytes): raise ValueError('prefix must be bytes, got {}'.format(type(content).__name__)) + + + def add(self, key, content, prefix=b''): + self.__check(key, content, prefix) key_hex = key.hex() entry_path = self.to_filepath(key_hex) + return self.__add(entry_path, key, content, key_hex, prefix=prefix) + + def __add(self, entry_path, key, content, display_key, prefix=b''): c = self.count() os.makedirs(os.path.dirname(entry_path), exist_ok=True) @@ -48,11 +60,19 @@ class HexDir(LevelDir): f.write(key) f.close() - logg.debug('created new hexdir entry {} idx {} in {}'.format(key_hex, c, entry_path)) + logg.debug('created new hexdir entry {} idx {} in {}'.format(display_key, c, entry_path)) return (c, entry_path) + def add_dir(self, file_key, key, content, prefix=b''): + self.__check(key, content, prefix) + key_hex = key.hex() + entry_path = self.to_filepath(key_hex) + entry_path = os.path.join(entry_path, file_key) + return self.__add(entry_path, key, content, key_hex, prefix=prefix) + + def __cursor(self, idx): return idx * (self.prefix_length + self.key_length) @@ -84,7 +104,7 @@ class HexDir(LevelDir): lead = '' for i in range(0, self.__levels, 2): lead += hx[i:i+2] + '/' - return lead.upper() + return self.formatter(lead) def to_dirpath(self, hx): @@ -94,14 +114,5 @@ class HexDir(LevelDir): def to_filepath(self, hx): dir_path = self.to_dirpath(hx) - file_path = os.path.join(dir_path, hx.upper()) + file_path = os.path.join(dir_path, self.formatter(hx)) return file_path - - - @staticmethod - def __prepare_directory(path): - os.makedirs(path, exist_ok=True) - state_file = os.path.join(path, 'master') - f = open(state_file, 'w') - f.close() - diff --git a/leveldir/numeric.py b/leveldir/numeric.py @@ -34,6 +34,7 @@ class NumDir(LevelDir): x = 0 d = [] v = 0 + logg.debug('dirpath {}'.format(n)) for t in self.thresholds: x = math.floor(c / t) y = x * t diff --git a/setup.cfg b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = leveldir -version = 0.1.0 +version = 0.2.0a1 description = Create multi-level directory structures author = Louis Holbrook author_email = dev@holbrook.no diff --git a/tests/test_hexdir.py b/tests/test_hexdir.py @@ -82,5 +82,22 @@ class HexDirTest(unittest.TestCase): self.assertEqual(b'cd', prefix) +class HexDirTestFormatter(unittest.TestCase): + + def setUp(self): + def lower_formatter(hx): + return hx.lower() + self.dir = tempfile.mkdtemp() + self.hexdir = HexDir(os.path.join(self.dir, 'q'), 4, 3, 2, formatter=lower_formatter) + logg.debug('setup hexdir root {}'.format(self.dir)) + + + def test_format(self): + self.hexdir.add(b'\xaa\xbb\xcc\xdd', b'foo', b'ab') + (prefix, key) = self.hexdir.get(1) + checkdir_path = os.path.join(self.hexdir.path, 'aa', 'bb', 'cc', 'aabbccdd') + os.stat(checkdir_path) + + if __name__ == '__main__': unittest.main()