feedwarrior

Slim, portable tooling for creating and distributing decentralized append logs
git clone git://git.defalsify.org/logwarrior.git
Log | Files | Refs | README | LICENSE

commit ca8b9fa760adcef2a4343693f38f1ec94d5b11f7
parent 2409e608f3d7d53196dcf12d1b9fd1b520b9fe28
Author: nolash <dev@holbrook.no>
Date:   Fri,  3 Jul 2020 23:46:11 +0200

Add reasonable couchdb adapter and first unit test for file and couchdb

Diffstat:
Afeedwarrior/adapters/__init__.py | 1+
Afeedwarrior/adapters/couchdbadapter.py | 33+++++++++++++++++++++++++++++++++
Mfeedwarrior/adapters/fileadapter.py | 15+++++++++++----
Mfeedwarrior/cmd/entry.py | 5+----
Mfeedwarrior/entry.py | 1-
Atest/test_couchdb.py | 59+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Atest/test_file.py | 46++++++++++++++++++++++++++++++++++++++++++++++
7 files changed, 151 insertions(+), 9 deletions(-)

diff --git a/feedwarrior/adapters/__init__.py b/feedwarrior/adapters/__init__.py @@ -0,0 +1 @@ +from .fileadapter import fileadapter diff --git a/feedwarrior/adapters/couchdbadapter.py b/feedwarrior/adapters/couchdbadapter.py @@ -0,0 +1,33 @@ +# standard imports +import copy +import json +import uuid + +# third party imports +import pycouchdb + +class couchdbadapter: + + def __init__(self, username, password, database='feedwarrior', host='localhost', port=5984, ssl=False): + scheme = 'http' + if ssl: + scheme += 's' + dsn = '{}://{}:{}@{}:{}'.format(scheme, username, password, host, port) + self.server = pycouchdb.Server(dsn) + self.database = self.server.database(database) + self.dsn = dsn + self.username = username + self.password = password + + + def put(self, uu, entry, **kwargs): + e = entry.serialize() + e.pop('uuid') + e['_id'] =uu.hex + self.database.save(e) + + + def get(self, uu, **kwargs): + e = self.database.get(uu.hex) + # reverse parsing this is going to take a bit of effort + print('entry ZZZZZZZZZZZZZZZZZZZZZZZZz', e) diff --git a/feedwarrior/adapters/fileadapter.py b/feedwarrior/adapters/fileadapter.py @@ -2,9 +2,11 @@ import os import gzip import logging +import json logg = logging.getLogger() + class fileadapter: def __init__(self, db_directory, uu): @@ -45,12 +47,17 @@ class fileadapter: return c - def put(self, uu, contents_bytes, **kwargs): + def put(self, uu, entry, **kwargs): + + entry_serialized = entry.serialize() + entry_json = json.dumps(entry_serialized) + contents_bytes = entry_json.encode('utf-8') + entry_path = os.path.join(self.src, 'entries', str(uu)) if os.path.exists(entry_path) or os.path.exists(entry_path + '.gz'): raise FileExistsError('record {} already exists'.format(str(uu))) f = None - if kwargs['compress']: + if kwargs.get('compress') != None: entry_path += '.gz' f = gzip.open(entry_path, 'xb') else: @@ -59,7 +66,7 @@ class fileadapter: f.write(contents_bytes) f.close() - feeds_entry_path = os.path.join(self.src, str(self.feeds_uuid), 'entries', str(uu)) - if kwargs['compress']: + feeds_entry_path = os.path.join(self.src, 'feeds', str(self.feeds_uuid), 'entries', str(uu)) + if kwargs.get('compress') != None: feeds_entry_path += '.gz' os.symlink(entry_path, feeds_entry_path) diff --git a/feedwarrior/cmd/entry.py b/feedwarrior/cmd/entry.py @@ -33,13 +33,10 @@ def execute(config, feed, args): uu = feedwarrior.common.parse_uuid(t) entry.add_extension(feedwarrior.extension.TASKWARRIOR, uu) - entry_serialized = entry.serialize() uu = str(entry.uuid) logg.debug('adding entry {}'.format(uu)) fa = fileadapter(config.data_dir, feed.uuid) - - entry_json = json.dumps(entry_serialized) - fa.put(entry.uuid, entry_json.encode('utf-8'), args.z) + fa.put(entry.uuid, entry, args.z) feed.add(entry) diff --git a/feedwarrior/entry.py b/feedwarrior/entry.py @@ -63,7 +63,6 @@ class entry: def from_multipart_file(filename, hashers=defaulthashers): -#def process_as_multipart_file(config, feed, filename): f = None try: f = open(filename, 'r') diff --git a/test/test_couchdb.py b/test/test_couchdb.py @@ -0,0 +1,59 @@ +# standard imports +import os +import unittest +import uuid +from email.message import EmailMessage + +# third party imports +import pycouchdb + +# local imports +from feedwarrior.adapters.couchdbadapter import couchdbadapter +from feedwarrior.entry import entry + +DBHOST = os.environ.get('FEEDWARRIOR_HOST', 'localhost') +DBPORT = os.environ.get('FEEDWARRIOR_PORT', 5984) +DBPASS = os.environ.get('FEEDWARRIOR_PASS') +DBUSER = os.environ.get('FEEDWARRIOR_USER') +DBSSL = os.environ.get('FEEDWARRIOR_SSL') + +uu = 'x' + uuid.uuid4().hex + +class TestCouchdbadapter(unittest.TestCase): + + def setUp(self): + ssl = DBSSL != None and DBSSL != '' + scheme = 'http' + if ssl: + scheme += 's' + self.srv = pycouchdb.Server('{}://{}:{}@{}:{}'.format(scheme, DBUSER, DBPASS, DBHOST, DBPORT)) + self.db_name = str(uu) + self.db = self.srv.create(self.db_name) + + + def tearDown(self): + self.db.cleanup() + self.srv.delete(self.db_name) + + + def test_init(self): + a = couchdbadapter(DBUSER, DBPASS, self.db_name) + + + def test_put(self): + a = couchdbadapter(DBUSER, DBPASS, self.db_name) + uu = uuid.uuid4() + + msg = EmailMessage() + msg.add_header('Date', 'Thu, 2 Jul 2020 12:00:58 +0200') + msg.add_header('Content-Type', 'multipart/mixed') + msg.add_attachment('foo') + + e = entry(uu, msg) + a.put(uu, e) + + a.get(uu) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_file.py b/test/test_file.py @@ -0,0 +1,46 @@ +# standard imports +import os +import unittest +import tempfile +import logging +import uuid +import shutil +from email.message import EmailMessage + +# local imports +from feedwarrior.adapters.fileadapter import fileadapter +from feedwarrior.entry import entry + +logging.basicConfig(level=logging.DEBUG) +logg = logging.getLogger() + + +class TestFileadapter(unittest.TestCase): + + def setUp(self): + uu = uuid.uuid4() + self.feed_uuid = uu + self.path = tempfile.mkdtemp() + + + def tearDown(self): + shutil.rmtree(self.path) + + + def test_put(self): + a = fileadapter(self.path, self.feed_uuid) + uu = uuid.uuid4() + + msg = EmailMessage() + msg.add_header('Date', 'Thu, 2 Jul 2020 12:00:58 +0200') + msg.add_header('Content-Type', 'multipart/mixed') + msg.add_attachment('foo') + + e = entry(uu, msg) + a.put(uu, e) + + print('ieeeee') + logg.debug(a.get(uu)) + +if __name__ == '__main__': + unittest.main()