testdata_asn1.py (20875B)
1 import enum 2 import os 3 import sys 4 import io 5 import zlib 6 import base64 7 import logging 8 import hashlib 9 from Crypto.Cipher import ChaCha20_Poly1305 10 from Crypto.PublicKey import ECC 11 import Crypto.IO.PKCS8 12 import Crypto.Util.asn1 13 from Crypto.Signature import eddsa 14 import lmdb 15 import time 16 import datetime 17 import shutil 18 import email.message 19 import random 20 from faker import Faker 21 from faker.providers import lorem 22 import varint 23 from pyasn1.codec.der.encoder import encode as der_encode 24 from pyasn1.codec.der.decoder import decode as der_decode 25 from pyasn1.codec.native.decoder import decode 26 from pyasn1.type.univ import Any 27 from pygcrypt.gctypes.sexpression import SExpression 28 from pygcrypt.gctypes.key import Key as GKey 29 30 from testdata_asn1schema import KeeEntryHead 31 from testdata_asn1schema import KeeEntry 32 from testdata_asn1schema import KeeTransport 33 34 logging.basicConfig(level=logging.DEBUG) 35 logg = logging.getLogger() 36 37 fake = Faker() 38 fake.add_provider(lorem) 39 40 41 FLAGS_SIGNER_IS_BOB = 1 << 0 42 43 def to_key_filename(keyname): 44 filename = keyname.lower() 45 filename = filename.replace(" ", "_") 46 return filename 47 48 49 class LedgerMode(enum.IntEnum): 50 REQUEST = 0 51 RESPONSE = 1 52 FINAL = 2 53 54 55 class LedgerRole(enum.Enum): 56 ALICE = 'alice' 57 BOB = 'bob' 58 59 60 class TransportCmd(enum.Enum): 61 IMPORT = b'\x00' 62 ID = b'\x01' 63 LEDGER = b'\x02' 64 65 66 NOBODY = b'\x00' * 64 67 NOSIG = b'' 68 PFX_LEDGER_HEAD = b'\x01' 69 PFX_LEDGER_ENTRY = b'\x02' 70 PFX_LEDGER_PUBKEY = b'\x03' 71 PFX_LEDGER_CACHE_SUMS = b'\x80' 72 73 random.seed(int(time.time_ns())) 74 75 76 def padbytes(b, padsize=4096): 77 l = padsize - (len(b) % padsize) 78 b += os.urandom(l) 79 return b 80 81 82 def db_init(d): 83 d = os.path.join(d, 'testdata', 'mdb') 84 logg.info('using d for db' + d) 85 86 try: 87 shutil.rmtree(d) 88 except FileNotFoundError: 89 pass 90 os.makedirs(d) 91 return d 92 93 94 def data_add(data_dir, k, v): 95 if data_dir == None: 96 return 97 fp = os.path.join(data_dir, k.hex()) 98 f = open(fp, 'wb') 99 f.write(v) 100 f.close() 101 102 103 class LedgerContent(email.message.EmailMessage): 104 105 def __init__(self, subject=None, body=None): 106 super(LedgerContent, self).__init__() 107 self.set_default_type("text/plain") 108 if subject == None: 109 subject = fake.sentence() 110 self.add_header("Subject", subject) 111 if body == None: 112 body = fake.paragraph() 113 self.set_content(body) 114 115 116 def kv(self): 117 b = self.as_bytes() 118 h = hashlib.new("sha512") 119 h.update(b) 120 z = h.digest() 121 return (z, b,) 122 123 124 class LedgerHeadContent(LedgerContent): 125 pass 126 127 128 class LedgerItemContent(LedgerContent): 129 pass 130 131 132 # TODO: do everything with pygcrypt, or calc keygrip with pycryptodome 8| 133 class LedgerSigner: 134 135 def __init__(self, crypto_dir): 136 self.signer = {} 137 self.keypair = {} 138 self.pubkey_rindex = {} 139 self.names = {} 140 self.crypto_dir = crypto_dir 141 142 143 def get_pubkey(self, k): 144 return self.keypair[k][1] 145 146 147 def get_name(self, k): 148 return self.names[k] 149 150 151 152 def __write_key(self, keyname, outdir, pin, alias=None): 153 (pk, pubk) = self.keypair[keyname] 154 wt = io.BytesIO() 155 wt.write(b"(8:key-data(10:public-key(3:ecc(5:curve7:Ed25519)(5:flags5:eddsa)(1:q32:") 156 wt.write(pubk) 157 wt.write(b")))(11:private-key(3:ecc(5:curve7:Ed25519)(5:flags5:eddsa)(1:q32:") 158 wt.write(pubk) 159 wt.write(b")(1:d32:") 160 wt.write(pk) 161 wt.write(b"))))") 162 b = wt.getvalue() 163 164 filename = to_key_filename(keyname) 165 fp = os.path.join(self.crypto_dir, filename + '.key.sexp') 166 w = open(fp, 'wb') 167 w.write(b) 168 w.close() 169 170 sexp = SExpression(b) 171 gk = GKey(sexp) 172 173 l = len(b) 174 bl = l.to_bytes(4, byteorder='little') 175 h = hashlib.new('sha256') 176 h.update(pin.encode('utf-8')) 177 z_pin = h.digest() 178 nonce = os.urandom(12) 179 cph = ChaCha20_Poly1305.new(key=z_pin, nonce=nonce) 180 r = cph.encrypt(bl + b) 181 r = padbytes(r) 182 183 fp = os.path.join(self.crypto_dir, filename + '.key.bin') 184 w = open(fp, 'wb') 185 w.write(nonce + r) 186 w.close() 187 188 # symlink key to keygrip 189 lp = os.path.join(self.crypto_dir, gk.keygrip) 190 os.symlink(fp, lp) 191 192 # symlink key to alias 193 if alias != None: 194 lp = os.path.join(self.crypto_dir, alias + '.key.bin') 195 os.symlink(fp, lp) 196 197 wt = io.BytesIO() 198 wt.write(b"(8:key-data(10:public-key(3:ecc(5:curve7:Ed25519)(5:flags5:eddsa)(1:q32:") 199 wt.write(pubk) 200 wt.write(b"))))") 201 b = wt.getvalue() 202 fp = os.path.join(self.crypto_dir, filename + '.pubkey.sexp') 203 w = open(fp, "wb") 204 w.write(b) 205 w.close() 206 207 fp = os.path.join(self.crypto_dir, filename + '.pubkey.txt') 208 w = open(fp, "w") 209 w.write(pubk.hex()) 210 w.close() 211 212 return gk.keygrip 213 214 215 def create_key(self, keyname, outdir=None, pin='1234', alias=None): 216 k = ECC.generate(curve='Ed25519') 217 pk_pkcs8 = k.export_key(format='DER') 218 pk_der = Crypto.IO.PKCS8.unwrap(pk_pkcs8) 219 pk = Crypto.Util.asn1.DerOctetString().decode(pk_der[1], strict=True).payload 220 pubk = k.public_key().export_key(format='raw') 221 222 self.signer[keyname] = eddsa.new(k, 'rfc8032') 223 self.keypair[keyname] = (pk, pubk) 224 self.pubkey_rindex[pubk] = keyname 225 226 keygrip = self.__write_key(keyname, outdir, pin, alias=alias) 227 228 self.names[keyname] = fake.name() 229 230 return (pubk, keygrip,) 231 232 233 def sign(self, keyname, msg): 234 h = hashlib.sha512() 235 h.update(msg) 236 z = h.digest() 237 238 fp = os.path.join(self.crypto_dir, z.hex()) 239 w = open(fp, 'wb') 240 w.write(msg) 241 w.close() 242 243 b = self.signer[keyname].sign(z) 244 245 logg.debug('signature material\t{}\n\t{}'.format(msg[:64].hex(), msg[64:].hex())) 246 logg.debug('signature for {}: {}'.format(z.hex(), b.hex())) 247 248 return b 249 250 251 252 class LedgerBundle: 253 254 def __init__(self, data_dir, ledger): 255 self.data_dir = data_dir 256 self.o = KeeTransport() 257 self.o.setComponentByPosition(0, ledger.to_asn1(data_dir)) 258 259 260 def to_asn1(self, ledger_item, mode): 261 self.o.setComponentByPosition(1, ledger_item.to_asn1(data_dir, mode)) 262 return self.o 263 264 265 def serialize(self, ledger_item, mode, w=sys.stdout.buffer): 266 o = self.to_asn1(ledger_item, mode) 267 b = der_encode(o) 268 b = TransportCmd.LEDGER.value + b 269 if w == None: 270 return b 271 w.write(b) 272 273 274 def encode(self, ledger_item, mode, w=sys.stdout.buffer): 275 b = self.serialize(ledger_item, mode, w=None) 276 b = zlib.compress(b, level=9) 277 b = base64.b64encode(b) 278 w.write(b) 279 280 281 class Ledger: 282 pass 283 284 # @classmethod 285 # def data_add(self, data_dir, k, v): 286 # fp = os.path.join(data_dir, k.hex()) 287 # f = open(fp, 'wb') 288 # f.write(v) 289 # f.close() 290 291 292 class LedgerGenerator: 293 294 credit_alice_min = 1000 295 credit_alice_max = 10000 296 credit_bob_min = 1000 297 credit_bob_max = 10000 298 299 def __init__(self): 300 self.credit_alice = random.randint(self.credit_alice_min, self.credit_alice_max) 301 self.credit_bob = random.randint(self.credit_bob_min, self.credit_bob_max) 302 self.collateral_alice = random.randint(self.credit_alice_min, self.credit_alice_max) 303 self.collateral_bob = random.randint(self.credit_bob_min, self.credit_bob_max) 304 self.count = 0 305 logg.debug('new generator with credit alice {} bob {}'.format(self.credit_alice, self.credit_bob)) 306 307 308 def delta(self, collateral=False, credit=True, signer_role=None): 309 delta_credit = 0 310 delta_collateral = 0 311 312 single_is_bob = False 313 314 initial_credit_role = None 315 if self.count < 2: 316 if signer_role == None: 317 if self.count == 0: 318 initial_credit_role = LedgerRole.ALICE 319 else: 320 initial_credit_role = LedgerRole.BOB 321 else: 322 initial_credit_role = signer_role 323 324 if initial_credit_role != None: 325 if initial_credit_role == LedgerRole.ALICE: 326 delta_credit = self.credit_alice 327 delta_collateral = self.collateral_alice 328 else: 329 delta_credit = self.credit_bob 330 delta_collateral = self.collateral_bob 331 single_is_bob = True 332 else: 333 if self.credit_bob == 0 and self.credit_alice == 0: 334 raise OverflowError("Both alice and bob are broke :'(") 335 336 if delta_credit == 0: 337 if signer_role == None: 338 single_is_bob = bool(random.randint(0, 1)) 339 elif signer_role == LedgerRole.ALICE: 340 single_is_bob = True 341 342 if self.credit_bob == 0: 343 if signer_role == LedgerRole.BOB: 344 raise OverflowError("bob is broke") 345 single_is_bob = False 346 elif self.credit_alice == 0: 347 single_is_bob = True 348 349 if credit: 350 if single_is_bob: 351 delta_credit = random.randint(1, self.credit_bob) * -1 352 self.credit_bob += delta_credit 353 else: 354 delta_credit = random.randint(1, self.credit_alice) * -1 355 self.credit_alice += delta_credit 356 357 if collateral: 358 if single_is_bob: 359 delta_collateral = random.randint(1, self.collateral_bob) * -1 360 self.collateral_bob += delta_collateral 361 else: 362 delta_collateral = random.randint(1, self.collateral_alice) * -1 363 self.collateral_alice += delta_collateral 364 365 delta_credit_bob = 0 366 delta_credit_alice = 0 367 if single_is_bob: 368 delta_credit_bob = delta_credit 369 else: 370 delta_credit_alice = delta_credit 371 372 delta_collateral_bob = 0 373 delta_collateral_alice = 0 374 if single_is_bob: 375 delta_collateral_bob = delta_collateral 376 else: 377 delta_collateral_alice = delta_collateral 378 379 self.count += 1 380 381 logg.debug('credit delta alice {} ({}) bob {} ({}) isbob {}'.format(delta_credit_alice, self.credit_alice, delta_credit_bob, self.credit_bob, single_is_bob)) 382 #logg.debug('collateral delta alice {} ({}) bob {} ({})'.format(delta_collateral_alice, self.collateral_alice, delta_collateral_bob, self.collateral_bob)) 383 return (delta_credit, delta_collateral, single_is_bob,) 384 385 386 class LedgerHead(Ledger): 387 388 def __init__(self, alice_key=None, bob_key=None, body=NOBODY): 389 self.uoa = "USD" 390 self.uoa_decimals = 2 391 if alice_key == None: 392 alice_key = os.urandom(65) 393 self.alice_pubkey_ref = alice_key 394 if bob_key == None: 395 bob_key = os.urandom(65) 396 self.bob_pubkey_ref = bob_key 397 self.body = LedgerHeadContent() 398 (k, v) = self.body.kv() 399 400 logg.info('new ledger header with alice {} bob {} body {}'.format(self.alice_pubkey_ref.hex(), self.bob_pubkey_ref.hex(), k.hex())) 401 402 403 def to_asn1(self, data_dir): 404 o = KeeEntryHead() 405 o['uoa'] = self.uoa 406 o['uoaDecimals'] = self.uoa_decimals 407 o['alicePubKey'] = self.alice_pubkey_ref 408 o['bobPubKey'] = self.bob_pubkey_ref 409 (k, v) = self.body.kv() 410 #self.data_add(data_dir, k, v) 411 data_add(data_dir, k, v) 412 o['body'] = k 413 return o 414 415 416 def serialize(self, data_dir, w=sys.stdout.buffer): 417 o = self.to_asn1(data_dir) 418 b = der_encode(o) 419 logg.debug('ledger header serialize ({}): {}'.format(len(b), b.hex())) 420 w.write(b) 421 422 423 @staticmethod 424 def to_key(b): 425 r = b'' 426 r += PFX_LEDGER_HEAD 427 t = time.time_ns() 428 v = int(t / 1000000000) 429 b = v.to_bytes(4, byteorder='big') 430 v = t - (v * 1000000000) 431 b += v.to_bytes(4, byteorder='big') 432 r += b 433 434 return r 435 436 437 class LedgerItem(Ledger): 438 439 credit_alice = 0 440 credit_bob = 0 441 collateral_alice = 0 442 collateral_bob = 0 443 ms = 0 444 445 def __init__(self, head, signer, generator, signer_name=None, parent=None, body=NOBODY, bob_name='bob'): 446 self.head = head 447 self.parent = parent 448 if self.parent == None: 449 self.parent = b'\x00' * 64 450 #self.timestamp = time.time_ns() 451 self.timestamp = b'' 452 t = time.time_ns() 453 v = int(t / 1000000000) 454 self.timestamp += v.to_bytes(4, byteorder='big') 455 v = t - (v * 1000000000) 456 self.timestamp += v.to_bytes(4, byteorder='big') 457 458 self.body = LedgerItemContent() 459 460 signer_role = None 461 if signer_name != None: 462 signer_role=LedgerRole(signer_name) 463 delta = generator.delta(signer_role=signer_role) 464 self.signer_sequence = [] 465 if delta[2]: 466 self.signer_sequence = [bob_name, 'alice'] 467 else: 468 self.signer_sequence = ['alice', bob_name] 469 self.credit_delta = delta[0] 470 self.collateral_delta = delta[1] 471 472 self.request_signature = NOSIG 473 self.response_signature = NOSIG 474 self.signer = signer 475 476 477 def to_asn1(self, data_dir, mode=LedgerMode.FINAL): 478 o = KeeEntry() 479 o['parent'] = self.parent 480 o['timestamp'] = self.timestamp 481 o['creditDelta'] = self.credit_delta 482 o['collateralDelta'] = self.collateral_delta 483 484 (k, v) = self.body.kv() 485 #self.data_add(data_dir, k, v) 486 data_add(data_dir, k, v) 487 o['body'] = k 488 489 o['response'] = False 490 o['signatureRequest'] = NOSIG 491 o['signatureResponse'] = NOSIG 492 493 if mode == LedgerMode.REQUEST: 494 return o 495 496 logg.debug('encoding new ledger_item for request signature {}: {}'.format(self.head.hex(), o)) 497 b = der_encode(o) 498 self.request_signature = self.signer.sign(self.signer_sequence[0], self.head + b) 499 o['signatureRequest'] = self.request_signature 500 501 if mode == LedgerMode.RESPONSE: 502 return o 503 504 if mode != LedgerMode.FINAL: 505 raise ValueError("invalid ledger mode: {}".format(mode)) 506 507 o['response'] = True 508 b = der_encode(o) 509 self.response_signature = self.signer.sign(self.signer_sequence[1], self.head + b) 510 o['signatureResponse'] = self.response_signature 511 return o 512 513 514 def serialize(self, data_dir, mode=LedgerMode.FINAL, w=sys.stdout.buffer): 515 o = self.to_asn1(data_dir, mode=mode) 516 b = der_encode(o) 517 flag = b'\x00' 518 if self.signer_sequence[0] != 'alice': 519 flag = b'\x01' 520 w.write(b + flag) 521 522 LedgerItem.ms += 1 523 if LedgerItem.ms > 999: 524 LedgerItem.ms = 0 525 526 527 @staticmethod 528 def to_key(v, k): 529 r = b'' 530 r += PFX_LEDGER_ENTRY 531 r += k 532 533 o = der_decode(v, asn1Spec=KeeEntry()) 534 #ts = o[0]['timestamp'] 535 #tsb = int(ts).to_bytes(8, byteorder='big') 536 tsb = o[0]['timestamp'].asOctets() 537 #logg.debug('ts {} ({}): of {}'.format(ts, tsb, v.hex())) 538 r += tsb 539 return r 540 541 542 def generate_ledger_item(data_dir, signer, generator, head, bob_name, parent): 543 o = LedgerItem(head, signer, generator, parent=parent, bob_name=bob_name) 544 w = io.BytesIO() 545 r = o.serialize(data_dir, w=w) 546 h = hashlib.new('sha512') 547 b = w.getvalue() 548 h.update(b) 549 z = h.digest() 550 return (z, b, o,) 551 552 553 def generate_ledger(dbi, data_dir, signer, bob_name, ledger_item_count=3, alice=None, bob=None): 554 r = [] 555 o = LedgerHead(alice_key=alice, bob_key=bob) 556 w = io.BytesIO() 557 o.serialize(data_dir, w=w) 558 h = hashlib.new('sha512') 559 b = w.getvalue() 560 h.update(b) 561 z = h.digest() 562 r.append((z, b, o,)) 563 564 k = z 565 parent = None 566 LedgerItem.credit_alice = random.randint(100, 1000) 567 LedgerItem.credit_bob = random.randint(100, 1000) 568 569 generator = LedgerGenerator() 570 for i in range(ledger_item_count): 571 try: 572 v = generate_ledger_item(data_dir, signer, generator, k, bob_name, parent) 573 except OverflowError: 574 break 575 # \todo generate key value already here 576 parent = v[0] 577 r.append(v) 578 579 return r 580 581 582 if __name__ == '__main__': 583 d = os.path.dirname(__file__) 584 data_dir = os.path.join(d, 'testdata', 'resource') 585 try: 586 shutil.rmtree(data_dir) 587 except FileNotFoundError: 588 pass 589 os.makedirs(data_dir) 590 591 v = b'foo' 592 h = hashlib.sha512() 593 h.update(v) 594 k = h.digest() 595 data_add(data_dir, k, v) 596 597 o = LedgerContent(subject='foo', body='bar') 598 (k, v) = o.kv() 599 data_add(data_dir, k, v) 600 601 d = os.path.dirname(__file__) 602 crypto_dir = os.path.join(d, 'testdata', 'crypt') 603 try: 604 shutil.rmtree(crypto_dir) 605 except FileNotFoundError: 606 pass 607 os.makedirs(crypto_dir) 608 609 d = os.path.dirname(__file__) 610 crypto_dir_r = os.path.join(d, 'testdata', 'crypt_reverse') 611 try: 612 shutil.rmtree(crypto_dir_r) 613 except FileNotFoundError: 614 pass 615 os.makedirs(crypto_dir_r) 616 617 d = db_init(d) 618 619 env = lmdb.open(d) 620 dbi = env.open_db() 621 622 signer = LedgerSigner(crypto_dir) 623 alice = signer.create_key('alice', outdir=data_dir) 624 #bob = bob(d) 625 626 keys = ['alice'] 627 628 alice_key = os.path.join(crypto_dir, 'alice.key.bin') 629 try: 630 os.unlink('kee.key') 631 except FileNotFoundError: 632 pass 633 alice_key_sym = 'kee.key' 634 os.symlink(alice_key, alice_key_sym) 635 alice_key_sym = os.path.join(crypto_dir, alice_key_sym) 636 os.symlink(alice_key, alice_key_sym) 637 638 count_ledgers = os.environ.get('COUNT', '1') 639 items_min_in = os.environ.get('ITEM_MIN', '1') 640 items_max_in = os.environ.get('ITEM_MAX', '20') 641 642 mainbob_keygrip = None 643 with env.begin(write=True) as tx: 644 for i in range(int(count_ledgers)): 645 bob_name = 'Bob ' + fake.last_name() 646 keys.append(bob_name) 647 alias = None 648 if i == 0: 649 alias = 'bob' 650 bob = signer.create_key(bob_name, outdir=data_dir, pin='4321', alias=alias) 651 if i == 0: 652 mainbob_keygrip = bob[1] 653 # bob_key = os.path.join(crypto_dir, 'bob.key.bin') 654 # bob_key_sym = os.path.join(crypto_dir_r, 'kee.key') 655 # try: 656 # os.unlink('kee.key') 657 # except FileNotFoundError: 658 # pass 659 # os.symlink(bob_key, bob_key_sym) 660 661 c = 2 + random.randint(int(items_min_in), int(items_max_in)) 662 663 r = generate_ledger(dbi, data_dir, signer, bob_name, ledger_item_count=c, alice=alice[0], bob=bob[0]) 664 665 v = r.pop(0) 666 667 ledger_object = v[2] 668 669 z = v[0] 670 k = LedgerHead.to_key(v[0]) 671 tx.put(k, v[1]) 672 # reverse lookup 673 kr = b'\xff' + v[0] 674 tx.put(kr, k[1:]) 675 676 for v in r: 677 k = LedgerItem.to_key(v[1], z) 678 tx.put(k, v[1]) 679 680 for k in keys: 681 pubk = signer.get_pubkey(k) 682 name = signer.get_name(k).encode('utf-8') 683 tx.put(PFX_LEDGER_PUBKEY + pubk, b'CN=' + name) 684 685 # generate ledger import 686 bob_name = 'Bob ' + fake.last_name() 687 keys.append(bob_name) 688 bob = signer.create_key(bob_name, outdir=data_dir) 689 bob_key = os.path.join(crypto_dir, 'bob.key.bin') 690 bob_key_sym = os.path.join(crypto_dir_r, 'kee.key') 691 try: 692 os.unlink('kee.key') 693 except FileNotFoundError: 694 pass 695 os.symlink(bob_key, bob_key_sym) 696 bob_keygrip_sym = os.path.join(crypto_dir_r, mainbob_keygrip) 697 os.symlink(bob_key, bob_keygrip_sym) 698 699 700 r = generate_ledger(dbi, data_dir, signer, bob_name, ledger_item_count=1, alice=alice[0], bob=bob[0]) 701 d = os.path.dirname(__file__) 702 import_dir = os.path.join(d, 'testdata', 'import') 703 try: 704 shutil.rmtree(import_dir) 705 except FileNotFoundError: 706 pass 707 os.makedirs(import_dir) 708 709 ledger_object = r[0][2] 710 ledger_item_object = r[1][2] 711 importer = LedgerBundle(data_dir, ledger_object) 712 w = io.BytesIO() 713 fp = os.path.join(import_dir, 'request') 714 f = open(fp, 'wb') 715 importer.encode(ledger_item_object, LedgerMode.REQUEST, w=f) 716 f.close() 717 718 w = io.BytesIO() 719 fp = os.path.join(import_dir, 'response') 720 f = open(fp, 'wb') 721 importer.encode(ledger_item_object, LedgerMode.RESPONSE, w=f) 722 f.close() 723 724 w = io.BytesIO() 725 fp = os.path.join(import_dir, 'final') 726 f = open(fp, 'wb') 727 importer.encode(ledger_item_object, LedgerMode.FINAL, w=f) 728 f.close()