testdata_asn1.py (20484B)
1 import enum 2 import os 3 import sys 4 import io 5 import zlib 6 import base64 7 import logging 8 import hashlib 9 from Crypto.Cipher import ChaCha20_Poly1305 10 from Crypto.PublicKey import ECC 11 import Crypto.IO.PKCS8 12 import Crypto.Util.asn1 13 from Crypto.Signature import eddsa 14 import lmdb 15 import time 16 import datetime 17 import shutil 18 import email.message 19 import random 20 from faker import Faker 21 from faker.providers import lorem 22 import varint 23 from pyasn1.codec.der.encoder import encode as der_encode 24 from pyasn1.codec.der.decoder import decode as der_decode 25 from pyasn1.codec.native.decoder import decode 26 from pyasn1.type.univ import Any 27 from pygcrypt.gctypes.sexpression import SExpression 28 from pygcrypt.gctypes.key import Key as GKey 29 30 from testdata_asn1schema import KeeEntryHead 31 from testdata_asn1schema import KeeEntry 32 from testdata_asn1schema import KeeTransport 33 34 logging.basicConfig(level=logging.DEBUG) 35 logg = logging.getLogger() 36 37 fake = Faker() 38 fake.add_provider(lorem) 39 40 41 FLAGS_SIGNER_IS_BOB = 1 << 0 42 43 def to_key_filename(keyname): 44 filename = keyname.lower() 45 filename = filename.replace(" ", "_") 46 return filename 47 48 49 class LedgerMode(enum.IntEnum): 50 REQUEST = 0 51 RESPONSE = 1 52 FINAL = 2 53 54 55 class LedgerRole(enum.Enum): 56 ALICE = 'alice' 57 BOB = 'bob' 58 59 60 class TransportCmd(enum.Enum): 61 IMPORT = b'\x00' 62 ID = b'\x01' 63 LEDGER = b'\x02' 64 65 66 NOBODY = b'\x00' * 64 67 NOSIG = b'' 68 PFX_LEDGER_HEAD = b'\x01' 69 PFX_LEDGER_ENTRY = b'\x02' 70 PFX_LEDGER_PUBKEY = b'\x03' 71 PFX_LEDGER_CACHE_SUMS = b'\x80' 72 73 random.seed(int(time.time_ns())) 74 75 76 def padbytes(b, padsize=4096): 77 l = padsize - (len(b) % padsize) 78 b += os.urandom(l) 79 return b 80 81 82 def db_init(d): 83 d = os.path.join(d, 'testdata', 'mdb') 84 logg.info('using d for db' + d) 85 86 try: 87 shutil.rmtree(d) 88 except FileNotFoundError: 89 pass 90 os.makedirs(d) 91 return d 92 93 94 def data_add(data_dir, k, v): 95 if data_dir == None: 96 return 97 fp = os.path.join(data_dir, k.hex()) 98 f = open(fp, 'wb') 99 f.write(v) 100 f.close() 101 102 103 class LedgerContent(email.message.EmailMessage): 104 105 def __init__(self, subject=None, body=None): 106 super(LedgerContent, self).__init__() 107 self.set_default_type("text/plain") 108 if subject == None: 109 subject = fake.sentence() 110 self.add_header("Subject", subject) 111 if body == None: 112 body = fake.paragraph() 113 self.set_content(body) 114 115 116 def kv(self): 117 b = self.as_bytes() 118 h = hashlib.new("sha512") 119 h.update(b) 120 z = h.digest() 121 return (z, b,) 122 123 124 class LedgerHeadContent(LedgerContent): 125 pass 126 127 128 class LedgerItemContent(LedgerContent): 129 pass 130 131 132 # TODO: do everything with pygcrypt, or calc keygrip with pycryptodome 8| 133 class LedgerSigner: 134 135 def __init__(self, crypto_dir): 136 self.signer = {} 137 self.keypair = {} 138 self.pubkey_rindex = {} 139 self.names = {} 140 self.crypto_dir = crypto_dir 141 142 143 def get_pubkey(self, k): 144 return self.keypair[k][1] 145 146 147 def get_name(self, k): 148 return self.names[k] 149 150 151 152 def __write_key(self, keyname, outdir, pin, alias=None): 153 (pk, pubk) = self.keypair[keyname] 154 wt = io.BytesIO() 155 wt.write(b"(8:key-data(10:public-key(3:ecc(5:curve7:Ed25519)(5:flags5:eddsa)(1:q32:") 156 wt.write(pubk) 157 wt.write(b")))(11:private-key(3:ecc(5:curve7:Ed25519)(5:flags5:eddsa)(1:q32:") 158 wt.write(pubk) 159 wt.write(b")(1:d32:") 160 wt.write(pk) 161 wt.write(b"))))") 162 b = wt.getvalue() 163 164 filename = to_key_filename(keyname) 165 fp = os.path.join(self.crypto_dir, filename + '.key.sexp') 166 w = open(fp, 'wb') 167 w.write(b) 168 w.close() 169 170 sexp = SExpression(b) 171 gk = GKey(sexp) 172 173 l = len(b) 174 bl = l.to_bytes(4, byteorder='little') 175 h = hashlib.new('sha256') 176 h.update(pin.encode('utf-8')) 177 z_pin = h.digest() 178 nonce = os.urandom(12) 179 cph = ChaCha20_Poly1305.new(key=z_pin, nonce=nonce) 180 r = cph.encrypt(bl + b) 181 r = padbytes(r) 182 183 fp = os.path.join(self.crypto_dir, filename + '.key.bin') 184 w = open(fp, 'wb') 185 w.write(nonce + r) 186 w.close() 187 188 # symlink key to keygrip 189 #lp = os.path.join(self.crypto_dir, gk.keygrip) 190 od = os.getcwd() 191 os.chdir(self.crypto_dir) 192 os.symlink(os.path.basename(fp), gk.keygrip) 193 194 # symlink key to alias 195 if alias != None: 196 lp = alias + '.key.bin' 197 os.symlink(os.path.basename(fp), lp) 198 os.chdir(od) 199 200 wt = io.BytesIO() 201 wt.write(b"(8:key-data(10:public-key(3:ecc(5:curve7:Ed25519)(5:flags5:eddsa)(1:q32:") 202 wt.write(pubk) 203 wt.write(b"))))") 204 b = wt.getvalue() 205 fp = os.path.join(self.crypto_dir, filename + '.pubkey.sexp') 206 w = open(fp, "wb") 207 w.write(b) 208 w.close() 209 210 fp = os.path.join(self.crypto_dir, filename + '.pubkey.txt') 211 w = open(fp, "w") 212 w.write(pubk.hex()) 213 w.close() 214 215 return gk.keygrip 216 217 218 def create_key(self, keyname, outdir=None, pin='1234', alias=None): 219 k = ECC.generate(curve='Ed25519') 220 pk_pkcs8 = k.export_key(format='DER') 221 pk_der = Crypto.IO.PKCS8.unwrap(pk_pkcs8) 222 pk = Crypto.Util.asn1.DerOctetString().decode(pk_der[1], strict=True).payload 223 pubk = k.public_key().export_key(format='raw') 224 225 self.signer[keyname] = eddsa.new(k, 'rfc8032') 226 self.keypair[keyname] = (pk, pubk) 227 self.pubkey_rindex[pubk] = keyname 228 229 keygrip = self.__write_key(keyname, outdir, pin, alias=alias) 230 231 self.names[keyname] = fake.name() 232 233 return (pubk, keygrip,) 234 235 236 def sign(self, keyname, msg): 237 h = hashlib.sha512() 238 h.update(msg) 239 z = h.digest() 240 241 fp = os.path.join(self.crypto_dir, z.hex()) 242 w = open(fp, 'wb') 243 w.write(msg) 244 w.close() 245 246 b = self.signer[keyname].sign(z) 247 248 logg.debug('signature material\t{}\n\t{}'.format(msg[:64].hex(), msg[64:].hex())) 249 logg.debug('signature for {}: {}'.format(z.hex(), b.hex())) 250 251 return b 252 253 254 255 class LedgerBundle: 256 257 def __init__(self, data_dir, ledger): 258 self.data_dir = data_dir 259 self.o = KeeTransport() 260 self.o.setComponentByPosition(0, ledger.to_asn1(data_dir)) 261 262 263 def to_asn1(self, ledger_item, mode): 264 self.o.setComponentByPosition(1, ledger_item.to_asn1(data_dir, mode)) 265 return self.o 266 267 268 def serialize(self, ledger_item, mode, w=sys.stdout.buffer): 269 o = self.to_asn1(ledger_item, mode) 270 b = der_encode(o) 271 b = TransportCmd.LEDGER.value + b 272 if w == None: 273 return b 274 w.write(b) 275 276 277 def encode(self, ledger_item, mode, w=sys.stdout.buffer): 278 b = self.serialize(ledger_item, mode, w=None) 279 b = zlib.compress(b, level=9) 280 b = base64.b64encode(b) 281 w.write(b) 282 283 284 class Ledger: 285 pass 286 287 # @classmethod 288 # def data_add(self, data_dir, k, v): 289 # fp = os.path.join(data_dir, k.hex()) 290 # f = open(fp, 'wb') 291 # f.write(v) 292 # f.close() 293 294 295 class LedgerGenerator: 296 297 credit_alice_min = 1000 298 credit_alice_max = 10000 299 credit_bob_min = 1000 300 credit_bob_max = 10000 301 302 def __init__(self): 303 self.credit_alice = random.randint(self.credit_alice_min, self.credit_alice_max) 304 self.credit_bob = random.randint(self.credit_bob_min, self.credit_bob_max) 305 self.collateral_alice = random.randint(self.credit_alice_min, self.credit_alice_max) 306 self.collateral_bob = random.randint(self.credit_bob_min, self.credit_bob_max) 307 self.count = 0 308 logg.debug('new generator with credit alice {} bob {}'.format(self.credit_alice, self.credit_bob)) 309 310 311 def delta(self, collateral=False, credit=True, signer_role=None): 312 delta_credit = 0 313 delta_collateral = 0 314 315 single_is_bob = False 316 317 initial_credit_role = None 318 if self.count < 2: 319 if signer_role == None: 320 if self.count == 0: 321 initial_credit_role = LedgerRole.ALICE 322 else: 323 initial_credit_role = LedgerRole.BOB 324 else: 325 initial_credit_role = signer_role 326 327 if initial_credit_role != None: 328 if initial_credit_role == LedgerRole.ALICE: 329 delta_credit = self.credit_alice 330 delta_collateral = self.collateral_alice 331 else: 332 delta_credit = self.credit_bob 333 delta_collateral = self.collateral_bob 334 single_is_bob = True 335 else: 336 if self.credit_bob == 0 and self.credit_alice == 0: 337 raise OverflowError("Both alice and bob are broke :'(") 338 339 if delta_credit == 0: 340 if signer_role == None: 341 single_is_bob = bool(random.randint(0, 1)) 342 elif signer_role == LedgerRole.ALICE: 343 single_is_bob = True 344 345 if self.credit_bob == 0: 346 if signer_role == LedgerRole.BOB: 347 raise OverflowError("bob is broke") 348 single_is_bob = False 349 elif self.credit_alice == 0: 350 single_is_bob = True 351 352 if credit: 353 if single_is_bob: 354 delta_credit = random.randint(1, self.credit_bob) * -1 355 self.credit_bob += delta_credit 356 else: 357 delta_credit = random.randint(1, self.credit_alice) * -1 358 self.credit_alice += delta_credit 359 360 if collateral: 361 if single_is_bob: 362 delta_collateral = random.randint(1, self.collateral_bob) * -1 363 self.collateral_bob += delta_collateral 364 else: 365 delta_collateral = random.randint(1, self.collateral_alice) * -1 366 self.collateral_alice += delta_collateral 367 368 delta_credit_bob = 0 369 delta_credit_alice = 0 370 if single_is_bob: 371 delta_credit_bob = delta_credit 372 else: 373 delta_credit_alice = delta_credit 374 375 delta_collateral_bob = 0 376 delta_collateral_alice = 0 377 if single_is_bob: 378 delta_collateral_bob = delta_collateral 379 else: 380 delta_collateral_alice = delta_collateral 381 382 self.count += 1 383 384 logg.debug('credit delta alice {} ({}) bob {} ({}) isbob {}'.format(delta_credit_alice, self.credit_alice, delta_credit_bob, self.credit_bob, single_is_bob)) 385 #logg.debug('collateral delta alice {} ({}) bob {} ({})'.format(delta_collateral_alice, self.collateral_alice, delta_collateral_bob, self.collateral_bob)) 386 return (delta_credit, delta_collateral, single_is_bob,) 387 388 389 class LedgerHead(Ledger): 390 391 def __init__(self, alice_key=None, bob_key=None, body=NOBODY): 392 self.uoa = "USD" 393 self.uoa_decimals = 2 394 if alice_key == None: 395 alice_key = os.urandom(65) 396 self.alice_pubkey_ref = alice_key 397 if bob_key == None: 398 bob_key = os.urandom(65) 399 self.bob_pubkey_ref = bob_key 400 self.body = LedgerHeadContent() 401 (k, v) = self.body.kv() 402 403 logg.info('new ledger header with alice {} bob {} body {}'.format(self.alice_pubkey_ref.hex(), self.bob_pubkey_ref.hex(), k.hex())) 404 405 406 def to_asn1(self, data_dir): 407 o = KeeEntryHead() 408 o['uoa'] = self.uoa 409 o['uoaDecimals'] = self.uoa_decimals 410 o['alicePubKey'] = self.alice_pubkey_ref 411 o['bobPubKey'] = self.bob_pubkey_ref 412 (k, v) = self.body.kv() 413 #self.data_add(data_dir, k, v) 414 data_add(data_dir, k, v) 415 o['body'] = k 416 return o 417 418 419 def serialize(self, data_dir, w=sys.stdout.buffer): 420 o = self.to_asn1(data_dir) 421 b = der_encode(o) 422 logg.debug('ledger header serialize ({}): {}'.format(len(b), b.hex())) 423 w.write(b) 424 425 426 @staticmethod 427 def to_key(b): 428 r = b'' 429 r += PFX_LEDGER_HEAD 430 t = time.time_ns() 431 v = int(t / 1000000000) 432 b = v.to_bytes(4, byteorder='big') 433 v = t - (v * 1000000000) 434 b += v.to_bytes(4, byteorder='big') 435 r += b 436 437 return r 438 439 440 class LedgerItem(Ledger): 441 442 credit_alice = 0 443 credit_bob = 0 444 collateral_alice = 0 445 collateral_bob = 0 446 ms = 0 447 448 def __init__(self, head, signer, generator, signer_name=None, parent=None, body=NOBODY, bob_name='bob'): 449 self.head = head 450 self.parent = parent 451 if self.parent == None: 452 self.parent = b'\x00' * 64 453 #self.timestamp = time.time_ns() 454 self.timestamp = b'' 455 t = time.time_ns() 456 v = int(t / 1000000000) 457 self.timestamp += v.to_bytes(4, byteorder='big') 458 v = t - (v * 1000000000) 459 self.timestamp += v.to_bytes(4, byteorder='big') 460 461 self.body = LedgerItemContent() 462 463 signer_role = None 464 if signer_name != None: 465 signer_role=LedgerRole(signer_name) 466 delta = generator.delta(signer_role=signer_role) 467 self.signer_sequence = [] 468 if delta[2]: 469 self.signer_sequence = [bob_name, 'alice'] 470 else: 471 self.signer_sequence = ['alice', bob_name] 472 self.credit_delta = delta[0] 473 self.collateral_delta = delta[1] 474 475 self.request_signature = NOSIG 476 self.response_signature = NOSIG 477 self.signer = signer 478 479 480 def to_asn1(self, data_dir, mode=LedgerMode.FINAL): 481 o = KeeEntry() 482 o['parent'] = self.parent 483 o['timestamp'] = self.timestamp 484 o['creditDelta'] = self.credit_delta 485 o['collateralDelta'] = self.collateral_delta 486 487 (k, v) = self.body.kv() 488 #self.data_add(data_dir, k, v) 489 data_add(data_dir, k, v) 490 o['body'] = k 491 492 o['response'] = False 493 o['signatureRequest'] = NOSIG 494 o['signatureResponse'] = NOSIG 495 496 if mode == LedgerMode.REQUEST: 497 return o 498 499 logg.debug('encoding new ledger_item for request signature {}: {}'.format(self.head.hex(), o)) 500 b = der_encode(o) 501 self.request_signature = self.signer.sign(self.signer_sequence[0], self.head + b) 502 o['signatureRequest'] = self.request_signature 503 504 if mode == LedgerMode.RESPONSE: 505 return o 506 507 if mode != LedgerMode.FINAL: 508 raise ValueError("invalid ledger mode: {}".format(mode)) 509 510 o['response'] = True 511 b = der_encode(o) 512 self.response_signature = self.signer.sign(self.signer_sequence[1], self.head + b) 513 o['signatureResponse'] = self.response_signature 514 return o 515 516 517 def serialize(self, data_dir, mode=LedgerMode.FINAL, w=sys.stdout.buffer): 518 o = self.to_asn1(data_dir, mode=mode) 519 b = der_encode(o) 520 flag = b'\x00' 521 if self.signer_sequence[0] != 'alice': 522 flag = b'\x01' 523 w.write(b + flag) 524 525 LedgerItem.ms += 1 526 if LedgerItem.ms > 999: 527 LedgerItem.ms = 0 528 529 530 @staticmethod 531 def to_key(v, k): 532 r = b'' 533 r += PFX_LEDGER_ENTRY 534 r += k 535 536 o = der_decode(v, asn1Spec=KeeEntry()) 537 #ts = o[0]['timestamp'] 538 #tsb = int(ts).to_bytes(8, byteorder='big') 539 tsb = o[0]['timestamp'].asOctets() 540 #logg.debug('ts {} ({}): of {}'.format(ts, tsb, v.hex())) 541 r += tsb 542 return r 543 544 545 def generate_ledger_item(data_dir, signer, generator, head, bob_name, parent): 546 o = LedgerItem(head, signer, generator, parent=parent, bob_name=bob_name) 547 w = io.BytesIO() 548 r = o.serialize(data_dir, w=w) 549 h = hashlib.new('sha512') 550 b = w.getvalue() 551 h.update(b) 552 z = h.digest() 553 return (z, b, o,) 554 555 556 def generate_ledger(dbi, data_dir, signer, bob_name, ledger_item_count=3, alice=None, bob=None): 557 r = [] 558 o = LedgerHead(alice_key=alice, bob_key=bob) 559 w = io.BytesIO() 560 o.serialize(data_dir, w=w) 561 h = hashlib.new('sha512') 562 b = w.getvalue() 563 h.update(b) 564 z = h.digest() 565 r.append((z, b, o,)) 566 567 k = z 568 parent = None 569 LedgerItem.credit_alice = random.randint(100, 1000) 570 LedgerItem.credit_bob = random.randint(100, 1000) 571 572 generator = LedgerGenerator() 573 for i in range(ledger_item_count): 574 try: 575 v = generate_ledger_item(data_dir, signer, generator, k, bob_name, parent) 576 except OverflowError: 577 break 578 # \todo generate key value already here 579 parent = v[0] 580 r.append(v) 581 582 return r 583 584 585 if __name__ == '__main__': 586 d = os.path.dirname(__file__) 587 data_dir = os.path.join(d, 'testdata', 'resource') 588 try: 589 shutil.rmtree(data_dir) 590 except FileNotFoundError: 591 pass 592 os.makedirs(data_dir) 593 594 v = b'foo' 595 h = hashlib.sha512() 596 h.update(v) 597 k = h.digest() 598 data_add(data_dir, k, v) 599 600 o = LedgerContent(subject='foo', body='bar') 601 (k, v) = o.kv() 602 data_add(data_dir, k, v) 603 604 d = os.path.dirname(__file__) 605 crypto_dir = os.path.join(d, 'testdata', 'crypt') 606 try: 607 shutil.rmtree(crypto_dir) 608 except FileNotFoundError: 609 pass 610 os.makedirs(crypto_dir) 611 612 d = os.path.dirname(__file__) 613 crypto_dir_r = os.path.join(d, 'testdata', 'crypt_reverse') 614 try: 615 shutil.rmtree(crypto_dir_r) 616 except FileNotFoundError: 617 pass 618 os.makedirs(crypto_dir_r) 619 620 d = db_init(d) 621 622 env = lmdb.open(d) 623 dbi = env.open_db() 624 625 signer = LedgerSigner(crypto_dir) 626 alice = signer.create_key('alice', outdir=data_dir) 627 628 keys = ['alice'] 629 630 alice_key = os.path.join(crypto_dir, 'alice.key.bin') 631 alice_key_sym = 'kee.key' 632 od = os.getcwd() 633 os.chdir(crypto_dir) 634 os.symlink(os.path.basename(alice_key), alice_key_sym) 635 os.chdir(od) 636 637 count_ledgers = os.environ.get('COUNT', '1') 638 items_min_in = os.environ.get('ITEM_MIN', '1') 639 items_max_in = os.environ.get('ITEM_MAX', '20') 640 641 mainbob_keygrip = None 642 with env.begin(write=True) as tx: 643 for i in range(int(count_ledgers)): 644 bob_name = 'Bob ' + fake.last_name() 645 keys.append(bob_name) 646 alias = None 647 if i == 0: 648 alias = 'bob' 649 bob = signer.create_key(bob_name, outdir=data_dir, pin='4321', alias=alias) 650 if i == 0: 651 mainbob_keygrip = bob[1] 652 653 c = 2 + random.randint(int(items_min_in), int(items_max_in)) 654 655 r = generate_ledger(dbi, data_dir, signer, bob_name, ledger_item_count=c, alice=alice[0], bob=bob[0]) 656 657 v = r.pop(0) 658 659 ledger_object = v[2] 660 661 z = v[0] 662 k = LedgerHead.to_key(v[0]) 663 tx.put(k, v[1]) 664 # reverse lookup 665 kr = b'\xff' + v[0] 666 tx.put(kr, k[1:]) 667 668 for v in r: 669 k = LedgerItem.to_key(v[1], z) 670 tx.put(k, v[1]) 671 672 for k in keys: 673 pubk = signer.get_pubkey(k) 674 name = signer.get_name(k).encode('utf-8') 675 tx.put(PFX_LEDGER_PUBKEY + pubk, b'CN=' + name) 676 677 # generate ledger import 678 bob_name = 'Bob ' + fake.last_name() 679 keys.append(bob_name) 680 bob = signer.create_key(bob_name, outdir=data_dir) 681 bob_key = os.path.join(crypto_dir, 'bob.key.bin') 682 bob_key_sym = 'kee.key' 683 od = os.getcwd() 684 os.chdir(crypto_dir_r) 685 os.symlink(os.path.basename(bob_key), bob_key_sym) 686 os.symlink(os.path.basename(bob_key), mainbob_keygrip) 687 os.chdir(od) 688 689 r = generate_ledger(dbi, data_dir, signer, bob_name, ledger_item_count=1, alice=alice[0], bob=bob[0]) 690 d = os.path.dirname(__file__) 691 import_dir = os.path.join(d, 'testdata', 'import') 692 try: 693 shutil.rmtree(import_dir) 694 except FileNotFoundError: 695 pass 696 os.makedirs(import_dir) 697 698 ledger_object = r[0][2] 699 ledger_item_object = r[1][2] 700 importer = LedgerBundle(data_dir, ledger_object) 701 w = io.BytesIO() 702 fp = os.path.join(import_dir, 'request') 703 f = open(fp, 'wb') 704 importer.encode(ledger_item_object, LedgerMode.REQUEST, w=f) 705 f.close() 706 707 w = io.BytesIO() 708 fp = os.path.join(import_dir, 'response') 709 f = open(fp, 'wb') 710 importer.encode(ledger_item_object, LedgerMode.RESPONSE, w=f) 711 f.close() 712 713 w = io.BytesIO() 714 fp = os.path.join(import_dir, 'final') 715 f = open(fp, 'wb') 716 importer.encode(ledger_item_object, LedgerMode.FINAL, w=f) 717 f.close()