index.py (7074B)
1 import tempfile 2 import json 3 import datetime 4 import logging 5 import hashlib 6 import uuid 7 import io 8 9 import lxml.etree 10 from epub_utils import Document 11 import ebooklib 12 from ebooklib import epub 13 14 logg = logging.getLogger('index') 15 16 17 def parse_ns_string(s): 18 c = s.rfind('}') 19 if c > 0: 20 c += 1 21 return s[c:].lower() 22 23 24 def meta_to_string(meta, fltr=[], prefix=False, single=False): 25 r = [] 26 if isinstance(meta, str): 27 return meta 28 elif isinstance(meta, list): 29 for v in meta: 30 v = meta_to_string(v, fltr=fltr, prefix=prefix) 31 if v != None: 32 if single: 33 return v 34 r.append(v) 35 return r 36 elif isinstance(meta, tuple): 37 v = None 38 i = 0 39 for k in meta[1].keys(): 40 i = 1 41 if len(fltr) > 0: 42 kk = parse_ns_string(k) 43 vv = meta[1][k].lower() 44 fltrv = '{}={}'.format(kk, vv) 45 if fltrv in fltr: 46 pfx = '' 47 if prefix: 48 pfx = vv + ':' 49 v = meta[0] 50 if len(pfx) == 0 or pfx not in meta[0]: 51 v = pfx + meta[0] 52 fltr=[] # skip further checks 53 else: 54 v = meta[0] 55 if i == 0: 56 v = meta[0] 57 return v 58 else: 59 raise ValueError('cannot process value type: ' + str(type(meta))) 60 61 62 def nav_to_jsonl(self): 63 pass 64 65 66 class TreeParser(): 67 68 def __init__(self): 69 self.items = [] 70 self.typ = None 71 self.s = '' 72 self.i = -1 73 74 75 def start(self, tag, attr): 76 tag = parse_ns_string(tag) 77 if tag.lower() == 'p': 78 self.typ = 'p' 79 elif tag.lower() == 'h1': 80 self.typ = 'h1' 81 else: 82 logg.info('unhandled tag {}'.format(tag)) 83 84 85 def end(self, tag): 86 if self.typ != None: 87 self.items.append((self.typ, self.s,)) 88 self.s = '' 89 self.typ = None 90 91 92 def comment(self, text): 93 pass 94 95 96 def data(self, text): 97 if len(self.s) > 0: 98 self.s += "\n" 99 self.s += text.strip() 100 101 102 def close(self): 103 return 'ok' 104 105 106 def __iter__(self): 107 self.i = 0 108 return self 109 110 111 def __next__(self): 112 v = None 113 try: 114 v = self.items[self.i] 115 except IndexError: 116 self.i = -1 117 raise StopIteration() 118 self.i += 1 119 return v 120 121 122 class Index(): 123 124 uuns = uuid.UUID('9104f188-2d2f-4d10-903e-7526c7df0df5') 125 126 def __init__(self, path, output_dir=None, uniquename=None): 127 self.path = path 128 self.title = None 129 self.authors = None 130 self.uu = None 131 self.z = None 132 self.ids = None 133 self.year = None 134 self.ln = [] 135 self.uniquename = uniquename 136 137 self._sum() 138 self.oa = Document(self.path) 139 self.ob = epub.read_epub(self.path) 140 141 142 def _sum(self): 143 h = hashlib.sha256() 144 f = open(self.path, 'rb') 145 v = f.read() 146 h.update(v) 147 f.close() 148 self.z = h.digest() 149 150 h = hashlib.sha256() 151 f = open(self.path, 'rb') 152 v = f.read() 153 h.update(v) 154 f.close() 155 self.z = h.digest() 156 157 if self.uniquename == None: 158 self.uniquename = self.z 159 if isinstance(self.uniquename, str): 160 self.uniquename = self.uniquename.encode('utf-8') 161 self.uu = uuid.uuid5(self.uuns, self.uniquename) 162 163 # 164 # def _lines_spine(self): 165 # for v in self.oa.package.spine.itemrefs: 166 # s = self.oa.find_content_by_id(v['idref']) 167 # #self.ln.append((0, 0, v['idref'], lxml.etree.tostring(s.to_xml()),)) 168 # self.ln.append((0, 0, v['idref'], '',)) 169 # 170 # 171 # def _lines_nav(self): 172 # nav = self.oa.ncx 173 # if nav == None: 174 # nav = self.oa.nav 175 # for v in nav.get_toc_items(): 176 # try: 177 # s = self.oa.find_content_by_id(v.id) 178 # except ValueError as e: 179 # logg.error('err {}'.format(e)) 180 # continue 181 # self.ln.append((v.order, v.level, v.label, s.to_str(),)) 182 # logg.debug('toc {}'.format(v)) 183 184 185 def _lines_nav_alt_content(self, s): 186 o = TreeParser() 187 parser = lxml.etree.XMLParser(target=o) 188 r = lxml.etree.XML(s, parser) 189 return o 190 191 192 def _lines_nav_alt_sub(self, o): 193 for v in o: 194 try: 195 s = self.oa.find_content_by_id(v.id) 196 except ValueError as e: 197 logg.error('err {}'.format(e)) 198 continue 199 i = 0 200 for item in self._lines_nav_alt_content(s.to_str()): 201 self.ln.append((v.order, v.level, i, v.label, item[0], item[1],)) 202 i += 1 203 logg.debug('child {}'.format(v)) 204 205 206 def _lines_nav_alt(self): 207 nav = self.oa.ncx 208 if nav == None: 209 nav = self.oa.nav 210 for v in nav.get_toc_items(): 211 if len(v.children) == 0: 212 continue 213 self._lines_nav_alt_sub(v.children) 214 215 216 def _lines(self): 217 return self._lines_nav_alt() 218 219 220 def load(self): 221 self.d = tempfile.mkdtemp() 222 self.title = self.ob.get_metadata('DC', 'title') 223 self.authors = self.ob.get_metadata('DC', 'creator') 224 self.ids = self.ob.get_metadata('DC', 'identifier') 225 self.year = self.ob.get_metadata('DC', 'date') 226 self._lines() 227 228 229 def export_metadata(self): 230 dt = meta_to_string(self.year, fltr=['event=publication'], single=True) 231 year = None 232 try: 233 year = datetime.datetime.fromisoformat(dt).year 234 except TypeError: 235 pass 236 title = meta_to_string(self.title, single=True) 237 authors = meta_to_string(self.authors, fltr=['role=aut']) 238 identifiers = meta_to_string(self.ids, fltr=['scheme=isbn', 'scheme=uuid'], prefix=True) 239 o = { 240 'uuid': str(self.uu), 241 'sha256': self.z.hex(), 242 'title': title, 243 'authors': authors, 244 'identifiers': identifiers, 245 'year': year, 246 } 247 return o 248 249 250 def _lines_split_content(self): 251 lxm 252 253 def export_lines(self, w, jsonl=False): 254 if not jsonl: 255 w.write("[") 256 i = 0 257 for v in self.ln: 258 if jsonl: 259 w.write("\n") 260 elif i > 0: 261 w.write(",") 262 o = { 263 'uuid': str(self.uu), 264 'globalseq': i, 265 'order': v[0], 266 'level': v[1], 267 'seq': v[2], 268 'label': v[3], 269 'type': v[4], 270 'contents': v[5], 271 } 272 w.write(json.dumps(o)) 273 i += 1 274 if not jsonl: 275 w.write("]") 276 277 278 #def __str__(self): 279 #return json.dumps(self.export()) 280 #return str(self.export_lines())