commit 9c5a606a498763fb226b3378ebbea9607c64c3d7 parent ca3b23daa5ac1dfbc59e2369fdab9bcd77aa9621 Author: lash <dev@holbrook.no> Date: Sat, 1 Oct 2022 13:43:51 +0000 Add footnote making kitab more accessible Diffstat:
16 files changed, 354 insertions(+), 336 deletions(-)
diff --git a/09c60de8503f5115c11d7059476de0a930bdeaf0511609d24ed93d15aadef47b.asc b/09c60de8503f5115c11d7059476de0a930bdeaf0511609d24ed93d15aadef47b.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4PhwACgkQzcvSTdHQ -4AGPuwf+KpTAkzdTTDfoFkXy9MvYE/sg0VDCckYS0jRgmAtwLhyVeXK8SlG/4Haa -LFOk0y/905kc60AjHdqZ5E3QjofylT9HR8v5wQ+OfqIITYvjGxIRmNMat99JygSe -35oiPv8x6eIwwczFeSteSU+pIaiZa1q4oyCv6Ivv6nONWNSgD9CmL4WdAvSK6hZm -LyDVk4xLjgroF+Vkj0faDYqCpzQZ2kG4AZgRyAbdckHy7OUTYRP6hqQuTle5uCT5 -ykOGaQsYCqSmckDh/re1UsJXza6vs31bdAwgopansThSDPEjhOr3as597VASkLe7 -i+Xye5Rq2bWBIbfJygc+4UNUpzAzjg== -=lNRe +iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4RBYACgkQzcvSTdHQ +4AFW0Qf+KBuzE+eT48GOVBo99JZJHmVExVDEc3QfhbiljgFqmW9riVRKFosBXCCY +DpQZ4fwkCgbPIzE4MRvaTJMb/a/SisR4+gfswZktIYJRoauMoDvgzM8dHNbURLK7 +g5+eesbUqU9IZ79Rpz5EaIHe5fl7Qv/YgtXi1swU8IUCxeLbT2hILhGLI+PFGrv+ +rMIenT+hd5Q7zJQu4U32tu1D/l/OYoqUfIIm5cUV197z1c8Pd6LiuORGQMvi+74A +nTnRo3VeCQt4tiBLybGsgzbwReTLLaL6FmY4QozHzLfmjLDjm4D5vWrNME4URGsq +nLrC4pAZwK+Bp9Jhp1xRn6Ta4dLdNA== +=F5qt -----END PGP SIGNATURE----- diff --git a/1147e630f5d17d981407c49cfc72bb0b340e2b672044f7df387490f23c8700c0.asc b/1147e630f5d17d981407c49cfc72bb0b340e2b672044f7df387490f23c8700c0.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4PhwACgkQzcvSTdHQ -4AFC9wf/fhTcnIrN9Tk6wY07hAMEUXYSad5YCr5ugAWH8d4YxvJPe8bGLBsCysIQ -IPc0WiXYbdAQFswZhe2n0BzfWX5fmlePVB8I8vqpM1CIFvP98f7Dg0MAmx8x4onk -f1gXHHRzekIdGocK7r59ZWJaX3Ah3rG4g1sYULri54fGomxMOg6iOntC/HE38Vqg -sxVFcgaq+yb+s2/V/l5FrL1/IZZMQSr2zbjcJRCbi9IUVGiCBWpJXERsckvT68nR -Ea3TNDMQDP4pYqkLm6OVsgIA9mF59uaOJHiiKPq6758GMDKe1SyGDDBozwLfccfB -LHZ5CAy30ttm+4uL6ckgrnJjW2Z7yw== -=NUFH +iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4RBYACgkQzcvSTdHQ +4AGsFAgAg85b7GdNy7TV4leildZ3ycpBnYStnfoBcZnZoCJcQpSdKRSBmYja3+dt +mcXPDhf1saD2QnRT4U8wurtdrIx9gGoAoAIpvAPmgjhBIq+U+F63BJ6zG1mJIdmt +0S02oWJFVui5IwCkoXbe53Q9XK6VEwICDbJWSCnfs3icFZ9IFEucsRsor6lK0SzW +PnSLSk3BnbjKqpEZ5xoLIeFZZE0bTDFXcH5r07AwiieAo+1BcMTc07srkvPcneIb +IBzAumXgdBIT22fDYy32imr6SpU9EnKKZZ9+HsZPKQ+cSCoXoZnPxeG0YgBDqgaC +FpP1fvzLEwMT4uQb2g84nOZNpzF8mg== +=/enN -----END PGP SIGNATURE----- diff --git a/24345ef2cf2edbf32d9ec3e55a6e0801a5a73f8337bb81bc1a7395ebbe3bf7e0.asc b/24345ef2cf2edbf32d9ec3e55a6e0801a5a73f8337bb81bc1a7395ebbe3bf7e0.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4PhwACgkQzcvSTdHQ -4AGHBQgAl8FigfzBG4qYcIa5tklugEF9w0ft4b6vYi6MnK9guK4F0oxVw/xxGKGR -LJ9BIovTSc+xLqW/BJQ9Kaqhs9rgiBL/1oQmURnh4ZPhmTIgkdKSW1NrnKA/KGCj -nXB98NvlmfGp1m+sNEZpA1Qq7a8UZaxzcDQXxOc3WmB9XnLkUkaep62i634bzd56 -jnuvWq/byHpz8wsVMJpK6QFYvsehpcVspbiH/2KHdp6BlF9ewZQjv2T2okkjawFg -ktZc0efkcCS6K4Fg1Z3t5bbxQPwdDcWtQMAWZ3NkL0kOPqXYrp7F9CwcUVGo4FRN -d/X0r0+kXv7sRfPMSggDAFBZiMK4Hg== -=VOlI +iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4RBYACgkQzcvSTdHQ +4AE4DwgAhMJoZbwt2N1FuCWUfoxaZzOeCgzI3Tz7ddhg+CLJ6cFTii54N9yRYPyX +N6rw3FAXOv3BFFuoJuhA78iGiPN15USMIhEoLKwKOAcocmX5teNMk+Ggg0v3vhPb +chBsQZbfoqO4ZHAvnO8Xc1VC7zRH2FX1p3N/qHM/kWiiNXitjYGZRIFcP6NifU64 +r5b/QNvvq8gwQqrz1TEr2rucKNTUljoDJEb5mr8Vy4uw9AOIe0OO4SaDOiBdNC17 +UazyOdHQYqG9kZHJt7547O7kIX5Xgykk4ztGCjwJ9n3Ywb0lb5A1ydF0dMPTNXFM +rm/04TMWY7Ok/z3gEG1rhGrcNNpm8Q== +=mTR3 -----END PGP SIGNATURE----- diff --git a/37f77922ca667d5ced9d58cdc642bacb89fa428365585bad4aa09a6e9ab67d82 b/37f77922ca667d5ced9d58cdc642bacb89fa428365585bad4aa09a6e9ab67d82 @@ -1,237 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> -<head> - <link rel="stylesheet" type="text/css" href="./theme/css/style.css" /> - <link rel="stylesheet" type="text/css" href="./theme/css/syntax.css" /> - <title>man bytes gnu - A portable book metadata exercise</title> - <meta charset="utf-8" /> - <meta name="generator" content="Pelican" /> - -</head> - -<body id="index" class="home"> - - <header id="banner" class="body"> - <h1>> <a href="./">man bytes gnu</a><span id="cursor">_</span></h1> - </header> - <nav id="menu"> - <ul> - <li class="active"><a href="./category/archiving.html">Archiving</a></li> - <li><a href="./category/code.html">Code</a></li> - <li><a href="./category/hygiene.html">Hygiene</a></li> - <li><a href="./category/offlining.html">Offlining</a></li> - <li><a href="./pages/identities.html">Identities</a></li> - <li><a href="./pages/shares.html">Shares</a></li> - - <li><a href="/tags.html">tags</a></li> - </ul> - <hr/> - </nav><!-- /#menu --> - <header> - <h1 class="entry-title top-body-title"> - <a href="./portable-book-metadata.html" rel="bookmark" - title="Permalink to A portable book metadata exercise">A portable book metadata exercise</a></h1> - - - <div class="category meta"> - Posted -<time class="published" datetime="2022-10-01T12:40:00+02:00"> - Sat 01 October 2022 - </time> -in <a class="category" href="./category/archiving.html">archiving</a> - <a href="./tag/hash.html">hash</a> - <a href="./tag/kitab.html">kitab</a> - <a href="./tag/literature.html">literature</a> - <a href="./tag/metadata.html">metadata</a> - <a href="./tag/dublincore.html">dublincore</a> - <a href="./tag/libgen.html">libgen</a> - </div> - - <div class="neighbors meta"> - <ul> - <li> - Previous: <a href="./backup-rsync-duplicity.html"> - Combining duplicity and rsync - </a> - </li> - </ul> - </div> - <div class="meta"> -<hr/> - </div> -</header> - -<section id="content" class="body"> - <div class="entry-content"> - <p>One of the things I have been working on the last few weeks is a rust application I have dubbed <a class="reference external" href="https://git.defalsify.net/kitab">kitab</a>.</p> -<p>In short, the application makes it easy to extract literary metadata to a separate file structure.</p> -<p>The metadata can in turn be applied as <em>extended attributes</em> recursively on a directory for files that match.</p> -<p>The way it's accomplished it simple: The file name of the metadata is the hex representation of the digest of the file. The same digest is used to match files to metadata when applying it back to the file.</p> -<p>There are two advantages to this:</p> -<ol class="arabic simple"> -<li>The digest of the media file need not be affected by the metadata, i.e. by embedding metadata in the file itself.</li> -<li>You do not need to use the file name to keep record of what a file is.</li> -</ol> -<div class="section" id="yarr-ye-matey-data"> -<h2>Yarr, ye matey-data</h2> -<p>Let's demonstrate with an example.</p> -<p>The fabulous <a class="reference external" href="https://libgen.rs">Library Genesis</a> project has made available an endpoint to retrieve <tt class="docutils literal">bibtex</tt> entries based on the <tt class="docutils literal">md5</tt> hash of the book media file.</p> -<p>A version of the <a class="reference external" href="https://libgen.rs/book/index.php?md5=BCD99F1AB4155F2A2A362E5B7938A852">Bitcoin White Paper</a>, under the <code>md5</code> hash <code>bcd99f1ab4155f2a2a362e5b7938a852</code>, can be found there.</p> -<p>If you download this file using a synchronous download link, the browser will provide you with a filename to go with the download.</p> -<p>However, if you use the torrent alternative, the filename will be the <tt class="docutils literal">md5</tt> hash itself. If you are torrenting a bunch of those files, it quickly becomes a nuisance to distinguish them.</p> -<p>And, of course: In either case there is no guarantee the any metadata comes with the file.</p> -<div class="section" id="inside-the-book"> -<h3>Inside the book</h3> -<p>Kitab (v0.0.2) is able to read metadata from both a bibtex source and xattr entries on a file, as well as its native <a class="reference external" href="https://www.w3.org/TR/turtle/">rdf-turtle</a> format.</p> -<p>In kitab's data store, every media file entity in rdf-turtle is keyed with a <a class="reference external" href="https://www.rfc-editor.org/info/rfc8141">URN</a> specifying a digest for the file.</p> -<p>To see exactly what that looks like, let's download and import the bibtex metadata for the paper <a class="footnote-reference" href="#footnote-1" id="footnote-reference-1">[1]</a>:</p> -<pre class="code bash literal-block"> -<span class="nv">bibtex_file</span><span class="o">=</span><span class="sb">`</span>mktemp<span class="sb">`</span> -<span class="nv">kitab_dir</span><span class="o">=</span><span class="sb">`</span>mktemp -d<span class="sb">`</span> -curl -s -X GET https://libgen.rs/book/bibtex.php?md5<span class="o">=</span>BCD99F1AB4155F2A2A362E5B7938A852 -o <span class="nv">$bibtex_file</span> -kitab --store <span class="nv">$kitab_dir</span> import --digest md5:BCD99F1AB4155F2A2A362E5B7938A852 <span class="nv">$bibtex_file</span> -cat <span class="nv">$kitab_dir</span>/* -</pre> -<p>The output of the above should be:</p> -<pre class="code turtle literal-block"> -<span class="nv"><URN:md5:bcd99f1ab4155f2a2a362e5b7938a852></span> <span class="nv"><https://purl.org/dc/terms/title></span> <span class="s">"Bitcoin: A Peer-to-Peer Electronic Cash System"</span> <span class="p">;</span> -<span class="nv"><https://purl.org/dc/terms/creator></span> <span class="s">"Satoshi Nakamoto"</span> <span class="p">;</span> -<span class="nv"><https://purl.org/dc/terms/type></span> <span class="s">"book"</span> <span class="p">.</span> -</pre> -<p>Now let's say the media file itself has been downloaded to <tt class="docutils literal"><span class="pre">~/.local/share/transmission</span></tt>. We can apply this metadata as extended attributes.</p> -<p>This time we turn on logging to see what's going on:</p> -<pre class="code console literal-block"> -<span class="gp">$ </span><span class="nv">RUST_LOG</span><span class="o">=</span>info kitab --store <span class="nv">$kitab_dir</span> apply --digest md5 ~/.local/share/transmission -<span class="go">[2022-10-01T11:14:59Z INFO kitab] have index directory "/tmp/tmp.r0jBm6q4hW" -[2022-10-01T11:14:59Z INFO kitab] using digest type md5 -[2022-10-01T11:14:59Z INFO kitab] apply from path "/home/lash/.local/share/transmission/" -[2022-10-01T11:14:59Z INFO kitab] apply DirEntry("/home/lash/.local/share/transmission/bcd99f1ab4155f2a2a362e5b7938a852") -> title "Bitcoin: A Peer-to-Peer Electronic Cash System" author "Satoshi Nakamoto" digest md5:bcd99f1ab4155f2a2a362e5b7938a852 - -</span><span class="gp">$ </span>find ~/.local/share/transmission -type f -regextype sed -regex <span class="s2">".*/[a-f0-9]\{32\}</span>$<span class="s2">"</span> -exec getfattr -d <span class="o">{}</span> <span class="se">\;</span> -<span class="gp"># </span>file: .local/share/transmission/bcd99f1ab4155f2a2a362e5b7938a852 -<span class="go">user.dcterms:creator="Satoshi Nakamoto" -user.dcterms:title="Bitcoin: A Peer-to-Peer Electronic Cash System" -user.dcterms:type="book"</span> -</pre> -</div> -<div class="section" id="let-the-right-one-in"> -<h3>Let the right one in</h3> -<p>Conversely, the metadata can be re-imported directly from the extended attributes. And this time, let's store it both under the <tt class="docutils literal">md5</tt> and the <tt class="docutils literal">sha512</tt> hash:</p> -<pre class="code bash literal-block"> -$ <span class="nv">kitab_dir_new</span><span class="o">=</span><span class="sb">`</span>mktemp -d<span class="sb">`</span> -$ kitab --store <span class="nv">$kitab_dir_new</span> import --digest md5 --digest sha512 .local/share/transmission/bcd99f1ab4155f2a2a362e5b7938a852 -$ find <span class="nv">$kitab_dir_new</span> -type f -exec cat <span class="o">{}</span> <span class="se">\;</span> -/tmp/tmp.B6j41YMmEM/493f2a720d63156d77187bcd5f0715e4e765a38d616ef47f24e0df817ee6b4f601d47a06ffae10ef1f6ba60bb5d2e99a26318f035f9cd56e30bfe7bcdf64a792 -<URN:sha512:493f2a720d63156d77187bcd5f0715e4e765a38d616ef47f24e0df817ee6b4f601d47a06ffae10ef1f6ba60bb5d2e99a26318f035f9cd56e30bfe7bcdf64a792> <https://purl.org/dc/terms/title> <span class="s2">"Bitcoin: A Peer-to-Peer Electronic Cash System"</span> <span class="p">;</span> - <https://purl.org/dc/terms/creator> <span class="s2">"Satoshi Nakamoto"</span> <span class="p">;</span> - <https://purl.org/dc/terms/type> <span class="s2">"book"</span> <span class="p">;</span> - <https://purl.org/dc/terms/MediaType> <span class="s2">"application/epub+zip"</span> . -/tmp/tmp.B6j41YMmEM/bcd99f1ab4155f2a2a362e5b7938a852 -<URN:md5:bcd99f1ab4155f2a2a362e5b7938a852> <https://purl.org/dc/terms/title> <span class="s2">"Bitcoin: A Peer-to-Peer Electronic Cash System"</span> <span class="p">;</span> - <https://purl.org/dc/terms/creator> <span class="s2">"Satoshi Nakamoto"</span> <span class="p">;</span> - <https://purl.org/dc/terms/type> <span class="s2">"book"</span> <span class="p">;</span> - <https://purl.org/dc/terms/MediaType> <span class="s2">"application/epub+zip"</span> . -</pre> -</div> -</div> -<div class="section" id="level-up"> -<h2>Level up</h2> -<p>Finally, a bash script <a class="footnote-reference" href="#footnote-2" id="footnote-reference-2">[2]</a> example that lets you retrieve and apply metadata for a batch of files found in the directory given as the <em>first positional arg</em>.</p> -<p>This script even renames the files according to the metadata applied.</p> -<pre class="code bash literal-block"> -<span class="ln"> 0 </span><span class="c1"># NOTE! this will only work if your fs supports xattr. -</span><span class="ln"> 1 </span><span class="c1"># That's why we cannot use tmpfs (mktemp) here; tmpfs does not support xattr. -</span><span class="ln"> 2 </span><span class="c1"></span> -<span class="ln"> 3 </span><span class="c1"># directory to copy media files to -</span><span class="ln"> 4 </span><span class="c1"></span><span class="nv">outdir</span><span class="o">=</span>./<span class="k">$(</span>uuidgen<span class="k">)</span> -<span class="ln"> 5 </span>mkdir -vp <span class="nv">$outdir</span> -<span class="ln"> 6 </span> -<span class="ln"> 7 </span><span class="c1"># Input dir is the first positional arg. -</span><span class="ln"> 8 </span><span class="c1"></span><span class="nv">indir</span><span class="o">=</span><span class="nv">$1</span> -<span class="ln"> 9 </span> -<span class="ln">10 </span><span class="nv">IFS</span><span class="o">=</span><span class="s1">$'\n'</span> -<span class="ln">11 </span> -<span class="ln">12 </span><span class="c1"># Retrieve metadata for each file and import it into the kitab store. -</span><span class="ln">13 </span><span class="c1"># Also copy the media file to the separate output directory. -</span><span class="ln">14 </span><span class="c1"></span><span class="k">for</span> f <span class="k">in</span> <span class="k">$(</span>find <span class="nv">$indir</span> -type f<span class="k">)</span><span class="p">;</span> <span class="k">do</span> -<span class="ln">15 </span> <span class="nv">sum</span><span class="o">=</span><span class="k">$(</span>md5sum <span class="nv">$f</span> <span class="p">|</span> awk <span class="s1">'{print $1;}'</span><span class="k">)</span> -<span class="ln">16 </span> <span class="nb">echo</span> <span class="s2">"downloading metadata for </span><span class="nv">$indir</span><span class="s2">/</span><span class="nv">$f</span><span class="s2">"</span> -<span class="ln">17 </span> <span class="nv">srct</span><span class="o">=</span><span class="k">$(</span>mktemp<span class="k">)</span> -<span class="ln">18 </span> curl -s -X GET https://libgen.rs/book/bibtex.php?md5<span class="o">=</span><span class="nv">$sum</span> -o <span class="nv">$srct</span> -<span class="ln">19 </span> <span class="nv">dstt</span><span class="o">=</span><span class="k">$(</span>mktemp<span class="k">)</span> -<span class="ln">20 </span> xmllint --html --xpath <span class="s1">'string(/html/body/textarea[@id="bibtext"])'</span> <span class="nv">$srct</span> > <span class="nv">$dstt</span> -<span class="ln">21 </span> kitab import --digest md5:<span class="nv">$sum</span> <span class="nv">$dstt</span> -<span class="ln">22 </span> cp <span class="nv">$f</span> <span class="nv">$outdir</span>/ -<span class="ln">23 </span><span class="k">done</span> -<span class="ln">24 </span> -<span class="ln">25 </span><span class="c1"># Apply metadata imported from bibtex as xattr for the media files. -</span><span class="ln">26 </span><span class="c1"></span><span class="nv">RUST_LOG</span><span class="o">=</span>info kitab apply --digest md5 <span class="nv">$outdir</span>/ -<span class="ln">27 </span> -<span class="ln">28 </span><span class="c1"># Rename the files according to the metadata title and media type. -</span><span class="ln">29 </span><span class="c1"></span><span class="k">for</span> f <span class="k">in</span> <span class="k">$(</span>ls <span class="nv">$outdir</span><span class="k">)</span><span class="p">;</span> <span class="k">do</span> -<span class="ln">30 </span> <span class="nv">title</span><span class="o">=</span><span class="k">$(</span>getfattr --only-values -n user.dcterms:title <span class="nv">$outdir</span>/<span class="nv">$f</span><span class="k">)</span> -<span class="ln">31 </span> -<span class="ln">32 </span> <span class="nv">f_typ</span><span class="o">=</span><span class="k">$(</span>file -b --mime-type <span class="nv">$outdir</span>/<span class="nv">$f</span><span class="k">)</span> -<span class="ln">33 </span> <span class="nv">f_ext</span><span class="o">=</span><span class="s2">""</span> -<span class="ln">34 </span> <span class="k">case</span> <span class="s2">"</span><span class="nv">$f_typ</span><span class="s2">"</span> <span class="k">in</span> -<span class="ln">35 </span> <span class="s2">"application/pdf"</span><span class="o">)</span> -<span class="ln">36 </span> <span class="nv">f_ext</span><span class="o">=</span><span class="s2">".pdf"</span> -<span class="ln">37 </span> <span class="p">;;</span> -<span class="ln">38 </span> <span class="s2">"application/epub+zip"</span><span class="o">)</span> -<span class="ln">39 </span> <span class="nv">f_ext</span><span class="o">=</span><span class="s2">".epub"</span> -<span class="ln">40 </span> <span class="p">;;</span> -<span class="ln">41 </span> <span class="s2">"application/x-mobipocket-ebook"</span><span class="o">)</span> -<span class="ln">42 </span> <span class="nv">f_ext</span><span class="o">=</span><span class="s2">".mobi"</span> -<span class="ln">43 </span> <span class="p">;;</span> -<span class="ln">44 </span> <span class="s2">"text/plain"</span><span class="o">)</span> -<span class="ln">45 </span> <span class="nv">f_ext</span><span class="o">=</span><span class="s2">".txt"</span> -<span class="ln">46 </span> <span class="p">;;</span> -<span class="ln">47 </span> <span class="s2">"text/html"</span><span class="o">)</span> -<span class="ln">48 </span> <span class="nv">f_ext</span><span class="o">=</span><span class="s2">".html"</span> -<span class="ln">49 </span> <span class="p">;;</span> -<span class="ln">50 </span> *<span class="o">)</span> -<span class="ln">51 </span> ><span class="p">&</span><span class="m">2</span> <span class="nb">echo</span> unhandled mime <span class="nb">type</span> <span class="nv">$f_typ</span> -<span class="ln">52 </span> <span class="nb">exit</span> <span class="m">1</span> -<span class="ln">53 </span> <span class="k">esac</span> -<span class="ln">54 </span> mv -v <span class="nv">$outdir</span>/<span class="nv">$f</span> <span class="nv">$outdir</span>/<span class="si">${</span><span class="nv">title</span><span class="si">}${</span><span class="nv">f_ext</span><span class="si">}</span> -<span class="ln">55 </span><span class="k">done</span> -</pre> -<p>This last example will result in:</p> -<ul class="simple"> -<li>A media file named <tt class="docutils literal">$outdir/Bitcoin: A <span class="pre">Peer-to-Peer</span> Electronic Cash System.epub</tt></li> -<li>... with metadata applied as extended attributes</li> -<li>An rdf-turtle metadata entry in <tt class="docutils literal"><span class="pre">~/.local/share/kitab/idx/bcd99f1ab4155f2a2a362e5b7938a852</span></tt></li> -</ul> -<!-- --> -<blockquote> -<table class="docutils footnote" frame="void" id="footnote-1" rules="none"> -<colgroup><col class="label" /><col /></colgroup> -<tbody valign="top"> -<tr><td class="label"><a class="fn-backref" href="#footnote-reference-1">[1]</a></td><td>The <code>kitab</code> command in the script assumes you have built the <em>kitab binary</em> and made it available in your path.</td></tr> -</tbody> -</table> -</blockquote> -<!-- --> -<blockquote> -<table class="docutils footnote" frame="void" id="footnote-2" rules="none"> -<colgroup><col class="label" /><col /></colgroup> -<tbody valign="top"> -<tr><td class="label"><a class="fn-backref" href="#footnote-reference-2">[2]</a></td><td>the script uses <code>xmllint</code> which on archlinux is provided by the <tt class="docutils literal">libxml2</tt> package.</td></tr> -</tbody> -</table> -</blockquote> -</div> - - </div><!-- /.entry-content --> - <footer class="meta"> - </footer> - -</section> - <footer id="contentinfo" class="body"> - <hr/> - <address id="about" class="vcard body"> - Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> - </address><!-- /#about --> - </footer><!-- /#contentinfo --> -</body> -</html> -\ No newline at end of file diff --git a/37f77922ca667d5ced9d58cdc642bacb89fa428365585bad4aa09a6e9ab67d82.asc b/37f77922ca667d5ced9d58cdc642bacb89fa428365585bad4aa09a6e9ab67d82.asc @@ -1,11 +0,0 @@ ------BEGIN PGP SIGNATURE----- - -iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4PhwACgkQzcvSTdHQ -4AEB9ggApnYXtjJpHP7hQyFwo2IKsyaF9pKLGeXEUfL2KAObqnNdAUkzbKo6yvxz -fquQv1C7FGeQagQcMaXggwPlpl1F7z9dFGNWFGm9MRSsLJZTdtO3DOqHe4outqsu -WWnfS1gP+/B3NoH7jf3zKSJTRms5lF+d979/g9zfJ0du34d2h5CI0BBVldMtrmZE -gQSAz48YuANXq0YYwde/I/KJdCXIymkoMXCPpG6YPiNuPt50yfWtS3jZsQ39yTbN -V4uEZxzE6CoodH+4OnIikt1JM46BgPIws9Yc1GTGecRSIk849laVwpCJvDxU92kW -oU728awqSSoLpR/mUdIDf2RTCB2Vzw== -=vn9K ------END PGP SIGNATURE----- diff --git a/617878ecd470e80c0f5fadc0cf7e669ceea086a54343e56a35aeaf8131038d6c.asc b/617878ecd470e80c0f5fadc0cf7e669ceea086a54343e56a35aeaf8131038d6c.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4PhwACgkQzcvSTdHQ -4AEdkggAhs4galtVUDOMY7lUeART8C498CCIwyZY4ZUt+fIguj294uyJRxsyywdl -U9G1dY6jniTi6O70+mFOzOcN4E27VrKBOETb6nv71HOXWV1IxBk8mLJVsSEKpzhU -UR9v0C1jZunykViArfRhSm9tRiJMhWIHQ9hYtlM/eECwQlT0bOr4AZsBtpn+aL3/ -DluGWtCUtJW78TgAQpzt1S6rwNIlI366pqs7re9LioLcWqsU4Vqv8bXAkUYQFwRk -6aEoIidwaTXiH57lukI+aio5/hcvpIbrSEH9T8OnXENtNFA9Zr0zaOXRfdbKlWJ0 -0E8+y1Nr8ft5+7KP+DFE8sDWiXZYOg== -=EVnR +iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4RBYACgkQzcvSTdHQ +4AH4LQf/QpPXHSeFij04B+m8/Msdzr6d3qNy94aGip1lceLPhxmbBi4/97R6KbtY +Ps3XjVw1+LGQQxs2VbWjSbyAIxXGxfcHaTkn6BVEY3d49TCMMeOc1Ac8W1ba7Ly5 +QXAJ5k8TJXiRSiAvEhpwjtIbov6YMVjFrj/n7lcF96p/aL6j7CO/f40gZEwkzP9s +Ey3TOj3cmS+8Ungl7qoUoGySWN21DrHLZ51oOa1HD1PegljrJu6kxn4GwVnc3U+X +rIXdeXFtynhNXr8hVPfIqLDUu8kdXA9I9hicrAa9J+1LqkzJZNKKT1MUEusp66Om +Mhx3kmyrqQ8xgnlCsOu5TAlU/La4Ng== +=afNx -----END PGP SIGNATURE----- diff --git a/788f1df1d56d3aeb5a7374de267f9ebd41865fefabacd46810270f1596856a71.asc b/788f1df1d56d3aeb5a7374de267f9ebd41865fefabacd46810270f1596856a71.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4PhwACgkQzcvSTdHQ -4AEhGwf/S64e0SlOcBxMl1AVD5foo5qf81Bj1y5PhW6yRxKWmb9FVLe1SZtV3W+n -xy0p22YYU5T992fB7Nr8gH3t1IXpj2aLvDYPf6NC4Wh058gF6XTydoftJBrssAHw -CS8LfDgzrxq8KRm71MLX9I1CDOKuwtiQcyVWU7jolhQp2UpNOhMvOcEvA7SEdZN5 -urMJdzYk6yzGIz39AUAA4cydWS+fgohkOMUU7yhskKogzZ6S0VDJIlS0WRWRk17V -+LVlJSl9/L76pAgxopnnyv1B9YJT+vOP/OQFIc/D0u/QrLWN3AyKY8wwMtkZMHBh -9r3NsgXN94wAF/QmqFrbwEF7V9ZloQ== -=2rFd +iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4RBYACgkQzcvSTdHQ +4AGOegf/VrVUSyijWhQOI5H31/m/4zru8ry/xng/84b1Jr/T1hB96FrCyJHNXbqq +RaStZ5g2Pn2COPFlP1bL494YsFb4/9DbNhjyn3a6IAfSC4e/PolRNPNc86ymPOdG +82TQ0+LI40ZCmHTDbeLdxsUyC8L3zFOrkkg5M5pZHRERZ8L8+FNs6OEkTEeZE2FW +xGcQMu+RYzakJ+33CywfrEUDu1wXVji/6S6hulT0VKTc6i8jADSuZXirrwj7phk2 +nT+P0NP0X2Ybsz+H/Roz0qnXlO5V7ZIy7S7qpNHOja7NOyEKtHx/NLJGntysplMl +sirpAhmlQ/aEs0Fr82gKo7zB16VIpQ== +=tNZt -----END PGP SIGNATURE----- diff --git a/80d5137e80648103735073b716f744b6e2210092a4c2ea556d87a931a3878c53.asc b/80d5137e80648103735073b716f744b6e2210092a4c2ea556d87a931a3878c53.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4PhwACgkQzcvSTdHQ -4AH1Qgf/QDq4Pnvx8JhCYOhdxWl5bPa7HPhew5CvApEjP3QHIM69pQ8bEwQ0WWUh -TdNbeKOChYSGMaYAsAh3OhkqwJQnVv2CSG2HX858AazvKHjbIo7cYlpwl5DcUZAi -kAfeW5XAxyTjQO3xA6S1j6Fw1Ngl8os3aYmvGyOAPCqkLtEXPCy0dEsKnQQXVSmY -AFwU8AWH3l+wRc5laErSUac2ehQ4X1HHpuU5i4EYPOnzCcHSaEFF9JCapJrN8T5R -D8vvJhf+lUiSvR2FPfO91ikzTpOrpwHA8sGhuw1XF/lxmmM5Z8NVRu9X3MQo6v3F -qM5Dh6KqKxVms5h1oImCzzk9Nl6ctQ== -=fHv+ +iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4RBYACgkQzcvSTdHQ +4AHTvgf/TnusNmm5xdYt5n9wv7D3C27bruLoKNl9ytXPFr3eqrTdXrHsjAjie5ue +4Iwe/g1KbEdH98i7amCGnSgp5UARxkDNHnX+5fbOsAve7AgFbMevkiLyF1ppR4oF +LJit4FXJKaZbHbG7UT3Jdrlz00SSFHHARfi/rYmV7xZgBuiM555hAGIKe9WG8PKD +TGVHKDjsY46JghiLydK++WDXQTcPDI0TR7hu26BfYeIfA8SRXvqxprjiwlA07T7K +tofxeXPphmBXsvH7m75uBaBzyxSqd1UAdYud6AW0YFwKYyuOj5QgTbOY5Xcy9bwW +aWfaOnCctDoq7jFMH67wayrPtj0LBw== +=QRPW -----END PGP SIGNATURE----- diff --git a/92eb83e7074b2760e6001d40b17eecffb874b23f036abd5b6d2328f28f49bac8.asc b/92eb83e7074b2760e6001d40b17eecffb874b23f036abd5b6d2328f28f49bac8.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4PhwACgkQzcvSTdHQ -4AE/Ggf+JafmzsI2bEJBTA9sLk4kVQCmUFfvnlsvExnZ622kp6om4N8Kfo5Ofjvr -x+wmAbPxZzJIqyAm46IQcMU5X7I3zgeDIMYvBJ5eQubXne4NN5ZLQgk9hkCWc3ly -gZP0bhoL68SasBIlufNvyT0zKBFO3+8LP0SiuVKRFu9gisfTjSEDVU5K5ofminZB -fk8YLaVXVkKSMCo0MTyGrsAzPiBiBwvIbNQCcFIjXk1DjoQE/2XErN+JGUTa2vbO -zx2JrPCXFERKoZ9DcSHyOqRz0HS+OhXVOkxqJZuduff9k2svHp/TyqEIB8m12E3B -jeD3WCzFDmvhZICTy73EJlakdfTLAQ== -=/TIs +iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4RBYACgkQzcvSTdHQ +4AFLMwgAg7UzICQj5qPvbTaWb+rLLmIYWUDIb2yn9Ci251noBs3PtJP/66x05n3t +OhwjFhJhKu5B7qgT+i/TxPZAlUkwwXtokmByEF0ZFO69yMDLyr4boWOa0NeL8lUb +6vjB9+seiG9FFZ2aF+Xgm8qXT0mpxVHAguWpZUiYSoCXznuQVUpjuMFtnAz0S0y9 +7CTisrfZvdIyYsuObBOcOO2+l2uHv6Yd1AGbwf6nikYP5zF6sWcNr4IMuFkIAKRP +AtOUbccxBTtXo3Ww3QFe82KbnQea4LCat9MFwCGJ4K8BcHotMppYB1JHy7t75gl/ +B6CoWDqjLNikWYtoymBCUFAOdEovkg== +=gm4q -----END PGP SIGNATURE----- diff --git a/c38ce3a51eeaa98f5524cf9902e84ee8ba5d179d6fbe18c49f9707554c896b6e.asc b/c38ce3a51eeaa98f5524cf9902e84ee8ba5d179d6fbe18c49f9707554c896b6e.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4PhwACgkQzcvSTdHQ -4AEz7wf/Vyp/u7rYTCJtpVWjYQ5n970LJpAQmpwnBHkg6J9dJ3lkXO1kCTHx1vKS -iJWrqL8Cb6tGi4OSyzI5ZK7OvSvv62TR+8e716sCFFdC3emilMeHKcNc9wWL9Hxq -dPUSnU+QT4POzr5SypuLqwxTPqv58nHZ9xfZ3s0Hrowl0pckXpTWhEkAQXRZ9ecX -i3qJB5bTmcar2NCM1YKDHVGsBOaYfH/cSzU6/DLi/FKKhGFpO7k3gA9klR3X0wVp -+zDONVToJf/mRKzitH85tRslebTLQYHWH/sJacTKZQTb7ZrkY7s/Df1ApoLIspX5 -b19MpqzILE9B6pEunYN9zpX+a9qRLA== -=tKN5 +iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4RBYACgkQzcvSTdHQ +4AGNKAf+MW45UwKWtcLlsRbm6Vdx9HpUB0ys7GLDnJQOZrmCxYkKF2BXoEfyiw32 +v6zQqdzPbE+GGdF5Cd5biEWvcJ+NzxLzNrlesg90tZ4WPlCK/jrsUsk/2PCRnkXy +l7XCko+IzEwOmDVULn7RU+kwcihRMvaU1PUpBMmJd659Ejdvapp1C6i/onhyshWf +NhXEvDjvXbepPTJGn6mcOgFIWCT16seBej4hPzpa74f64ojcRX2khlOlCCsava1X +EL+ip6e6TqOg22Eqb29k51xx/ZQSttyIj1Jj1JvPF9QM+r5wLNhVuONY4qFSLwK8 ++VrqFVTM7Xq+gORpTV+ZHACImw/u6g== +=FqT6 -----END PGP SIGNATURE----- diff --git a/cd80d0ad6005957cb7d4d6f35f4db78435728bef1fc3a2c3476697c343f7683b b/cd80d0ad6005957cb7d4d6f35f4db78435728bef1fc3a2c3476697c343f7683b @@ -0,0 +1,246 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <link rel="stylesheet" type="text/css" href="./theme/css/style.css" /> + <link rel="stylesheet" type="text/css" href="./theme/css/syntax.css" /> + <title>man bytes gnu - A portable book metadata exercise</title> + <meta charset="utf-8" /> + <meta name="generator" content="Pelican" /> + +</head> + +<body id="index" class="home"> + + <header id="banner" class="body"> + <h1>> <a href="./">man bytes gnu</a><span id="cursor">_</span></h1> + </header> + <nav id="menu"> + <ul> + <li class="active"><a href="./category/archiving.html">Archiving</a></li> + <li><a href="./category/code.html">Code</a></li> + <li><a href="./category/hygiene.html">Hygiene</a></li> + <li><a href="./category/offlining.html">Offlining</a></li> + <li><a href="./pages/identities.html">Identities</a></li> + <li><a href="./pages/shares.html">Shares</a></li> + + <li><a href="/tags.html">tags</a></li> + </ul> + <hr/> + </nav><!-- /#menu --> + <header> + <h1 class="entry-title top-body-title"> + <a href="./portable-book-metadata.html" rel="bookmark" + title="Permalink to A portable book metadata exercise">A portable book metadata exercise</a></h1> + + + <div class="category meta"> + Posted +<time class="published" datetime="2022-10-01T12:40:00+02:00"> + Sat 01 October 2022 + </time> +in <a class="category" href="./category/archiving.html">archiving</a> + <a href="./tag/hash.html">hash</a> + <a href="./tag/kitab.html">kitab</a> + <a href="./tag/literature.html">literature</a> + <a href="./tag/metadata.html">metadata</a> + <a href="./tag/dublincore.html">dublincore</a> + <a href="./tag/libgen.html">libgen</a> + </div> + + <div class="neighbors meta"> + <ul> + <li> + Previous: <a href="./backup-rsync-duplicity.html"> + Combining duplicity and rsync + </a> + </li> + </ul> + </div> + <div class="meta"> +<hr/> + </div> +</header> + +<section id="content" class="body"> + <div class="entry-content"> + <p>One of the things I have been working on the last few weeks is a rust application I have dubbed <a class="reference external" href="https://git.defalsify.net/kitab">kitab</a> <a class="footnote-reference" href="#footnote-1" id="footnote-reference-1">[1]</a>.</p> +<p>In short, the application makes it easy to extract literary metadata to a separate file structure.</p> +<p>The metadata can in turn be applied as <em>extended attributes</em> recursively on a directory for files that match.</p> +<p>The way it's accomplished it simple: The file name of the metadata is the hex representation of the digest of the file. The same digest is used to match files to metadata when applying it back to the file.</p> +<p>There are two advantages to this:</p> +<ol class="arabic simple"> +<li>The digest of the media file need not be affected by the metadata, i.e. by embedding metadata in the file itself.</li> +<li>You do not need to use the file name to keep record of what a file is.</li> +</ol> +<div class="section" id="yarr-ye-matey-data"> +<h2>Yarr, ye matey-data</h2> +<p>Let's demonstrate with an example.</p> +<p>The fabulous <a class="reference external" href="https://libgen.rs">Library Genesis</a> project has made available an endpoint to retrieve <tt class="docutils literal">bibtex</tt> entries based on the <tt class="docutils literal">md5</tt> hash of the book media file.</p> +<p>A version of the <a class="reference external" href="https://libgen.rs/book/index.php?md5=BCD99F1AB4155F2A2A362E5B7938A852">Bitcoin White Paper</a>, under the <code>md5</code> hash <code>bcd99f1ab4155f2a2a362e5b7938a852</code>, can be found there.</p> +<p>If you download this file using a synchronous download link, the browser will provide you with a filename to go with the download.</p> +<p>However, if you use the torrent alternative, the filename will be the <tt class="docutils literal">md5</tt> hash itself. If you are torrenting a bunch of those files, it quickly becomes a nuisance to distinguish them.</p> +<p>And, of course: In either case there is no guarantee the any metadata comes with the file.</p> +<div class="section" id="inside-the-book"> +<h3>Inside the book</h3> +<p>Kitab (v0.0.2) is able to read metadata from both a bibtex source and xattr entries on a file, as well as its native <a class="reference external" href="https://www.w3.org/TR/turtle/">rdf-turtle</a> format.</p> +<p>In kitab's data store, every media file entity in rdf-turtle is keyed with a <a class="reference external" href="https://www.rfc-editor.org/info/rfc8141">URN</a> specifying a digest for the file.</p> +<p>To see exactly what that looks like, let's download and import the bibtex metadata for the paper <a class="footnote-reference" href="#footnote-2" id="footnote-reference-2">[2]</a>:</p> +<pre class="code bash literal-block"> +<span class="nv">bibtex_file</span><span class="o">=</span><span class="sb">`</span>mktemp<span class="sb">`</span> +<span class="nv">kitab_dir</span><span class="o">=</span><span class="sb">`</span>mktemp -d<span class="sb">`</span> +curl -s -X GET https://libgen.rs/book/bibtex.php?md5<span class="o">=</span>BCD99F1AB4155F2A2A362E5B7938A852 -o <span class="nv">$bibtex_file</span> +kitab --store <span class="nv">$kitab_dir</span> import --digest md5:BCD99F1AB4155F2A2A362E5B7938A852 <span class="nv">$bibtex_file</span> +cat <span class="nv">$kitab_dir</span>/* +</pre> +<p>The output of the above should be:</p> +<pre class="code turtle literal-block"> +<span class="nv"><URN:md5:bcd99f1ab4155f2a2a362e5b7938a852></span> <span class="nv"><https://purl.org/dc/terms/title></span> <span class="s">"Bitcoin: A Peer-to-Peer Electronic Cash System"</span> <span class="p">;</span> +<span class="nv"><https://purl.org/dc/terms/creator></span> <span class="s">"Satoshi Nakamoto"</span> <span class="p">;</span> +<span class="nv"><https://purl.org/dc/terms/type></span> <span class="s">"book"</span> <span class="p">.</span> +</pre> +<p>Now let's say the media file itself has been downloaded to <tt class="docutils literal"><span class="pre">~/.local/share/transmission</span></tt>. We can apply this metadata as extended attributes.</p> +<p>This time we turn on logging to see what's going on:</p> +<pre class="code console literal-block"> +<span class="gp">$ </span><span class="nv">RUST_LOG</span><span class="o">=</span>info kitab --store <span class="nv">$kitab_dir</span> apply --digest md5 ~/.local/share/transmission +<span class="go">[2022-10-01T11:14:59Z INFO kitab] have index directory "/tmp/tmp.r0jBm6q4hW" +[2022-10-01T11:14:59Z INFO kitab] using digest type md5 +[2022-10-01T11:14:59Z INFO kitab] apply from path "/home/lash/.local/share/transmission/" +[2022-10-01T11:14:59Z INFO kitab] apply DirEntry("/home/lash/.local/share/transmission/bcd99f1ab4155f2a2a362e5b7938a852") -> title "Bitcoin: A Peer-to-Peer Electronic Cash System" author "Satoshi Nakamoto" digest md5:bcd99f1ab4155f2a2a362e5b7938a852 + +</span><span class="gp">$ </span>find ~/.local/share/transmission -type f -regextype sed -regex <span class="s2">".*/[a-f0-9]\{32\}</span>$<span class="s2">"</span> -exec getfattr -d <span class="o">{}</span> <span class="se">\;</span> +<span class="gp"># </span>file: .local/share/transmission/bcd99f1ab4155f2a2a362e5b7938a852 +<span class="go">user.dcterms:creator="Satoshi Nakamoto" +user.dcterms:title="Bitcoin: A Peer-to-Peer Electronic Cash System" +user.dcterms:type="book"</span> +</pre> +</div> +<div class="section" id="let-the-right-one-in"> +<h3>Let the right one in</h3> +<p>Conversely, the metadata can be re-imported directly from the extended attributes. And this time, let's store it both under the <tt class="docutils literal">md5</tt> and the <tt class="docutils literal">sha512</tt> hash:</p> +<pre class="code bash literal-block"> +$ <span class="nv">kitab_dir_new</span><span class="o">=</span><span class="sb">`</span>mktemp -d<span class="sb">`</span> +$ kitab --store <span class="nv">$kitab_dir_new</span> import --digest md5 --digest sha512 .local/share/transmission/bcd99f1ab4155f2a2a362e5b7938a852 +$ find <span class="nv">$kitab_dir_new</span> -type f -exec cat <span class="o">{}</span> <span class="se">\;</span> +/tmp/tmp.B6j41YMmEM/493f2a720d63156d77187bcd5f0715e4e765a38d616ef47f24e0df817ee6b4f601d47a06ffae10ef1f6ba60bb5d2e99a26318f035f9cd56e30bfe7bcdf64a792 +<URN:sha512:493f2a720d63156d77187bcd5f0715e4e765a38d616ef47f24e0df817ee6b4f601d47a06ffae10ef1f6ba60bb5d2e99a26318f035f9cd56e30bfe7bcdf64a792> <https://purl.org/dc/terms/title> <span class="s2">"Bitcoin: A Peer-to-Peer Electronic Cash System"</span> <span class="p">;</span> + <https://purl.org/dc/terms/creator> <span class="s2">"Satoshi Nakamoto"</span> <span class="p">;</span> + <https://purl.org/dc/terms/type> <span class="s2">"book"</span> <span class="p">;</span> + <https://purl.org/dc/terms/MediaType> <span class="s2">"application/epub+zip"</span> . +/tmp/tmp.B6j41YMmEM/bcd99f1ab4155f2a2a362e5b7938a852 +<URN:md5:bcd99f1ab4155f2a2a362e5b7938a852> <https://purl.org/dc/terms/title> <span class="s2">"Bitcoin: A Peer-to-Peer Electronic Cash System"</span> <span class="p">;</span> + <https://purl.org/dc/terms/creator> <span class="s2">"Satoshi Nakamoto"</span> <span class="p">;</span> + <https://purl.org/dc/terms/type> <span class="s2">"book"</span> <span class="p">;</span> + <https://purl.org/dc/terms/MediaType> <span class="s2">"application/epub+zip"</span> . +</pre> +</div> +</div> +<div class="section" id="level-up"> +<h2>Level up</h2> +<p>Finally, a bash script <a class="footnote-reference" href="#footnote-3" id="footnote-reference-3">[3]</a> example that lets you retrieve and apply metadata for a batch of files found in the directory given as the <em>first positional arg</em>.</p> +<p>This script even renames the files according to the metadata applied.</p> +<pre class="code bash literal-block"> +<span class="ln"> 0 </span><span class="c1"># NOTE! this will only work if your fs supports xattr. +</span><span class="ln"> 1 </span><span class="c1"># That's why we cannot use tmpfs (mktemp) here; tmpfs does not support xattr. +</span><span class="ln"> 2 </span><span class="c1"></span> +<span class="ln"> 3 </span><span class="c1"># directory to copy media files to +</span><span class="ln"> 4 </span><span class="c1"></span><span class="nv">outdir</span><span class="o">=</span>./<span class="k">$(</span>uuidgen<span class="k">)</span> +<span class="ln"> 5 </span>mkdir -vp <span class="nv">$outdir</span> +<span class="ln"> 6 </span> +<span class="ln"> 7 </span><span class="c1"># Input dir is the first positional arg. +</span><span class="ln"> 8 </span><span class="c1"></span><span class="nv">indir</span><span class="o">=</span><span class="nv">$1</span> +<span class="ln"> 9 </span> +<span class="ln">10 </span><span class="nv">IFS</span><span class="o">=</span><span class="s1">$'\n'</span> +<span class="ln">11 </span> +<span class="ln">12 </span><span class="c1"># Retrieve metadata for each file and import it into the kitab store. +</span><span class="ln">13 </span><span class="c1"># Also copy the media file to the separate output directory. +</span><span class="ln">14 </span><span class="c1"></span><span class="k">for</span> f <span class="k">in</span> <span class="k">$(</span>find <span class="nv">$indir</span> -type f<span class="k">)</span><span class="p">;</span> <span class="k">do</span> +<span class="ln">15 </span> <span class="nv">sum</span><span class="o">=</span><span class="k">$(</span>md5sum <span class="nv">$f</span> <span class="p">|</span> awk <span class="s1">'{print $1;}'</span><span class="k">)</span> +<span class="ln">16 </span> <span class="nb">echo</span> <span class="s2">"downloading metadata for </span><span class="nv">$indir</span><span class="s2">/</span><span class="nv">$f</span><span class="s2">"</span> +<span class="ln">17 </span> <span class="nv">srct</span><span class="o">=</span><span class="k">$(</span>mktemp<span class="k">)</span> +<span class="ln">18 </span> curl -s -X GET https://libgen.rs/book/bibtex.php?md5<span class="o">=</span><span class="nv">$sum</span> -o <span class="nv">$srct</span> +<span class="ln">19 </span> <span class="nv">dstt</span><span class="o">=</span><span class="k">$(</span>mktemp<span class="k">)</span> +<span class="ln">20 </span> xmllint --html --xpath <span class="s1">'string(/html/body/textarea[@id="bibtext"])'</span> <span class="nv">$srct</span> > <span class="nv">$dstt</span> +<span class="ln">21 </span> kitab import --digest md5:<span class="nv">$sum</span> <span class="nv">$dstt</span> +<span class="ln">22 </span> cp <span class="nv">$f</span> <span class="nv">$outdir</span>/ +<span class="ln">23 </span><span class="k">done</span> +<span class="ln">24 </span> +<span class="ln">25 </span><span class="c1"># Apply metadata imported from bibtex as xattr for the media files. +</span><span class="ln">26 </span><span class="c1"></span><span class="nv">RUST_LOG</span><span class="o">=</span>info kitab apply --digest md5 <span class="nv">$outdir</span>/ +<span class="ln">27 </span> +<span class="ln">28 </span><span class="c1"># Rename the files according to the metadata title and media type. +</span><span class="ln">29 </span><span class="c1"></span><span class="k">for</span> f <span class="k">in</span> <span class="k">$(</span>ls <span class="nv">$outdir</span><span class="k">)</span><span class="p">;</span> <span class="k">do</span> +<span class="ln">30 </span> <span class="nv">title</span><span class="o">=</span><span class="k">$(</span>getfattr --only-values -n user.dcterms:title <span class="nv">$outdir</span>/<span class="nv">$f</span><span class="k">)</span> +<span class="ln">31 </span> +<span class="ln">32 </span> <span class="nv">f_typ</span><span class="o">=</span><span class="k">$(</span>file -b --mime-type <span class="nv">$outdir</span>/<span class="nv">$f</span><span class="k">)</span> +<span class="ln">33 </span> <span class="nv">f_ext</span><span class="o">=</span><span class="s2">""</span> +<span class="ln">34 </span> <span class="k">case</span> <span class="s2">"</span><span class="nv">$f_typ</span><span class="s2">"</span> <span class="k">in</span> +<span class="ln">35 </span> <span class="s2">"application/pdf"</span><span class="o">)</span> +<span class="ln">36 </span> <span class="nv">f_ext</span><span class="o">=</span><span class="s2">".pdf"</span> +<span class="ln">37 </span> <span class="p">;;</span> +<span class="ln">38 </span> <span class="s2">"application/epub+zip"</span><span class="o">)</span> +<span class="ln">39 </span> <span class="nv">f_ext</span><span class="o">=</span><span class="s2">".epub"</span> +<span class="ln">40 </span> <span class="p">;;</span> +<span class="ln">41 </span> <span class="s2">"application/x-mobipocket-ebook"</span><span class="o">)</span> +<span class="ln">42 </span> <span class="nv">f_ext</span><span class="o">=</span><span class="s2">".mobi"</span> +<span class="ln">43 </span> <span class="p">;;</span> +<span class="ln">44 </span> <span class="s2">"text/plain"</span><span class="o">)</span> +<span class="ln">45 </span> <span class="nv">f_ext</span><span class="o">=</span><span class="s2">".txt"</span> +<span class="ln">46 </span> <span class="p">;;</span> +<span class="ln">47 </span> <span class="s2">"text/html"</span><span class="o">)</span> +<span class="ln">48 </span> <span class="nv">f_ext</span><span class="o">=</span><span class="s2">".html"</span> +<span class="ln">49 </span> <span class="p">;;</span> +<span class="ln">50 </span> *<span class="o">)</span> +<span class="ln">51 </span> ><span class="p">&</span><span class="m">2</span> <span class="nb">echo</span> unhandled mime <span class="nb">type</span> <span class="nv">$f_typ</span> +<span class="ln">52 </span> <span class="nb">exit</span> <span class="m">1</span> +<span class="ln">53 </span> <span class="k">esac</span> +<span class="ln">54 </span> mv -v <span class="nv">$outdir</span>/<span class="nv">$f</span> <span class="nv">$outdir</span>/<span class="si">${</span><span class="nv">title</span><span class="si">}${</span><span class="nv">f_ext</span><span class="si">}</span> +<span class="ln">55 </span><span class="k">done</span> +</pre> +<p>This last example will result in:</p> +<ul class="simple"> +<li>A media file named <tt class="docutils literal">$outdir/Bitcoin: A <span class="pre">Peer-to-Peer</span> Electronic Cash System.epub</tt></li> +<li>... with metadata applied as extended attributes</li> +<li>An rdf-turtle metadata entry in <tt class="docutils literal"><span class="pre">~/.local/share/kitab/idx/bcd99f1ab4155f2a2a362e5b7938a852</span></tt></li> +</ul> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="footnote-1" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#footnote-reference-1">[1]</a></td><td>The relevant documentation for <tt class="docutils literal">kitab</tt> at the time of writing is <a class="reference external" href="https://defalsify.org/doc/crates/kitab/0.0.2/kitab/">here</a>. To build kitab, simply <em>clone</em> the repository and build with <code>cargo build --all-features</code>.</td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="footnote-2" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#footnote-reference-2">[2]</a></td><td>The <code>kitab</code> command in the script assumes you have built the <em>kitab binary</em> and made it available in your path.</td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="footnote-3" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#footnote-reference-3">[3]</a></td><td>the script uses <code>xmllint</code> which on archlinux is provided by the <tt class="docutils literal">libxml2</tt> package.</td></tr> +</tbody> +</table> +</blockquote> +</div> + + </div><!-- /.entry-content --> + <footer class="meta"> + </footer> + +</section> + <footer id="contentinfo" class="body"> + <hr/> + <address id="about" class="vcard body"> + Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> + </address><!-- /#about --> + </footer><!-- /#contentinfo --> +</body> +</html> +\ No newline at end of file diff --git a/cd80d0ad6005957cb7d4d6f35f4db78435728bef1fc3a2c3476697c343f7683b.asc b/cd80d0ad6005957cb7d4d6f35f4db78435728bef1fc3a2c3476697c343f7683b.asc @@ -0,0 +1,11 @@ +-----BEGIN PGP SIGNATURE----- + +iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4RBEACgkQzcvSTdHQ +4AF87gf/fIP9XeRpslIvWfTS2Fy4jmdWsnW4uXb3A5Q1R5ONosj2sklnV3RaHbF4 +HhV0qzk9p3HZWGbtHDyj/aA/zJphqhkAhbkTvzAoWBapVqFuLQhb8aAKwU+vbqsA +uNc/NtunLzlUlR8HHGQPDsE/PzmYKyXVAoaiIc6EoDSKVGILEjJMRrApIY5+0pc3 +eRg/WvhaouJfhkK7rQDjFMEWUSz2xsFZM7GOxRA5lg3FPIVV1B+z+i6J3V2HsmM3 +S+5qMDManFCFHMsfRLVgJ012S5XzMbZE4ZshhpahA4FlD77JAsv3d0HC6cKUZ4Tq +HU2NlVCwXNonx6oSIP1HhntVjOLiRw== +=ha7U +-----END PGP SIGNATURE----- diff --git a/dab643d2a647cfd037878b4fd6bd447457b8b568c3992f411350fc88977d4d88.asc b/dab643d2a647cfd037878b4fd6bd447457b8b568c3992f411350fc88977d4d88.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4PhwACgkQzcvSTdHQ -4AG1qQf9H7slJlzPtU5F8nrl42aWrJvoUnSuU/ZVhZXglUCBcoZWYTJduUP9n+jG -+9yKkVCYqDFqTitGuupRQ1Ih9m6a2Uxw6lhd/K5g96VTvv9EL/d83q+VAQBjawpT -1Eb7q8MhZfEAaSKHBmbncd3Qno1T6jaIfmX2TnS/XstzawVuc4Dez+hCnoHj3aBS -rjP/cI3ON7npmtkDHc873gnytqSjnXcUW2rkP/7qLglC+V30LiqLXIXjbSfNsNu7 -Dld3txfy4Jbh+8lJf7Nt4X9Zsig8vjsfzDmblbfJpWOpQXck9DuoLt+K6UX7Dree -ZXcwi/EJE2UJcUv3tGxnCIgvrgn4Kw== -=OgeR +iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4RBYACgkQzcvSTdHQ +4AFUmAgAp0g2dUiky9TZIr6Ll/w2jG+6s3Ysf+3iw0QhmPjm+YC2uyTHIx/aQXne +T0MnwxvXnSyg5uQSrDP8mfp0EXxEAF+Apyk6PSeTYL3Yvyq2RPG9WP+74X9Ex9BG +L66prbz9BoFxMHP6pJgC6ThS1hvUkbHPjpP/p4qZNrAPhiGQd6a0AFiGUHqURs9E +cnmuChyXNWmpPsh1ba91aLP6uYEHN9RPl49hdz98CbBS1RuDnTqlHeHasFfK8iXN +7qNZPmeM4ZlGdB6vzajUYNJxpEBm/akNkg1J5EqFL5BHtKWaRbENaonXDMgZkF58 +eSIhGDsdSRSsbuUu89quUI6I8flG/w== +=vmYR -----END PGP SIGNATURE----- diff --git a/f72f88a69eb494ae7f8cbf441683bf156bebf44039b25e723d03d5ed44e0c3e4.asc b/f72f88a69eb494ae7f8cbf441683bf156bebf44039b25e723d03d5ed44e0c3e4.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4PhwACgkQzcvSTdHQ -4AGKDgf+JgAjRBL6z3URsL6Bq9dXT6+bll98S4a8PuS2K8/ne/L1GdSv3bExwFri -HXaQWYxiCXssHScdhOIU+2WMA2e/if2U45gYA9/oVNFR2Kg/S4YsqK25Pl5Nk3D7 -IbLejnzLYbJsCP5j/H7X8YFCNpj9AW5+bnEZT21NLExUaI+ZN9jjsw1M5cOSQswb -u20dc3nonNHRp/hx5AtDpnRgSnup030sr45MmZQn9SM2ws/4r1JIAWf+ZI8IctTx -hOHaOeRB+i9vZu1ezYdXVsERHI4xVRkiTVcJga+7csauMcCGMBn+bWcRaSl7wvZw -Qngdz7s8kNCwertylkDHD7r4/JxjiA== -=OaEZ +iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmM4RBYACgkQzcvSTdHQ +4AHmUwf/RO4BbdSG61qBeGtopARvlMIpDpzYFV5ZcPjKnseCK+VDpL8F5o2VvWqw +b48gUenSyTySAqM2gAlYdfeSuf9eiIII97GmisK/mCIBsWejePrzRUSJNq3GpuUq +C9KKvP6zn+5eZcRHb3c8wK3M6bW1llIR1/jU9pQelCRuFU19KliorbkrjoE746Qu +a/GMRKVEDax9cHoJlTwhQ9f2JllSNo6GueKkcMOF9U6bRj3855PJ80JUFUxO1ntD +akFq6XBsFZhB9qMQM1yDYBd2PrDWKJ0wuofbCZfTB9GRWf6x0cdwXSOl/9B1sXA4 +MUs/UCiOZQU883CHRWvbd0OU4f2d6Q== +=ugtt -----END PGP SIGNATURE----- diff --git a/portable-book-metadata.html b/portable-book-metadata.html @@ -63,7 +63,7 @@ in <a class="category" href="./category/archiving.html">archiving</a> <section id="content" class="body"> <div class="entry-content"> - <p>One of the things I have been working on the last few weeks is a rust application I have dubbed <a class="reference external" href="https://git.defalsify.net/kitab">kitab</a>.</p> + <p>One of the things I have been working on the last few weeks is a rust application I have dubbed <a class="reference external" href="https://git.defalsify.net/kitab">kitab</a> <a class="footnote-reference" href="#footnote-1" id="footnote-reference-1">[1]</a>.</p> <p>In short, the application makes it easy to extract literary metadata to a separate file structure.</p> <p>The metadata can in turn be applied as <em>extended attributes</em> recursively on a directory for files that match.</p> <p>The way it's accomplished it simple: The file name of the metadata is the hex representation of the digest of the file. The same digest is used to match files to metadata when applying it back to the file.</p> @@ -84,7 +84,7 @@ in <a class="category" href="./category/archiving.html">archiving</a> <h3>Inside the book</h3> <p>Kitab (v0.0.2) is able to read metadata from both a bibtex source and xattr entries on a file, as well as its native <a class="reference external" href="https://www.w3.org/TR/turtle/">rdf-turtle</a> format.</p> <p>In kitab's data store, every media file entity in rdf-turtle is keyed with a <a class="reference external" href="https://www.rfc-editor.org/info/rfc8141">URN</a> specifying a digest for the file.</p> -<p>To see exactly what that looks like, let's download and import the bibtex metadata for the paper <a class="footnote-reference" href="#footnote-1" id="footnote-reference-1">[1]</a>:</p> +<p>To see exactly what that looks like, let's download and import the bibtex metadata for the paper <a class="footnote-reference" href="#footnote-2" id="footnote-reference-2">[2]</a>:</p> <pre class="code bash literal-block"> <span class="nv">bibtex_file</span><span class="o">=</span><span class="sb">`</span>mktemp<span class="sb">`</span> <span class="nv">kitab_dir</span><span class="o">=</span><span class="sb">`</span>mktemp -d<span class="sb">`</span> @@ -136,7 +136,7 @@ $ find <span class="nv">$kitab_dir_new</span> -type f -exec cat <span class="o"> </div> <div class="section" id="level-up"> <h2>Level up</h2> -<p>Finally, a bash script <a class="footnote-reference" href="#footnote-2" id="footnote-reference-2">[2]</a> example that lets you retrieve and apply metadata for a batch of files found in the directory given as the <em>first positional arg</em>.</p> +<p>Finally, a bash script <a class="footnote-reference" href="#footnote-3" id="footnote-reference-3">[3]</a> example that lets you retrieve and apply metadata for a batch of files found in the directory given as the <em>first positional arg</em>.</p> <p>This script even renames the files according to the metadata applied.</p> <pre class="code bash literal-block"> <span class="ln"> 0 </span><span class="c1"># NOTE! this will only work if your fs supports xattr. @@ -207,7 +207,7 @@ $ find <span class="nv">$kitab_dir_new</span> -type f -exec cat <span class="o"> <table class="docutils footnote" frame="void" id="footnote-1" rules="none"> <colgroup><col class="label" /><col /></colgroup> <tbody valign="top"> -<tr><td class="label"><a class="fn-backref" href="#footnote-reference-1">[1]</a></td><td>The <code>kitab</code> command in the script assumes you have built the <em>kitab binary</em> and made it available in your path.</td></tr> +<tr><td class="label"><a class="fn-backref" href="#footnote-reference-1">[1]</a></td><td>The relevant documentation for <tt class="docutils literal">kitab</tt> at the time of writing is <a class="reference external" href="https://defalsify.org/doc/crates/kitab/0.0.2/kitab/">here</a>. To build kitab, simply <em>clone</em> the repository and build with <code>cargo build --all-features</code>.</td></tr> </tbody> </table> </blockquote> @@ -216,7 +216,16 @@ $ find <span class="nv">$kitab_dir_new</span> -type f -exec cat <span class="o"> <table class="docutils footnote" frame="void" id="footnote-2" rules="none"> <colgroup><col class="label" /><col /></colgroup> <tbody valign="top"> -<tr><td class="label"><a class="fn-backref" href="#footnote-reference-2">[2]</a></td><td>the script uses <code>xmllint</code> which on archlinux is provided by the <tt class="docutils literal">libxml2</tt> package.</td></tr> +<tr><td class="label"><a class="fn-backref" href="#footnote-reference-2">[2]</a></td><td>The <code>kitab</code> command in the script assumes you have built the <em>kitab binary</em> and made it available in your path.</td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="footnote-3" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#footnote-reference-3">[3]</a></td><td>the script uses <code>xmllint</code> which on archlinux is provided by the <tt class="docutils literal">libxml2</tt> package.</td></tr> </tbody> </table> </blockquote> diff --git a/portable-book-metadata.sha256 b/portable-book-metadata.sha256 @@ -1 +1 @@ -37f77922ca667d5ced9d58cdc642bacb89fa428365585bad4aa09a6e9ab67d82 portable-book-metadata.html -\ No newline at end of file +cd80d0ad6005957cb7d4d6f35f4db78435728bef1fc3a2c3476697c343f7683b portable-book-metadata.html +\ No newline at end of file