commit b69f947acca539f702a22851bed29b3108162867 parent 27673aaa5c3eb26703f3dc817661695000186c17 Author: lash <dev@holbrook.no> Date: Sun, 16 Jan 2022 18:43:18 +0000 Remote drafts, add missing proofs and digests Diffstat:
| A | 132b445a5c24f134b82b8cb95e7112fd053ca2c15531afa694105a5d1a1d872a | | | 272 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | 132b445a5c24f134b82b8cb95e7112fd053ca2c15531afa694105a5d1a1d872a.asc | | | 11 | +++++++++++ |
| A | 1c19dbdeef6f5dea9dfc688b2aff4784fe2c5dbb2a8af928974d41c536f6afdd | | | 316 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | 1c19dbdeef6f5dea9dfc688b2aff4784fe2c5dbb2a8af928974d41c536f6afdd.asc | | | 11 | +++++++++++ |
| A | 2868ebaf23ffa05cd78bead77dbae8a6c0e596d67979ef45c9fd8e54db2cfbc9 | | | 332 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | 2868ebaf23ffa05cd78bead77dbae8a6c0e596d67979ef45c9fd8e54db2cfbc9.asc | | | 11 | +++++++++++ |
| A | 2918deb778c2f4961cc75fcf66f21c6bbd1c6a848e0a5aebb2e3a9afdac25ce6 | | | 364 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | 2918deb778c2f4961cc75fcf66f21c6bbd1c6a848e0a5aebb2e3a9afdac25ce6.asc | | | 11 | +++++++++++ |
| A | 5f38d7d3ab58689e73a237c1e10bf588e66a50457b4c4fea08fe423bc8dbd460 | | | 292 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | 5f38d7d3ab58689e73a237c1e10bf588e66a50457b4c4fea08fe423bc8dbd460.asc | | | 11 | +++++++++++ |
| A | 6cd7d22162d74eba04b00bdc004e18907c32ba245e0bf210ccb26e431789b18b | | | 329 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | 6cd7d22162d74eba04b00bdc004e18907c32ba245e0bf210ccb26e431789b18b.asc | | | 11 | +++++++++++ |
| A | a5006a0269c2c925630e27977d97e59941f38432eae7b684a8342b92d655ec31 | | | 225 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | a5006a0269c2c925630e27977d97e59941f38432eae7b684a8342b92d655ec31.asc | | | 11 | +++++++++++ |
| A | af20d83900b9bea7a7258e13dccec33be38aa849bf2387fcdd4ab6ab1e6069f1 | | | 250 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | af20d83900b9bea7a7258e13dccec33be38aa849bf2387fcdd4ab6ab1e6069f1.asc | | | 11 | +++++++++++ |
| D | drafts/celery-document-graph.html | | | 88 | ------------------------------------------------------------------------------- |
| D | drafts/celery-document-graph.sha256 | | | 2 | -- |
| D | drafts/clortho.html | | | 93 | ------------------------------------------------------------------------------- |
| D | drafts/clortho.sha256 | | | 2 | -- |
| D | drafts/docker-offline-4-debian.html | | | 83 | ------------------------------------------------------------------------------- |
| D | drafts/docker-offline-4-debian.sha256 | | | 2 | -- |
| D | drafts/docker-offline-5-npm-bundle.html | | | 132 | ------------------------------------------------------------------------------- |
| D | drafts/docker-offline-5-npm-bundle.sha256 | | | 2 | -- |
| D | drafts/docker-vpn.html | | | 103 | ------------------------------------------------------------------------------- |
| D | drafts/docker-vpn.sha256 | | | 2 | -- |
| D | drafts/ipfs-api-get.html | | | 102 | ------------------------------------------------------------------------------- |
| D | drafts/ipfs-api-get.sha256 | | | 2 | -- |
| D | drafts/keccak-benchmarks.html | | | 314 | ------------------------------------------------------------------------------- |
| D | drafts/keccak-benchmarks.sha256 | | | 2 | -- |
| D | drafts/python-pipe-args.html | | | 92 | ------------------------------------------------------------------------------- |
| D | drafts/python-pipe-args.sha256 | | | 2 | -- |
| D | drafts/python-virtualenv-interpreter.html | | | 77 | ----------------------------------------------------------------------------- |
| D | drafts/python-virtualenv-interpreter.sha256 | | | 2 | -- |
| D | drafts/qemu-host-fw.html | | | 78 | ------------------------------------------------------------------------------ |
| D | drafts/qemu-host-fw.sha256 | | | 2 | -- |
| D | drafts/qemu-raw-partitions.html | | | 103 | ------------------------------------------------------------------------------- |
| D | drafts/qemu-raw-partitions.sha256 | | | 2 | -- |
| D | drafts/swarm-qemu.html | | | 104 | ------------------------------------------------------------------------------- |
| D | drafts/swarm-qemu.sha256 | | | 2 | -- |
| A | e37125a796b63784152584d8a08a4399c56f488978c5194cc33cc4af10cdbce7 | | | 217 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | e37125a796b63784152584d8a08a4399c56f488978c5194cc33cc4af10cdbce7.asc | | | 11 | +++++++++++ |
| A | e728dd8f874e06549c187d3bf96943585aa8c3c4b51666511f400d7191c0f8ac | | | 177 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | e728dd8f874e06549c187d3bf96943585aa8c3c4b51666511f400d7191c0f8ac.asc | | | 11 | +++++++++++ |
44 files changed, 2884 insertions(+), 1393 deletions(-)
diff --git a/132b445a5c24f134b82b8cb95e7112fd053ca2c15531afa694105a5d1a1d872a b/132b445a5c24f134b82b8cb95e7112fd053ca2c15531afa694105a5d1a1d872a @@ -0,0 +1,271 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <link rel="stylesheet" type="text/css" href="./theme/css/style.css" /> + <link rel="stylesheet" type="text/css" href="./theme/css/syntax.css" /> + <title>man bytes gnu - Proving what you link to</title> + <meta charset="utf-8" /> + <meta name="generator" content="Pelican" /> + +</head> + +<body id="index" class="home"> + + <header id="banner" class="body"> + <h1>> <a href="./">man bytes gnu</a><span id="cursor">_</span></h1> + </header> + <nav id="menu"> + <ul> + <li class="active"><a href="./category/archiving.html">Archiving</a></li> + <li><a href="./category/code.html">Code</a></li> + <li><a href="./category/hygiene.html">Hygiene</a></li> + <li><a href="./category/offlining.html">Offlining</a></li> + <li><a href="./pages/identities.html">Identities</a></li> + <li><a href="./pages/shares.html">Shares</a></li> + + <li><a href="/tags.html">tags</a></li> + </ul> + <hr/> + </nav><!-- /#menu --> + <header> + <h1 class="entry-title top-body-title"> + <a href="./web-snapshot.html" rel="bookmark" + title="Permalink to Proving what you link to">Proving what you link to</a></h1> + + + <div class="category meta"> + Posted +<time class="published" datetime="2021-05-03T14:22:00+02:00"> + Mon 03 May 2021 + </time> +in <a class="category" href="./category/archiving.html">archiving</a> + <a href="./tag/web.html">web</a> + <a href="./tag/hash.html">hash</a> + <a href="./tag/chromium.html">chromium</a> + </div> + + <div class="neighbors meta"> + <ul> + <li> + Previous: <a href="./docker-offline-2-python.html"> + Local python repository + </a> + </li> + | + + <li> + Next: <a href="./wasm-c-bare.html"> + wasm and C - The bare necessities + </a> + </li> + </ul> + </div> + <div class="meta"> +<hr/> + </div> +</header> + +<section id="content" class="body"> + <div class="entry-content"> + <p>When we send a link to someone, we are trusting that whoever is behind that link will serve that someone the content we actually saw. Give or take an ad or two.</p> +<p>Also, when we bookmark the link for later retrieval. We are trusting that the entity will be serving that content at any time in the future that you may want it.</p> +<p>This may not be a huge problem if the page is merely a list of <a class="reference external" href="https://dead-baby-joke.com/">dead baby jokes</a>. They are objectively funny, of course. But you can also get along without them.</p> +<p>But what of the case of formal, scientific texts we may depend on, and that use citations from the web as part of their source material? Usually, they refer to a source by <em>link</em> and <em>date of retrieval</em>. This is not of much use unless the actual source and/or render they saw at that time also is available.</p> +<p>That may not always be the case.</p> +<div class="section" id="take-care-of-your-shelf"> +<h2>Take care of your shelf</h2> +<blockquote> +"No worries, the <a class="reference external" href="https://archive.org">Wayback Machine</a> has me covered."</blockquote> +<p>Yes. But no. The Wayback Machine is a (thus far) centralized entity that depend on a few idealists and donations to keep going. If they stop to keep going, they depend on passing the buck to someone else. If that someone is evil, they may take it and rewrite history to suit themselves. If that someone else cannot be found, it becomes a garbage collection blip on Bezos' <a class="reference external" href="https://gizmodo.com/i-tried-to-block-amazon-from-my-life-it-was-impossible-1830565336">infrastructure monopoly</a> dashboard. <a class="footnote-reference" href="#id6" id="id1">[1]</a></p> +<p>That aside, sources like Wayback Machine are like libraries. Libraries are, of course, essential. Not only because they serve as one of the pillars of democracy, providing free access to knowledge for everyone. They are also essential because it's simply not very practical for you to pre-emptively own all the books that you may at some point want to read. Let alone ask around in your neighborhood if they happen to have a copy (although a crowd-sourced library app sounds like a fun decentralization project to explore).</p> +<p>You may however want to keep a copy of the books you <em>depend</em> on, and the ones you <em>really like</em>. Just to make really sure you have it available when you want it. Then, if some New Public Management clowns get the chance to gut public infrastructure where you live, or someone starts a good old-fashioned fascist book burning, you have yourself covered.</p> +</div> +<div class="section" id="a-lack-of-friction"> +<h2>A lack of friction</h2> +<p>Yes, stuff may disappear on the web. Just as books may.</p> +<p>On the web that stuff can get <em>rewritten</em>. Books may be rewritten, too. <a class="footnote-reference" href="#id7" id="id2">[2]</a> The previous editions of the books will still exist as independent physical objects until they degrade, as long as something is not actively done to them. But so too with data on storage media. If it is not renewed or copied during that lifetime it may degrade until it becomes illegible. And of course, it may also simply be deleted. And without the smell of smoke at that.</p> +<p>That's the difference that seems to leap out when using this imagery. How <em>easy</em> it is to change and destroy stuff at scale on the web compared with the real world of books. And how inconspicuously it can happen, without <em>anyone</em> noticing. And for those who notice, it is very hard to prove what has changed, unless you have a copy. <a class="footnote-reference" href="#id10" id="id3">[5]</a></p> +<p>So what can we do? Copies and proofs are definitely keywords here. Fortunately, copying is what computers are all about. And making a cryptographical proof of what you see is easy enough these days, too. The tricky bit is to build <em>credibility</em> around that proof. But let's stick our head in the sand and start with the easy part and see where it takes us.</p> +</div> +<div class="section" id="look-no-head"> +<h2>Look, no head</h2> +<!-- .. WARNING:: + + Nerdy zone ahead --> +<p>A good start is to dump the document source to disk, then calculating and storing the sum of it.</p> +<p>In many cases this will be insufficient, though, as many sites populates the DOM through scripts, either in part or in full. As the use-case here is humans voting on what they see is what they get, human aids will be needed. In other words, we need to render the page to store what we actually see.</p> +<p>Printing to PDF from the browser is an option, but that is really difficult to automate. Fortunately, modern browser engines provide command line access to rendering. Since I mostly use <a class="reference external" href="https://brave.com/">Brave Browser</a> these days, we'll use <em>headless Chromium</em> here.</p> +<p>In addition to source, sum and rendering, we should also include a copy of the request headers for good measure.</p> +<p>Thus, we end up with something like this.</p> +<pre class="code bash literal-block"> +<span class="ch">#!/bin/bash +</span> +<span class="nv">f</span><span class="o">=</span><span class="si">${</span><span class="nv">WEBSHOT_OUTPUT_DIR</span><span class="k">:-</span><span class="p">/tmp</span><span class="si">}</span> +<span class="nv">url</span><span class="o">=</span><span class="nv">$1</span> +<span class="nv">title</span><span class="o">=</span><span class="nv">$2</span> +><span class="p">&</span><span class="m">2</span> <span class="nb">echo</span> using outdir <span class="nv">$f</span> + +<span class="nb">set</span> +e + +<span class="c1"># prepare +</span><span class="nv">d</span><span class="o">=</span><span class="sb">`</span><span class="nv">TZ</span><span class="o">=</span>UTC date +%Y%m%d%H%M<span class="sb">`</span> +<span class="nv">t</span><span class="o">=</span><span class="sb">`</span>mktemp -d<span class="sb">`</span> +<span class="nb">pushd</span> <span class="nv">$t</span> + +<span class="c1"># store raw outputs +</span><span class="nb">echo</span> <span class="nv">$1</span> > url.txt +curl -s -I <span class="nv">$1</span> > headers.txt +curl -s -X GET <span class="nv">$1</span> > contents.txt +<span class="nv">z</span><span class="o">=</span><span class="sb">`</span>sha256sum contents.txt<span class="sb">`</span> +<span class="nb">echo</span> <span class="nv">$z</span> > contents.txt.sha256 +<span class="nv">h</span><span class="o">=</span><span class="sb">`</span><span class="nb">echo</span> -n <span class="nv">$z</span> <span class="p">|</span> awk <span class="s1">'{ print $1; }'</span><span class="sb">`</span> + +<span class="k">if</span> <span class="o">[</span> -z <span class="s2">"</span><span class="nv">$title</span><span class="s2">"</span> <span class="o">]</span><span class="p">;</span> <span class="k">then</span> + <span class="nv">title</span><span class="o">=</span><span class="nv">$h</span> +<span class="k">fi</span> +><span class="p">&</span><span class="m">2</span> <span class="nb">echo</span> using title <span class="nv">$title</span> + +<span class="c1"># rendered snapshot +</span>chromium --headless --print-to-pdf <span class="nv">$url</span> +<span class="nv">n</span><span class="o">=</span><span class="si">${</span><span class="nv">d</span><span class="si">}</span>_<span class="si">${</span><span class="nv">h</span><span class="si">}</span> +mv output.pdf <span class="nv">$n</span>.pdf + +<span class="c1"># store result +</span>mkdir -p <span class="s2">"</span><span class="nv">$f</span><span class="s2">/</span><span class="nv">$title</span><span class="s2">"</span> +tar -zcvf <span class="s2">"</span><span class="nv">$f</span><span class="s2">/</span><span class="nv">$title</span><span class="s2">/</span><span class="nv">$n</span><span class="s2">.tar.gz"</span> * + +<span class="c1"># clean up +</span><span class="nb">popd</span> + +<span class="nb">set</span> -e +</pre> +</div> +<div class="section" id="what-does-this-mean"> +<h2>What does this mean</h2> +<p>Let's sum up what information we've managed to store with this operation.</p> +<ul class="simple"> +<li>We have a copy of the unrendered source. It may or may not include all the information we want to store.</li> +<li>We have a fingerprint of that source.</li> +<li>We have a copy of the headers we were served when retrieving the document. <a class="footnote-reference" href="#id8" id="id4">[3]</a></li> +<li>We have a image copy of what we actually saw when visiting the page.</li> +<li>We have a date and time for retrieval (file attributes).</li> +</ul> +<p>To link the headers together with the visual copy, we could sum the header file and image file aswell, put those sums together with the content sum in a deterministic order, and calculate the sum of those sums. E.g. <a class="footnote-reference" href="#id9" id="id5">[4]</a></p> +<div class="highlight"><pre><span></span>$ cp contents.txt.sha256 sums.txt +$ sha256sum headers.txt <span class="p">|</span> awk <span class="s1">'{ print $1; }'</span> >> sums.txt +$ sha256sum <pdf file> <span class="p">|</span> awk <span class="s1">'{ print $1; }'</span> >> sums.txt +$ sha256sum sums.txt <span class="p">|</span> awk <span class="s1">'{ print $1; }'</span> > topsum.txt +</pre></div> +<p>If we now sign <em>this</em> sum, we are confirming that for this particuar resource:</p> +<blockquote> +"This was the source. These were the headers for that source. This is how that source served in that manner looked for <strong>ME</strong> at the time"</blockquote> +</div> +<div class="section" id="proving-links-in-this-post"> +<h2>Proving links in this post</h2> +<p>This post makes use of several external links to articles. So as a final step, let's eat our own dogfood and add proofs for them.</p> +<table border="1" class="colwidths-given docutils"> +<caption>Article retrieval proofs</caption> +<colgroup> +<col width="34%" /> +<col width="22%" /> +<col width="22%" /> +<col width="22%" /> +</colgroup> +<thead valign="bottom"> +<tr><th class="head">Link</th> +<th class="head">Content</th> +<th class="head">Header</th> +<th class="head">Image</th> +</tr> +</thead> +<tbody valign="top"> +<tr><td><a class="reference external" href="https://gizmodo.com/i-tried-to-block-amazon-from-my-life-it-was-impossible-1830565336">https://gizmodo.com/i-tried-to-block-amazon-from-my-life-it-was-impossible-1830565336</a></td> +<td><a class="reference external" href="./misc/web_snapshot/0c657ec25e55e72702b0473f62e6d632dece992485f67c407ed1f748b3f40bc2.txt">0c657ec2</a></td> +<td><a class="reference external" href="./misc/web_snapshot/360125fa513b8db8380eb2b62c4479164baa5b48d8544b2242bcc7305bad0de4.txt">360125fa</a></td> +<td><a class="reference external" href="./misc/web_snapshot/a2584ca07ba16b15b9fad315a767cbb0c4b7124abdfd578cab2afb4dce9d1971.pdf">a2584ca0</a></td> +</tr> +<tr><td><a class="reference external" href="https://www.thelocal.se/20111109/37244/">https://www.thelocal.se/20111109/37244/</a></td> +<td><a class="reference external" href="./misc/web_snapshot/be8741ff91c6e3eac760a3a4652b57f47bce92fa9b617662296c2ab5b3c5fe31.txt">be8741ff</a></td> +<td><a class="reference external" href="./misc/web_snapshot/6c20e7678a0235467687425b9818718ab143fd477afee2087d7f79c933abdc75.txt">6c20e767</a></td> +<td><a class="reference external" href="./misc/web_snapshot/f1c557b9555149dde570976ed956ffb17d17b99cea5f2651020f66408dacf301.pdf">f1c557b9</a></td> +</tr> +<tr><td><a class="reference external" href="https://www.statista.com/chart/18819/worldwide-market-share-of-leading-cloud-infrastructure-service-providers/">https://www.statista.com/chart/18819/worldwide-market-share-of-leading-cloud-infrastructure-service-providers/</a></td> +<td><a class="reference external" href="./misc/web_snapshot/595a07b85e6b75a41fe0880c8538f15c4b6da5770db230d986efa2e080ca479a.txt">595a07b8</a></td> +<td><a class="reference external" href="./misc/web_snapshot/f2aa95b42c5420cb11ccbf1cb084d5e5ccc3fc6cd575c51e9ee473b9df19c890.txt">f2aa95b4</a></td> +<td><a class="reference external" href="./misc/web_snapshot/6a610fa69d4909cc9aa1dcb7105203cc50c1bcf26733411964f95cbcbdf37eb5.pdf">6a610fa6</a></td> +</tr> +<tr><td><a class="reference external" href="https://web.archive.org/web/20170710185409/https://www.botkyrka.se/arkiv/nyhetsarkiv/nyheter-startsida/2017-07-10-angaende-uttalanden-av-journalisten-janne-josefsson-om-bibliotek-botkyrka.html">https://web.archive.org/web/20170710185409/https://www.botkyrka.se/arkiv/nyhetsarkiv/nyheter-startsida/2017-07-10-angaende-uttalanden-av-journalisten-janne-josefsson-om-bibliotek-botkyrka.html</a></td> +<td><a class="reference external" href="./misc/web_snapshot/5a806be410da82986a85f68658279234c1a5cf3bb6dc55da137d5274dc722f26.txt">5a806be4</a></td> +<td><a class="reference external" href="./misc/web_snapshot/078d9fe6be070de0d378a6e903f8558a7da4917cba5c95ca453ae1936541e4f6.txt">078d9fe6</a></td> +<td><a class="reference external" href="./misc/web_snapshot/2d0f0e69e7c8b6ffe9ff8ffc8702a78d6a2d46ab1edd4123e84eb211171d6cde.pdf">2d0f0e69</a></td> +</tr> +<tr><td><a class="reference external" href="https://www.breitbart.com/europe/2017/07/19/swedens-libraries-pulp-traditional-children-books-racist-phrases/">https://www.breitbart.com/europe/2017/07/19/swedens-libraries-pulp-traditional-children-books-racist-phrases/</a></td> +<td><a class="reference external" href="./misc/web_snapshot/a8c6b61cec2cce1a58bcf7a65091a6c2e8510ca5fa17f2d242d286f087d95cd5.txt">a8c6b61c</a></td> +<td><a class="reference external" href="./misc/web_snapshot/3ba9094b295a898cfe8cba256f4ebf65ef98ff05bb3034e048e517d52fc13d33.txt">3ba9094b</a></td> +<td><a class="reference external" href="./misc/web_snapshot/eb76a87f224b0e4f4e5d1673b1505aaeee28d8ad03ce544656b6f5ac7c4d9983.pdf">eb76a87f</a></td> +</tr> +</tbody> +</table> +<p>Clicking on the "image" links, we see that thanks to the recent ubiquity of cookie nag boxes screaming "accept all" at you, those very boxes are now blocking the content we want to get at. So we will need more work to get to where we want by automation. But it's s start.</p> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id6" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id1">[1]</a></td><td>Early 2021 survey puts Amazon at one-third of the global market share. <a class="reference external" href="https://www.statista.com/chart/18819/worldwide-market-share-of-leading-cloud-infrastructure-service-providers/">https://www.statista.com/chart/18819/worldwide-market-share-of-leading-cloud-infrastructure-service-providers/</a></td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id7" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id2">[2]</a></td><td>2011 sparked a <a class="reference external" href="https://www.thelocal.se/20111109/37244/">controversy</a> around Astrid Lindgren's Pippi Longstockings. Echoing those viewpoints, the books were edited in 2015 during which some alleged "racist" content was altered. The rabid rightwing media later spun a <a class="reference external" href="https://www.breitbart.com/europe/2017/07/19/swedens-libraries-pulp-traditional-children-books-racist-phrases/">false tale of mass purges</a> of Pippi books around one single swedish library's decision to throw out copies of the originial "racist" versions. Case-in-point, a public statement in which the library tries to justify its actions is no longer available on their website, and has to be retrieved by the Wayback Machine. <a class="reference external" href="https://web.archive.org/web/20170710185409/https://www.botkyrka.se/arkiv/nyhetsarkiv/nyheter-startsida/2017-07-10-angaende-uttalanden-av-journalisten-janne-josefsson-om-bibliotek-botkyrka.html">https://web.archive.org/web/20170710185409/https://www.botkyrka.se/arkiv/nyhetsarkiv/nyheter-startsida/2017-07-10-angaende-uttalanden-av-journalisten-janne-josefsson-om-bibliotek-botkyrka.html</a> (in swedish)</td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id8" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id4">[3]</a></td><td>Well actually, not quite. We did the same request twice, but they were two separate requests. Using a single request would improve the script.</td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id9" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id5">[4]</a></td><td>We use the hex representation for clarity here. A proper tool would convert the hex values to bytes before calculating sum on them.</td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id10" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id3">[5]</a></td><td>Another important difference is that the book does not need to be <em>interpreted</em> by a <em>machine</em> in order to make sense for a human. Availability of tooling is definitely an equally important topic in this discussion. However this post limits focus to the actual data itself.</td></tr> +</tbody> +</table> +</blockquote> +<!-- .. _data availability: https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.433.3480 --> +<!-- https://web.archive.org/web/20170710185409/https://www.botkyrka.se/arkiv/nyhetsarkiv/nyheter-startsida/2017-07-10-angaende-uttalanden-av-journalisten-janne-josefsson-om-bibliotek-botkyrka.html <- not available on link, no result on search https://www.mhpbooks.com/the-trouble-with-pippi/ --> +</div> + + </div><!-- /.entry-content --> + <footer class="meta"> + </footer> + +</section> + <footer id="contentinfo" class="body"> + <hr/> + <address id="about" class="vcard body"> + Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> + </address><!-- /#about --> + </footer><!-- /#contentinfo --> +</body> +</html> +\ No newline at end of file diff --git a/132b445a5c24f134b82b8cb95e7112fd053ca2c15531afa694105a5d1a1d872a.asc b/132b445a5c24f134b82b8cb95e7112fd053ca2c15531afa694105a5d1a1d872a.asc @@ -0,0 +1,11 @@ +-----BEGIN PGP SIGNATURE----- + +iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmHkZ0UACgkQzcvSTdHQ +4AH7oggAo8ILCivCHh3RxAOQxTuwRUZbJaPgqTm5LlvehBp2H5l/Fi5ZOcrhswVw +ROt/Q9AwJOpTHQa6EKnZF1it0Ho+N6A3KbTy665iLSb0GZSYyQ6HmAFv+hihuty9 +L7B2DSbUTp3wkwvB8sy88VTFr+MgSuJKvP8MzdYEa/+sVngsldYflhxqJU0AdlYq +ib9aemr1vJNZGb9VGWJwSnyzHUM8nLlF3bM9ts11/dQ9pDr5t7TWQBSxupKsgSCL +Vv8/ci4SmPPvSpVauOZ59D/xbZ1HyqF8o9+QSv1V40o8njEXgFD7cJxJ7nC1fH2g +ieashaiyddU7pjoGpKo2WcUCoTKZzg== +=i0t/ +-----END PGP SIGNATURE----- diff --git a/1c19dbdeef6f5dea9dfc688b2aff4784fe2c5dbb2a8af928974d41c536f6afdd b/1c19dbdeef6f5dea9dfc688b2aff4784fe2c5dbb2a8af928974d41c536f6afdd @@ -0,0 +1,315 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <link rel="stylesheet" type="text/css" href="./theme/css/style.css" /> + <link rel="stylesheet" type="text/css" href="./theme/css/syntax.css" /> + <title>man bytes gnu - Combining duplicity and rsync</title> + <meta charset="utf-8" /> + <meta name="generator" content="Pelican" /> + +</head> + +<body id="index" class="home"> + + <header id="banner" class="body"> + <h1>> <a href="./">man bytes gnu</a><span id="cursor">_</span></h1> + </header> + <nav id="menu"> + <ul> + <li class="active"><a href="./category/archiving.html">Archiving</a></li> + <li><a href="./category/code.html">Code</a></li> + <li><a href="./category/hygiene.html">Hygiene</a></li> + <li><a href="./category/offlining.html">Offlining</a></li> + <li><a href="./pages/identities.html">Identities</a></li> + <li><a href="./pages/shares.html">Shares</a></li> + + <li><a href="/tags.html">tags</a></li> + </ul> + <hr/> + </nav><!-- /#menu --> + <header> + <h1 class="entry-title top-body-title"> + <a href="./backup-rsync-duplicity.html" rel="bookmark" + title="Permalink to Combining duplicity and rsync">Organizing backups: Combining duplicity and rsync</a></h1> + + + <div class="category meta"> + Posted +<time class="published" datetime="2022-01-15T16:57:00+01:00"> + Sat 15 January 2022 + </time> +in <a class="category" href="./category/archiving.html">archiving</a> + <a href="./tag/backup.html">backup</a> + <a href="./tag/rsync.html">rsync</a> + <a href="./tag/duplicity.html">duplicity</a> + <a href="./tag/bash.html">bash</a> + </div> + + <div class="neighbors meta"> + <ul> + <li> + Previous: <a href="./internet-up-monitor.html"> + Homemade internet state monitor + </a> + </li> + </ul> + </div> + <div class="entry-series meta"> +Part 1 from the series "Organizing backups" + <ol class="entry-series-parts"> + </ol> + </div> + <div class="meta"> +<hr/> + </div> +</header> + +<section id="content" class="body"> + <div class="entry-content"> + <p>There are two awesome, weathered tools out there that are all you really need for your personal backups. <a class="footnote-reference" href="#id4" id="id1">[1]</a> One is the <a class="reference external" href="https://rsync.samba.org/">rsync cli</a>, the other is <a class="reference external" href="https://duplicity.gitlab.io/duplicity-web/">duplicity</a>.</p> +<p>The former should need no introduction.</p> +<p>The latter operates more like tar. But it still works over ssh like rsync. In fact, it's based on <a class="reference external" href="http://librsync.sourcefrog.net/">librsync</a> which implements the <a class="reference external" href="https://rsync.samba.org/tech_report/">rsync protocol</a>. The special sauce however is, of course, <em>encryption</em>.</p> +<div class="section" id="backup-categories"> +<h2>Backup categories</h2> +<p>Let's for the sake of argument say that our personal backups can be divided in three categories:</p> +<div class="section" id="stuff-that-can-be-public"> +<h3>Stuff that can be public</h3> +<p>Code snippets, git repositories, public data store states (e.g. blockchain ledgers), copies of OS packages and any other assets assets without redistribution issues.</p> +<p>For this we will use <a class="reference external" href="https://rsync.samba.org/">rsync</a>.</p> +</div> +<div class="section" id="sensitive-stuff"> +<h3>Sensitive stuff</h3> +<p>Passwords, keys, contacts, calendars, contracts, invoices, task lists, databases, system configurations, application data.</p> +<p>For this we will use <a class="reference external" href="https://duplicity.gitlab.io/duplicity-web/">duplicity</a>.</p> +</div> +<div class="section" id="secret-stuff"> +<h3>Secret stuff</h3> +<p>Long-lived keys, password- and volume decryption keys, cryptocurrency keys and meta-information about the backups themselves.</p> +<p>This will not be addressed now.</p> +</div> +</div> +<div class="section" id="why-not-just-one-or-the-other"> +<h2>Why not just one or the other?</h2> +<p><a class="reference external" href="https://duplicity.gitlab.io/duplicity-web/">Duplicity</a> stores everything in an archive file format. That means that you must first authenticate, decrypt and unpack the archive in order to even browse the files inside.</p> +<p>If there is no reason to keep the files from prying eyes, then it's much more practical to be able to browse the files where they lie, with the regular filesystem tools. In such a case, <a class="reference external" href="https://rsync.samba.org/">rsync</a> will scratch your itch.</p> +<p>For the <strong>sensitive</strong> and <strong>secret stuff</strong>, there would be no real need to use <a class="reference external" href="https://duplicity.gitlab.io/duplicity-web/">duplicity</a> if you were only operating on your local host. You'd just use an encrypted volume <a class="footnote-reference" href="#id5" id="id2">[2]</a> and <a class="reference external" href="https://rsync.samba.org/">rsync</a> everything in there.</p> +<p>But half the point here is to keep remote copies aswell as your local ones. You know, in case of fire, hardware-eating locust swarms or some totalitarian minions nabbing all your electronics. Unless "remote" here means some box hidden in some moated leisure castle of yours, you'll want to encrypt everything <em>before</em> you ship it off. And that's where <a class="reference external" href="https://duplicity.gitlab.io/duplicity-web/">duplicity</a> comes in.</p> +</div> +<div class="section" id="vive-la-difference"> +<h2>Vive la difference</h2> +<p>Of course, it would be too much to hope for that <a class="reference external" href="https://duplicity.gitlab.io/duplicity-web/">duplicity</a> and <a class="reference external" href="https://rsync.samba.org/">rsync cli</a> have aligned the ways they parse their invocation parameters.</p> +<p>Here are some examples <a class="footnote-reference" href="#id6" id="id3">[3]</a> of how they do <em>not</em> match:</p> +<div class="section" id="local-to-local"> +<h3>local to local</h3> +<div class="highlight"><pre><span></span>$ rsync -a src/ /path/to/dst/ + +$ duplicity src/ file:///path/to/dst/ +</pre></div> +</div> +<div class="section" id="local-to-remote-relative-path"> +<h3>local to remote, relative path</h3> +<div class="highlight"><pre><span></span>$ rsync -a src/ user@remotehost:path/to/dst/ + +$ duplicity src/ scp://user@remotehost/path/to/dst +</pre></div> +</div> +<div class="section" id="toggle-dotfiles-from-current-path"> +<h3>toggle dotfiles from current path</h3> +<div class="highlight"><pre><span></span><span class="c1"># include only .foo/foo.txt given the current structure:</span> +$ tree src/ -a +src/ +├── .bar +├── baz +└── .foo + └── foo.txt + +$ rsync --exclude<span class="o">=</span><span class="s2">".b*"</span> --include<span class="o">=</span><span class="s2">".*/***"</span> --exclude<span class="o">=</span><span class="s2">"*"</span> ./ ../dst/ + +$ duplicity --exclude<span class="o">=</span><span class="s2">"./.b*"</span> --include<span class="o">=</span><span class="s2">"./.*/***"</span> --exclude<span class="o">=</span><span class="s2">"*"</span> ./ file:///home/lash/tmp/dst/ +</pre></div> +</div> +<div class="section" id="logging"> +<h3>logging</h3> +<div class="highlight"><pre><span></span><span class="c1"># spill the beans</span> +$ rsync -vv ... + +$ duplicity -v debug +</pre></div> +</div> +</div> +<div class="section" id="batchin"> +<h2>Batchin'</h2> +<p>Since you will want to select up front which tool to use for which sensititivy category, you'll be writing the includes and excludes specifically for the tool anyway.</p> +<p>So the only real issue with the above is the way remote host is specified.</p> +<p>Let's say we choose to stick to the <a class="reference external" href="https://rsync.samba.org/">rsync cli</a> host format. That means we need to make the following translations:</p> +<table border="1" class="colwidths-given docutils"> +<colgroup> +<col width="50%" /> +<col width="50%" /> +</colgroup> +<thead valign="bottom"> +<tr><th class="head">rsync</th> +<th class="head">duplicity</th> +</tr> +</thead> +<tbody valign="top"> +<tr><td><tt class="docutils literal">foo/bar</tt></td> +<td><tt class="docutils literal"><span class="pre">file://foo/bar</span></tt></td> +</tr> +<tr><td><tt class="docutils literal">/foo/bar</tt></td> +<td><tt class="docutils literal"><span class="pre">file:///foo/bar</span></tt></td> +</tr> +<tr><td><tt class="docutils literal">user@host:foo/bar</tt></td> +<td><tt class="docutils literal"><span class="pre">scp://user@host/foo/bar</span></tt></td> +</tr> +<tr><td><tt class="docutils literal"><span class="pre">user@host:/foo/bar</span></tt></td> +<td><tt class="docutils literal"><span class="pre">scp://user@host//foo/bar</span></tt></td> +</tr> +</tbody> +</table> +<p>Expressed in <tt class="docutils literal">bash</tt> that could look like this:</p> +<pre class="code bash literal-block"> +to_duplicity_remote<span class="o">()</span> <span class="o">{</span> + + <span class="c1"># remote host is defined in rsync format +</span> <span class="c1"># ... and we will only support scp +</span> <span class="c1"># $remote_base is the path we want to parse +</span> <span class="nv">remote_duplicity_base</span><span class="o">=</span> + <span class="nv">remote_base</span><span class="o">=</span><span class="nv">$1</span> + + <span class="c1"># substring up until the first slash +</span> <span class="nv">s_firstslash</span><span class="o">=</span><span class="si">${</span><span class="nv">remote_base</span><span class="p">%%/*</span><span class="si">}</span> + + <span class="c1"># substring up until the first colon +</span> <span class="nv">s_firstcolon</span><span class="o">=</span><span class="si">${</span><span class="nv">remote_base</span><span class="p">%%:*</span><span class="si">}</span> + + <span class="c1"># string index of the first slash +</span> <span class="nv">i_firstslash</span><span class="o">=</span><span class="k">$((</span><span class="si">${#</span><span class="nv">s_firstslash</span><span class="si">}</span><span class="k">))</span> + + <span class="c1"># string index of the first colon +</span> <span class="nv">i_firstcolon</span><span class="o">=</span><span class="k">$((</span><span class="si">${#</span><span class="nv">s_firstcolon</span><span class="si">}</span><span class="k">))</span> + + <span class="c1"># if colon is before first slash that most likely means we have a remote host +</span> <span class="c1"># (Exception is if first directory of path happens to have ":" in it. Seriously, don't use ":" in filenames) +</span> <span class="k">if</span> <span class="o">[</span> <span class="s2">"</span><span class="nv">$i_firstcolon</span><span class="s2">"</span> -gt <span class="s2">"0"</span> <span class="o">]</span><span class="p">;</span> <span class="k">then</span> + + <span class="k">if</span> <span class="o">[</span> <span class="s2">"</span><span class="nv">$i_firstcolon</span><span class="s2">"</span> -lt <span class="s2">"</span><span class="nv">$i_firstslash</span><span class="s2">"</span> <span class="o">]</span><span class="p">;</span> <span class="k">then</span> + + <span class="c1"># pexpect addition due to lack of implicit private key fetch, without pexpect works only with key wo pwd +</span> <span class="c1"># https://serverfault.com/questions/982591/duplicity-backup-fails-private-key-file-is-encrypted +</span> <span class="nv">remote_duplicity_base</span><span class="o">=</span><span class="s2">"pexpect+scp://"</span> + + <span class="c1"># trim url so that colon after hostname is removed +</span> <span class="c1"># (no support here for setting an alternate port number) +</span> <span class="nv">remote_duplicity_base</span><span class="o">=</span><span class="si">${</span><span class="nv">remote_duplicity_base</span><span class="si">}${</span><span class="nv">remote_base</span><span class="p">:</span><span class="nv">0</span><span class="p">:</span><span class="nv">$i_firstcolon</span><span class="si">}</span> + <span class="nv">remote_duplicity_base</span><span class="o">=</span><span class="si">${</span><span class="nv">remote_duplicity_base</span><span class="si">}</span>/<span class="si">${</span><span class="nv">remote_base</span><span class="p">:((</span><span class="nv">$i_firstcolon</span><span class="p">+1))</span><span class="si">}</span> + + <span class="c1"># indicate that we have a remote host +</span> <span class="nv">remote</span><span class="o">=</span><span class="m">1</span> + <span class="k">fi</span> + <span class="k">fi</span> + + <span class="c1"># If it's not a remote host, treat it as a file +</span> <span class="k">if</span> <span class="o">[</span> -z <span class="nv">$remote_duplicity_base</span> <span class="o">]</span><span class="p">;</span> <span class="k">then</span> + <span class="nv">remote_duplicity_base</span><span class="o">=</span><span class="s2">"file://</span><span class="si">${</span><span class="nv">remote_base</span><span class="si">}</span><span class="s2">"</span> + <span class="k">fi</span> +<span class="o">}</span> + +<span class="k">if</span> <span class="o">[</span> ! -z <span class="s2">"</span><span class="nv">$BAK_TEST</span><span class="s2">"</span> <span class="o">]</span><span class="p">;</span> <span class="k">then</span> + <span class="nv">src</span><span class="o">=(</span>/foo/bar foo/bar localhost:foo/bar localhost:/foo/bar<span class="o">)</span> + <span class="nv">res</span><span class="o">=(</span>file:///foo/bar file://foo/bar pexpect+scp://localhost/foo/bar pexpect+scp://localhost//foo/bar<span class="o">)</span> + + <span class="nv">i</span><span class="o">=</span><span class="m">0</span> + <span class="k">for</span> case_src <span class="k">in</span> <span class="si">${</span><span class="nv">src</span><span class="p">[@]</span><span class="si">}</span><span class="p">;</span> <span class="k">do</span> + <span class="nv">case_res</span><span class="o">=</span><span class="si">${</span><span class="nv">res</span><span class="p">[</span><span class="nv">$i</span><span class="p">]</span><span class="si">}</span> + + to_duplicity_remote <span class="nv">$case_src</span> + <span class="k">if</span> <span class="o">[</span> <span class="s2">"</span><span class="nv">$remote_duplicity_base</span><span class="s2">"</span> !<span class="o">=</span> <span class="s2">"</span><span class="nv">$case_res</span><span class="s2">"</span> <span class="o">]</span><span class="p">;</span> <span class="k">then</span> + ><span class="p">&</span><span class="m">2</span> <span class="nb">echo</span> <span class="s2">"expected </span><span class="nv">$case_res</span><span class="s2"> got </span><span class="nv">$remote_duplicity_base</span><span class="s2"> from </span><span class="nv">$case_src</span><span class="s2">"</span> + <span class="nb">exit</span> <span class="m">1</span> + <span class="k">elif</span> <span class="o">[</span> <span class="s2">"</span><span class="nv">$remote_base</span><span class="s2">"</span> !<span class="o">=</span> <span class="s2">"</span><span class="nv">$case_src</span><span class="s2">"</span> <span class="o">]</span><span class="p">;</span> <span class="k">then</span> + ><span class="p">&</span><span class="m">2</span> <span class="nb">echo</span> <span class="s2">"</span><span class="nv">$case_src</span><span class="s2"> got mangled into </span><span class="nv">$remote_base</span><span class="s2">"</span> + <span class="nb">exit</span> <span class="m">1</span> + <span class="k">fi</span> + <span class="nv">i</span><span class="o">=</span><span class="k">$((</span>i+1<span class="k">))</span> + <span class="k">done</span> +<span class="k">fi</span> +</pre> +<p>Let's behave and test our code:</p> +<pre class="code bash literal-block"> +<span class="k">if</span> <span class="o">[</span> ! -z <span class="s2">"</span><span class="nv">$BAK_TEST</span><span class="s2">"</span> <span class="o">]</span><span class="p">;</span> <span class="k">then</span> + <span class="nv">src</span><span class="o">=(</span>/foo/bar foo/bar localhost:foo/bar localhost:/foo/bar<span class="o">)</span> + <span class="nv">res</span><span class="o">=(</span>file:///foo/bar file://foo/bar pexpect+scp://localhost/foo/bar pexpect+scp://localhost//foo/bar<span class="o">)</span> + + <span class="nv">i</span><span class="o">=</span><span class="m">0</span> + <span class="k">for</span> case_src <span class="k">in</span> <span class="si">${</span><span class="nv">src</span><span class="p">[@]</span><span class="si">}</span><span class="p">;</span> <span class="k">do</span> + <span class="nv">case_res</span><span class="o">=</span><span class="si">${</span><span class="nv">res</span><span class="p">[</span><span class="nv">$i</span><span class="p">]</span><span class="si">}</span> + + to_duplicity_remote <span class="nv">$case_src</span> + <span class="k">if</span> <span class="o">[</span> <span class="s2">"</span><span class="nv">$remote_duplicity_base</span><span class="s2">"</span> !<span class="o">=</span> <span class="s2">"</span><span class="nv">$case_res</span><span class="s2">"</span> <span class="o">]</span><span class="p">;</span> <span class="k">then</span> + ><span class="p">&</span><span class="m">2</span> <span class="nb">echo</span> <span class="s2">"expected </span><span class="nv">$case_res</span><span class="s2"> got </span><span class="nv">$remote_duplicity_base</span><span class="s2"> from </span><span class="nv">$case_src</span><span class="s2">"</span> + <span class="nb">exit</span> <span class="m">1</span> + <span class="k">elif</span> <span class="o">[</span> <span class="s2">"</span><span class="nv">$remote_base</span><span class="s2">"</span> !<span class="o">=</span> <span class="s2">"</span><span class="nv">$case_src</span><span class="s2">"</span> <span class="o">]</span><span class="p">;</span> <span class="k">then</span> + ><span class="p">&</span><span class="m">2</span> <span class="nb">echo</span> <span class="s2">"</span><span class="nv">$case_src</span><span class="s2"> got mangled into </span><span class="nv">$remote_base</span><span class="s2">"</span> + <span class="nb">exit</span> <span class="m">1</span> + <span class="k">fi</span> + <span class="nv">i</span><span class="o">=</span><span class="k">$((</span>i+1<span class="k">))</span> + <span class="k">done</span> +<span class="k">fi</span> +</pre> +<div class="highlight"><pre><span></span><span class="c1"># 0 == good!</span> +$ <span class="nv">BAK_TEST</span><span class="o">=</span><span class="m">1</span> bash remote.sh <span class="o">&&</span> <span class="nb">echo</span> <span class="nv">$?</span> +<span class="m">0</span> +</pre></div> +<p>Now we can use the <a class="reference external" href="https://rsync.samba.org/">rsync cli</a> path input, and use that same input to a batch of single backup steps, each which may use <a class="reference external" href="https://rsync.samba.org/">rsync cli</a> or <a class="reference external" href="https://duplicity.gitlab.io/duplicity-web/">duplicity</a></p> +<div class="highlight"><pre><span></span>to_duplicity_remote localhost:/foo/bar + +rsync -avzP pub/ <span class="nv">$remote_base</span>:src/ + +duplicity -v info secret/ <span class="nv">$remote_duplicity_base</span>:secret/ +</pre></div> +</div> +<div class="section" id="see-also"> +<h2>See also</h2> +<ul class="simple"> +<li><a class="reference external" href="https://git.defalsify.org/rsync-duplicity-backups">https://git.defalsify.org/rsync-duplicity-backups</a></li> +</ul> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id4" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id1">[1]</a></td><td>Ok, I know, I assuming that you are using <tt class="docutils literal">git</tt> in daily life, too.</td></tr> +</tbody> +</table> +<table class="docutils footnote" frame="void" id="id5" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id2">[2]</a></td><td>Provided, of course, that it's an encrypted volume that you don't keep unlocked all the time.</td></tr> +</tbody> +</table> +<table class="docutils footnote" frame="void" id="id6" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id3">[3]</a></td><td>Duplicity needs at a minimum a password for symmetric encryption, and will prompt for it unless it's set in the environment. Simply <tt class="docutils literal">export PASSPHRASE=test</tt> for these examples to relieve you of the annoyance.</td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +</blockquote> +</div> + + </div><!-- /.entry-content --> + <footer class="meta"> + </footer> + +</section> + <footer id="contentinfo" class="body"> + <hr/> + <address id="about" class="vcard body"> + Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> + </address><!-- /#about --> + </footer><!-- /#contentinfo --> +</body> +</html> +\ No newline at end of file diff --git a/1c19dbdeef6f5dea9dfc688b2aff4784fe2c5dbb2a8af928974d41c536f6afdd.asc b/1c19dbdeef6f5dea9dfc688b2aff4784fe2c5dbb2a8af928974d41c536f6afdd.asc @@ -0,0 +1,11 @@ +-----BEGIN PGP SIGNATURE----- + +iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmHkZ0UACgkQzcvSTdHQ +4AG4Mgf7BS4wTvEqRqo5QpUekr7cQ7DwFrcrwIt6pYnUjqbtXnaIqXrIOBBOWrcg +Cck0FXDHvLSvPBI/RKCSCJz7RaTgkxiAZBgB2e0eXJjPuILSk0p7X2vivm9TEcHB +6ckcxhG7Le6gjK/5/8ImJ+rTB9lVMmFlUSVliw359tbx/yUVidmxUK14kYxAV0k7 +cAaebixofVkjutR8YvFiskv9t5/0sPKzFK1yRt//KP9IFzKhjyF9LTf3lOm/EUT+ +mp4Is4QtSdobCuVdS+CHAHgGBZGupRgU4RLIdX61SiIrgNoGNT3rbp1sILHfwNnG +ZVyFLO+mz0xJoaKJHKo/PAa7AF2Ffw== +=6Y6p +-----END PGP SIGNATURE----- diff --git a/2868ebaf23ffa05cd78bead77dbae8a6c0e596d67979ef45c9fd8e54db2cfbc9 b/2868ebaf23ffa05cd78bead77dbae8a6c0e596d67979ef45c9fd8e54db2cfbc9 @@ -0,0 +1,331 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <link rel="stylesheet" type="text/css" href="./theme/css/style.css" /> + <link rel="stylesheet" type="text/css" href="./theme/css/syntax.css" /> + <title>man bytes gnu - Local npm repository</title> + <meta charset="utf-8" /> + <meta name="generator" content="Pelican" /> + +</head> + +<body id="index" class="home"> + + <header id="banner" class="body"> + <h1>> <a href="./">man bytes gnu</a><span id="cursor">_</span></h1> + </header> + <nav id="menu"> + <ul> + <li><a href="./category/archiving.html">Archiving</a></li> + <li><a href="./category/code.html">Code</a></li> + <li><a href="./category/hygiene.html">Hygiene</a></li> + <li class="active"><a href="./category/offlining.html">Offlining</a></li> + <li><a href="./pages/identities.html">Identities</a></li> + <li><a href="./pages/shares.html">Shares</a></li> + + <li><a href="/tags.html">tags</a></li> + </ul> + <hr/> + </nav><!-- /#menu --> + <header> + <h1 class="entry-title top-body-title"> + <a href="./docker-offline-3-npm.html" rel="bookmark" + title="Permalink to Local npm repository">Offline Docker: Local npm repository</a></h1> + + + <div class="category meta"> + Posted +<time class="published" datetime="2021-05-25T11:47:00+02:00"> + Tue 25 May 2021 + </time> +in <a class="category" href="./category/offlining.html">offlining</a> + <a href="./tag/docker.html">docker</a> + <a href="./tag/npm.html">npm</a> + <a href="./tag/nodejs.html">nodejs</a> + <a href="./tag/javascript.html">javascript</a> + <a href="./tag/devops.html">devops</a> + </div> + + <div class="neighbors meta"> + <ul> + <li> + Previous: <a href="./village-network-protocol.html"> + A village network protocol + </a> + </li> + | + + <li> + Next: <a href="./eth-log-bloom.html"> + Making sense of Ethereum log bloom filters + </a> + </li> + </ul> + </div> + <div class="entry-series meta"> +Part 3 from the series "Offline Docker" + <ol class="entry-series-parts"> + <li>| <a href="docker-offline-1-routing.html" title="The routing to freedom ">Part 1</a></li> + <li>| <a href="docker-offline-2-python.html" title="Local python repository ">Part 2</a></li> + </ol> + </div> + <div class="meta"> +<hr/> + </div> +</header> + +<section id="content" class="body"> + <div class="entry-content"> + <p>As expected, serving a local <tt class="docutils literal">npm</tt> repository is a <em>lot</em> less straightforward than that for <tt class="docutils literal">pip</tt> and <tt class="docutils literal">python</tt>.</p> +<p>The npm registry uses the <code>CouchDB</code> nosql server as backend to resolve dependencies and serve resource metadata. The <code>npm</code> CLI tool expects a corresponding json response to requests that cite the <em>name</em> (and not the path) of the package.</p> +<p>Let's ask the <a class="reference external" href="https://registry.npmjs.org">registry</a> <a class="footnote-reference" href="#id6" id="id1">[1]</a> for the package <tt class="docutils literal">ftp</tt>, and have a look at the response (excerpt):</p> +<div class="highlight"><pre><span></span><span class="gp">$ </span>curl -X GET https://registry.npmjs.org/ftp <span class="p">|</span> jq +<span class="go">{</span> +<span class="go"> "_id": "ftp",</span> +<span class="go"> "_rev": "113-89fe76508a7ece41b4c9a157114f966f",</span> +<span class="go"> "name": "ftp",</span> +<span class="go"> "description": "An FTP client module for node.js",</span> +<span class="go"> "dist-tags": {</span> +<span class="go"> "latest": "0.3.10"</span> +<span class="go"> },</span> +<span class="go"> "versions": {</span> + + + +<span class="go"> "0.3.10": {</span> +<span class="go"> "name": "ftp",</span> +<span class="go"> "version": "0.3.10",</span> +<span class="go"> "author": {</span> +<span class="go"> "name": "Brian White",</span> +<span class="go"> "email": "mscdex@mscdex.net"</span> +<span class="go"> },</span> +<span class="go"> "description": "An FTP client module for node.js",</span> +<span class="go"> "main": "./lib/connection",</span> +<span class="go"> "engines": {</span> +<span class="go"> "node": ">=0.8.0"</span> +<span class="go"> },</span> +<span class="go"> "dependencies": {</span> +<span class="go"> "xregexp": "2.0.0",</span> +<span class="go"> "readable-stream": "1.1.x"</span> +<span class="go"> },</span> +<span class="go"> "scripts": {</span> +<span class="go"> "test": "node test/test.js"</span> +<span class="go"> },</span> +<span class="go"> "keywords": [</span> +<span class="go"> "ftp",</span> +<span class="go"> "client",</span> +<span class="go"> "transfer"</span> +<span class="go"> ],</span> +<span class="go"> "licenses": [</span> +<span class="go"> {</span> +<span class="go"> "type": "MIT",</span> +<span class="go"> "url": "http://github.com/mscdex/node-ftp/raw/master/LICENSE"</span> +<span class="go"> }</span> +<span class="go"> ],</span> +<span class="go"> "repository": {</span> +<span class="go"> "type": "git",</span> +<span class="go"> "url": "http://github.com/mscdex/node-ftp.git"</span> +<span class="go"> },</span> +<span class="go"> "bugs": {</span> +<span class="go"> "url": "https://github.com/mscdex/node-ftp/issues"</span> +<span class="go"> },</span> +<span class="go"> "homepage": "https://github.com/mscdex/node-ftp",</span> +<span class="go"> "_id": "ftp@0.3.10",</span> +<span class="go"> "_shasum": "9197d861ad8142f3e63d5a83bfe4c59f7330885d",</span> +<span class="go"> "_from": "https://github.com/mscdex/node-ftp/tarball/v0.3.10",</span> +<span class="go"> "_resolved": "https://github.com/mscdex/node-ftp/tarball/v0.3.10",</span> +<span class="go"> "_npmVersion": "1.4.28",</span> +<span class="go"> "_npmUser": {</span> +<span class="go"> "name": "mscdex",</span> +<span class="go"> "email": "mscdex@mscdex.net"</span> +<span class="go"> },</span> +<span class="go"> "maintainers": [</span> +<span class="go"> {</span> +<span class="go"> "name": "mscdex",</span> +<span class="go"> "email": "mscdex@mscdex.net"</span> +<span class="go"> }</span> +<span class="go"> ],</span> +<span class="go"> "dist": {</span> +<span class="go"> "shasum": "9197d861ad8142f3e63d5a83bfe4c59f7330885d",</span> +<span class="go"> "tarball": "https://registry.npmjs.org/ftp/-/ftp-0.3.10.tgz"</span> +<span class="go"> },</span> +<span class="go"> "directories": {}</span> +<span class="go"> }</span> +<span class="go"> },</span> +</pre></div> +<p>That looks a lot like embellished contents of the basic package.json set up by <code>npm init</code>, except it's wrapped in a versions array. It also explicitly defines an <em>absolute</em> url to a <cite>tarball</cite> under the <cite>dist</cite> key. <a class="footnote-reference" href="#id7" id="id2">[2]</a></p> +<div class="section" id="how-low-can-you-go"> +<h2>How low can you go</h2> +<p>It seems that all that's needed is a json index page served for the package name subpath of repository path, together with the tarball itself. We check this by putting together an utterly pointless package <tt class="docutils literal">foobarbarbar</tt>:</p> +<p><strong>foobarbarbar/index.js</strong></p> +<pre class="code javascript literal-block" id="foobarbarbar-index-js"> +<span class="nx">module</span><span class="p">.</span><span class="nx">exports</span> <span class="o">=</span> <span class="p">{</span> + <span class="s1">'smth'</span><span class="o">:</span> <span class="nx">smth</span><span class="p">,</span> +<span class="p">}</span> + +<span class="kd">function</span> <span class="nx">smth</span><span class="p">()</span> <span class="p">{</span> + <span class="nx">console</span><span class="p">.</span><span class="nx">log</span><span class="p">(</span><span class="s1">'foo'</span><span class="p">);</span> +<span class="p">}</span> +</pre> +<p><strong>foobarbarbar/package.json</strong></p> +<pre class="code json literal-block" id="foobarbarbar-package-json"> +<span class="p">{</span> + <span class="nt">"name"</span><span class="p">:</span> <span class="s2">"foobarbarbar"</span><span class="p">,</span> + <span class="nt">"version"</span><span class="p">:</span> <span class="s2">"1.0.0"</span><span class="p">,</span> + <span class="nt">"description"</span><span class="p">:</span> <span class="s2">"Foo repo"</span><span class="p">,</span> + <span class="nt">"main"</span><span class="p">:</span> <span class="s2">"index.js"</span><span class="p">,</span> + <span class="nt">"author"</span><span class="p">:</span> <span class="s2">"Foo Bar"</span><span class="p">,</span> + <span class="nt">"license"</span><span class="p">:</span> <span class="s2">"GPL3"</span><span class="p">,</span> +<span class="p">}</span> +</pre> +<p>Then make a tarball from those two files named <tt class="docutils literal"><span class="pre">foobarbarbar-1.0.0.tgz</span></tt>, take the sha1 sum of it,</p> +<p>Remembering our <a class="reference external" href="./docker-offline-2-python.html">virtual interface setup from the pip example</a> we stick it behind our webserver (here with npm sub-path) and add our minimal version json wrapper:</p> +<p><strong>package.json</strong></p> +<pre class="code json literal-block" id="package-json"> +<span class="p">{</span> + <span class="nt">"name"</span><span class="p">:</span> <span class="s2">"foobarbarbar"</span><span class="p">,</span> + <span class="nt">"versions"</span><span class="p">:</span> <span class="p">{</span> + <span class="nt">"1.0.0"</span><span class="p">:</span> <span class="p">{</span> + <span class="nt">"name"</span><span class="p">:</span> <span class="s2">"foobarbarbar"</span><span class="p">,</span> + <span class="nt">"version"</span><span class="p">:</span> <span class="s2">"1.0.0"</span><span class="p">,</span> + <span class="nt">"description"</span><span class="p">:</span> <span class="s2">"Foo repo"</span><span class="p">,</span> + <span class="nt">"main"</span><span class="p">:</span> <span class="s2">"index.js"</span><span class="p">,</span> + <span class="nt">"author"</span><span class="p">:</span> <span class="s2">"Foo Bar"</span><span class="p">,</span> + <span class="nt">"license"</span><span class="p">:</span> <span class="s2">"GPL3"</span><span class="p">,</span> + <span class="nt">"dist"</span><span class="p">:</span> <span class="p">{</span> + <span class="nt">"shasum"</span><span class="p">:</span> <span class="s2">"2ccd68498ef5f2bfa00f0e1e59f44686fdb296ee"</span><span class="p">,</span> + <span class="nt">"tarball"</span><span class="p">:</span> <span class="s2">"http://10.1.2.1/npm/foobarbarbar-1.0.0.tgz"</span> + <span class="p">}</span> + + <span class="p">}</span> + <span class="p">}</span> +<span class="p">}</span> +</pre> +</div> +<div class="section" id="making-introductions"> +<h2>Making introductions</h2> +<p>The central trick here is to serve a json document as the "directory index" of the HTTP server. It's useful to remind ourselves at this point that we are <em>not</em> setting up a <em>registry</em> but a <em>repository</em> that contains provisions for a <em>locked</em> or <em>frozen</em> dependency graph. The former would surely need some of the <tt class="docutils literal">CouchDB</tt> magic to resolve dependencies. Our assumption is that the latter can theoretically be realized using static files.</p> +<p>In other words; just as with the previous <a class="reference external" href="./docker-offline-2-python.html">python repository example</a>, we don't try to handle dependency resolutions for pip, but merely serve the actual package files after dependences have been resolved.</p> +<p>With Apache Web Server, using the <tt class="docutils literal">package.json</tt> as the directory index is as easy as:</p> +<div class="highlight"><pre><span></span><Directory "/srv/http/npm"> + DirectoryIndex package.json +</Directory> +</pre></div> +<p>Of course adjusting the <cite>Directory</cite> path as needed to match the local setup.</p> +<p>Make sure that the tarball and the latter <cite>package.json</cite> can be found in the <cite>foobarbarbar</cite> virtual subfolder of the above <cite>Directory</cite> directive path:</p> +<div class="highlight"><pre><span></span><span class="gp">$ </span>ls /srv/http/npm/foobarbarbar/ +<span class="go">-rw-r--r-- 1 root root 351 May 24 18:38 foobarbarbar-1.0.0.tgz</span> +<span class="go">-rw-r--r-- 1 root root 380 May 25 09:36 package.json</span> +</pre></div> +<p>Set the registry entry in your <tt class="docutils literal"><span class="pre">~./npmrc</span></tt> as follows:</p> +<div class="highlight"><pre><span></span><span class="na">registry</span><span class="o">=</span><span class="s">http://10.1.2.1/npm</span> +</pre></div> +<p>Then restart the Apache server and give it a go:</p> +<div class="highlight"><pre><span></span><span class="gp">$ </span>npm install --verbose foobarbarbar +<span class="go">npm verb cli [ '/usr/bin/node', '/usr/bin/npm', 'install', '--verbose', 'foobarbarbar' ]</span> +<span class="go">npm info using npm@7.13.0</span> +<span class="go">npm info using node@v16.1.0</span> +<span class="go">[...]</span> +<span class="go">npm http fetch GET 200 http://10.1.2.1/npm/foobarbarbar/ 6ms</span> +<span class="go">[...]</span> +<span class="go">npm http fetch GET 200 http://10.1.2.1/npm/foobarbarbar/foobarbarbar-1.0.0.tgz 25ms</span> +<span class="go">[...]</span> +<span class="go">added 2 packages in 545ms</span> +<span class="go">npm timing command:install Completed in 70ms</span> +<span class="go">[...]</span> +</pre></div> +</div> +<div class="section" id="cache-dodging"> +<h2>Cache dodging</h2> +<p>Seriously, <tt class="docutils literal">npm <span class="pre">--help</span></tt> leaves a lot to be desired. However, the online <tt class="docutils literal">npm install</tt> documentation does not yield any more clues as to whether it gives you an option of ignoring local cache like you can with <tt class="docutils literal">pip <span class="pre">--no-cache</span></tt>. This is a major pain in the ass, as you have to remember to keep your <tt class="docutils literal"><span class="pre">~/.npm</span></tt> folder clean between each install attempt. Otherwise, changes you make to the repository won't be used by <tt class="docutils literal">npm install</tt>.</p> +<p>While testing, it pays to use a folder fairly close to the fs root, like a subfolder in <cite>tmp</cite>. That way, you won't be thrown off by some stray <cite>node_modules</cite> folder somewhere down the tree.</p> +</div> +<div class="section" id="think-locally-act-locally"> +<h2>Think locally, act locally</h2> +<p>Serving packages globally apparently comes down to this:</p> +<ol class="arabic simple"> +<li>Pull the <tt class="docutils literal">package.json</tt> versions list from a proper registry.</li> +<li>Get the tarball</li> +<li>Transform the absolute package url to our host instead. (sigh)</li> +<li>(optional) Prune all the versions and metadata which is not needed (anything that's not in our minimal <tt class="docutils literal">foobarbarbar</tt> example above).</li> +</ol> +<p>Obviously, this is an annoyingly large amount of work to code up parsing and retrieval for.</p> +<p>Incidentally, there is a very nice tool called <a class="reference external" href="https://verdaccio.org">verdaccio</a> which gets us most of the way there. <a class="footnote-reference" href="#id8" id="id3">[3]</a> Its <em>storage</em> directory <a class="footnote-reference" href="#id9" id="id4">[4]</a> uses exactly the directory structure we need. After retrieving a package collection using it as a proxy, getting the files in place is merely a case of copying <a class="footnote-reference" href="#id10" id="id5">[5]</a> the files from the <em>storage</em> directory to the corresponding webserver directory.</p> +<p>Looking at the <tt class="docutils literal">package.json</tt> versions wrapper saved by <tt class="docutils literal">verdaccio</tt>, however, we see that it still preserves the absolute path of the upstream registry. Sadly we are still left with doing the third task ourselves.</p> +<p>As parsing json is one of my least favorite things in the world, I won't include the code for this last step here. For now it's sufficient that we understand the minimums required for manually setting up and serving a static, local, offline npm repository. Regardless of the pain involved.</p> +</div> +<div class="section" id="making-it-personal"> +<h2>Making it personal</h2> +<p>Whether we massage jsons ourselves or lazily resort to <tt class="docutils literal">verdaccio</tt>, the final step we need to take is the same: Setting the registry url in the npm configuration of the docker image.</p> +<p>This is merely a case of setting the registry url in the <a class="reference external" href="https://docs.npmjs.com/cli/v6/configuring-npm/npmrc#files">npmrc</a> inside the container.</p> +<div class="highlight"><pre><span></span><span class="gp">$ </span>docker run -it <span class="o">[</span>...<span class="o">]</span> npm config ls -l <span class="p">|</span> grep etc/npmrc +<span class="go">globalconfig = "/usr/etc/npmrc"</span> +</pre></div> +<p>To get at our manually provided <tt class="docutils literal">foobarbarbar</tt> package from before, the dockerfile will be (using archlinux):</p> +<div class="highlight"><pre><span></span><span class="o">[</span>...<span class="o">]</span> +<span class="k">RUN</span> pacman -S nodejs npm +<span class="k">RUN</span> mkdir -vp /usr/etc +<span class="k">RUN</span> <span class="nb">echo</span> <span class="s2">"registry=http:/10.1.2.1/npm"</span> > /usr/etc/npmrc +<span class="k">WORKDIR</span><span class="s"> /root</span> +<span class="k">RUN</span> npm install foobarbarbar +</pre></div> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id6" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id1">[1]</a></td><td>This was the registry address I had in my <cite>~/.npmrc</cite> or whether I put it in myself.</td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id7" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id2">[2]</a></td><td>I have tried using a relative path instead, both with and without a leading <tt class="docutils literal">/</tt>, but in either case the install then errors our complaining that the lock file is "corrupt." Whether the schema allows a base-url setting I do not know, as the schema documentation doesn't seem to be readily available.</td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id8" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id3">[3]</a></td><td>In fact, <tt class="docutils literal">verdaccio</tt> by itself solves the particular problem that we're trying to solve; providing an offline repository alternative. The task at hand, however, is to understand how to cope <em>without</em> using other tools than what can be considered base infrastructure (i.e. a web server).</td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id9" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id4">[4]</a></td><td>Should be <tt class="docutils literal">/etc/verdaccio/storage</tt>; see <tt class="docutils literal">/etc/verdaccio/config.yaml</tt></td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id10" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id5">[5]</a></td><td>An alternative approach would be to set the storage path to point to the same folder as the web server is serving. However, since we need to mess with the tarball paths</td></tr> +</tbody> +</table> +</blockquote> +</div> + + </div><!-- /.entry-content --> + <footer class="meta"> + </footer> + +</section> + <footer id="contentinfo" class="body"> + <hr/> + <address id="about" class="vcard body"> + Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> + </address><!-- /#about --> + </footer><!-- /#contentinfo --> +</body> +</html> +\ No newline at end of file diff --git a/2868ebaf23ffa05cd78bead77dbae8a6c0e596d67979ef45c9fd8e54db2cfbc9.asc b/2868ebaf23ffa05cd78bead77dbae8a6c0e596d67979ef45c9fd8e54db2cfbc9.asc @@ -0,0 +1,11 @@ +-----BEGIN PGP SIGNATURE----- + +iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmHkZ0UACgkQzcvSTdHQ +4AH+Igf/T59Ll3NXuF2UsG48pMveE2ywxCzER2Twm2PYM2omdUWgZ2bT5bs1ieTh +lrMCrU7yb+RxcMlP2mhca86ju+YHdAkPZA+21ZDwpSCfSnKp2E/8xN8qNIXo6TeB +S5QBDKmLNK1fLqPXnjpbwptD569XAifYzFvdvP/XWXXPKn7FzG339KFLc4xJYxpB +F5gRngN25pArUqR5Fr84PP8VI47DSV90+RKcot6klKbog7nWMVbCf2oMb+RrEqn7 +YJtuYhYhBLfLuJuY5h1SyC8bBQuOkXbLdf5+Nl6gUN+Q4FlEuZJttSBYnmaz9tj+ +x8oBBSwylLJsGfiPGNRxHvbjB+MI/g== +=/Jj5 +-----END PGP SIGNATURE----- diff --git a/2918deb778c2f4961cc75fcf66f21c6bbd1c6a848e0a5aebb2e3a9afdac25ce6 b/2918deb778c2f4961cc75fcf66f21c6bbd1c6a848e0a5aebb2e3a9afdac25ce6 @@ -0,0 +1,363 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <link rel="stylesheet" type="text/css" href="./theme/css/style.css" /> + <link rel="stylesheet" type="text/css" href="./theme/css/syntax.css" /> + <title>man bytes gnu - wasm and C - The bare necessities</title> + <meta charset="utf-8" /> + <meta name="generator" content="Pelican" /> + +</head> + +<body id="index" class="home"> + + <header id="banner" class="body"> + <h1>> <a href="./">man bytes gnu</a><span id="cursor">_</span></h1> + </header> + <nav id="menu"> + <ul> + <li><a href="./category/archiving.html">Archiving</a></li> + <li class="active"><a href="./category/code.html">Code</a></li> + <li><a href="./category/hygiene.html">Hygiene</a></li> + <li><a href="./category/offlining.html">Offlining</a></li> + <li><a href="./pages/identities.html">Identities</a></li> + <li><a href="./pages/shares.html">Shares</a></li> + + <li><a href="/tags.html">tags</a></li> + </ul> + <hr/> + </nav><!-- /#menu --> + <header> + <h1 class="entry-title top-body-title"> + <a href="./wasm-c-bare.html" rel="bookmark" + title="Permalink to wasm and C - The bare necessities">wasm and C - The bare necessities</a></h1> + + + <div class="category meta"> + Posted +<time class="published" datetime="2021-05-05T06:15:00+02:00"> + Wed 05 May 2021 + </time> +in <a class="category" href="./category/code.html">code</a> + <a href="./tag/wasm.html">wasm</a> + <a href="./tag/c.html">c</a> + <a href="./tag/clang.html">clang</a> + <a href="./tag/llvm.html">llvm</a> + </div> + + <div class="neighbors meta"> + <ul> + <li> + Previous: <a href="./web-snapshot.html"> + Proving what you link to + </a> + </li> + | + + <li> + Next: <a href="./nft-tokenid-content.html"> + The NFT token id as URI + </a> + </li> + </ul> + </div> + <div class="meta"> +<hr/> + </div> +</header> + +<section id="content" class="body"> + <div class="entry-content"> + <p>I am currently resuming my self-improvement task of learning Webassembly. Or, I should rather say, <em>using</em> Webassembly.</p> +<p>Since I have some optimized small code chunks that I want to use in an embedded environment, it seemed sensible that the first step would be to establish two-way communication with C.</p> +<p>In my last outing around three years ago, I was using <a class="reference external" href="https://emscripten.org/">Emscripten</a> to bridge the gap. That tool adds quite a few bells and whistles, and doesn't quite yield that warm, fuzzy bare-metal rush. <a class="reference external" href="https://emscripten.org/">Emscripten</a> relies on <a class="reference external" href="http://clang.org/">Clang</a> and <a class="reference external" href="https://llvm.org/">LLVM</a>, all of which seem to have gotten their <code>wasm</code> support built-in in the meantime (at least on my archlinux system). This it integrates nicely with <a class="reference external" href="https://github.com/WebAssembly/wabt">wabt</a> - the swiss-army knife of Webassembly.</p> +<p>So how far do we get with just <a class="reference external" href="http://clang.org/">clang</a>, <a class="reference external" href="https://llvm.org/">LLVM</a> and <a class="reference external" href="https://github.com/WebAssembly/wabt">wabt</a> ? Let's see if we at least can set up a code snippet which simply writes <em>"foobar"</em> to memory. The host will write <em>"foo"</em>, and <code>wasm</code> will write <em>"bar"</em>.</p> +<div class="section" id="without-libc"> +<h2>Without libc</h2> +<p>This excellent <a class="reference external" href="https://surma.dev/things/c-to-webassembly/">tutorial by Surma</a> provides a good starting point. Go ahead and <strong>read that first</strong>. This text is not a Webassembly primer, so the following will make a lot more sense if you do.</p> +<p>That setup still adds some magic. Namely, the <em>memory</em> and <em>symbol table</em> are here added by the <em>wasm linker</em>. It would be even more fun to pass this from the host system instead.</p> +<p>And so we start: <a class="footnote-reference" href="#id4" id="id1">[1]</a></p> +<pre class="code c literal-block"> +<span class="ln"> 0 </span><span class="cp">#include</span><span class="w"> </span><span class="cpf">"string.h"</span><span class="cp"> +</span><span class="ln"> 1 </span><span class="cp"></span><span class="w"> +</span><span class="ln"> 2 </span><span class="w"></span><span class="k">extern</span><span class="w"> </span><span class="kt">unsigned</span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="n">__heap_base</span><span class="p">;</span><span class="w"> +</span><span class="ln"> 3 </span><span class="w"></span><span class="k">extern</span><span class="w"> </span><span class="kt">void</span><span class="w"> </span><span class="nf">call_me_sometime</span><span class="p">(</span><span class="kt">unsigned</span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="o">*</span><span class="n">b</span><span class="p">);</span><span class="w"> +</span><span class="ln"> 4 </span><span class="w"> +</span><span class="ln"> 5 </span><span class="w"></span><span class="kt">void</span><span class="w"> </span><span class="nf">foo</span><span class="p">()</span><span class="w"> </span><span class="p">{</span><span class="w"> +</span><span class="ln"> 6 </span><span class="w"> </span><span class="kt">unsigned</span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="o">*</span><span class="n">buf</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">(</span><span class="kt">unsigned</span><span class="w"> </span><span class="kt">char</span><span class="o">*</span><span class="p">)</span><span class="o">&</span><span class="n">__heap_base</span><span class="p">;</span><span class="w"> +</span><span class="ln"> 7 </span><span class="w"> </span><span class="o">*</span><span class="p">(</span><span class="n">buf</span><span class="o">+</span><span class="mi">3</span><span class="p">)</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="sc">'b'</span><span class="p">;</span><span class="w"> +</span><span class="ln"> 8 </span><span class="w"> </span><span class="o">*</span><span class="p">(</span><span class="n">buf</span><span class="o">+</span><span class="mi">4</span><span class="p">)</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="sc">'a'</span><span class="p">;</span><span class="w"> +</span><span class="ln"> 9 </span><span class="w"> </span><span class="o">*</span><span class="p">(</span><span class="n">buf</span><span class="o">+</span><span class="mi">5</span><span class="p">)</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="sc">'r'</span><span class="p">;</span><span class="w"> +</span><span class="ln">10 </span><span class="w"> </span><span class="n">call_me_sometime</span><span class="p">(</span><span class="n">buf</span><span class="p">);</span><span class="w"> +</span><span class="ln">11 </span><span class="w"></span><span class="p">}</span> +</pre> +<p>Compiling this without linking gives us a hint on what needs to be defined.</p> +<div class="highlight"><pre><span></span>$ clang --target<span class="o">=</span>wasm32 -nostdlib -nostartfiles -o bare.wasm -c bare.c +$ wasm-objdump -x bare.wasm +bare.wasm: file format wasm 0x1 + +Section Details: + +Type<span class="o">[</span><span class="m">2</span><span class="o">]</span>: + - type<span class="o">[</span><span class="m">0</span><span class="o">]</span> <span class="o">()</span> -> nil + - type<span class="o">[</span><span class="m">1</span><span class="o">]</span> <span class="o">(</span>i32<span class="o">)</span> -> nil +Import<span class="o">[</span><span class="m">4</span><span class="o">]</span>: + - memory<span class="o">[</span><span class="m">0</span><span class="o">]</span> pages: <span class="nv">initial</span><span class="o">=</span><span class="m">0</span> <- env.__linear_memory + - table<span class="o">[</span><span class="m">0</span><span class="o">]</span> <span class="nv">type</span><span class="o">=</span>funcref <span class="nv">initial</span><span class="o">=</span><span class="m">0</span> <- env.__indirect_function_table + - global<span class="o">[</span><span class="m">0</span><span class="o">]</span> i32 <span class="nv">mutable</span><span class="o">=</span><span class="m">1</span> <- env.__stack_pointer + - func<span class="o">[</span><span class="m">0</span><span class="o">]</span> <span class="nv">sig</span><span class="o">=</span><span class="m">1</span> <env.call_me_sometime> <- env.call_me_sometime +Function<span class="o">[</span><span class="m">1</span><span class="o">]</span>: + - func<span class="o">[</span><span class="m">1</span><span class="o">]</span> <span class="nv">sig</span><span class="o">=</span><span class="m">0</span> <foo> +Code<span class="o">[</span><span class="m">1</span><span class="o">]</span>: + - func<span class="o">[</span><span class="m">1</span><span class="o">]</span> <span class="nv">size</span><span class="o">=</span><span class="m">138</span> <foo> +Custom: + - name: <span class="s2">"linking"</span> + - symbol table <span class="o">[</span><span class="nv">count</span><span class="o">=</span><span class="m">4</span><span class="o">]</span> + - <span class="m">0</span>: F <foo> <span class="nv">func</span><span class="o">=</span><span class="m">1</span> <span class="nv">binding</span><span class="o">=</span>global <span class="nv">vis</span><span class="o">=</span>hidden + - <span class="m">1</span>: G <env.__stack_pointer> <span class="nv">global</span><span class="o">=</span><span class="m">0</span> undefined <span class="nv">binding</span><span class="o">=</span>global <span class="nv">vis</span><span class="o">=</span>default + - <span class="m">2</span>: D <__heap_base> undefined <span class="nv">binding</span><span class="o">=</span>global <span class="nv">vis</span><span class="o">=</span>default + - <span class="m">3</span>: F <env.call_me_sometime> <span class="nv">func</span><span class="o">=</span><span class="m">0</span> undefined <span class="nv">binding</span><span class="o">=</span>global <span class="nv">vis</span><span class="o">=</span>default +Custom: + - name: <span class="s2">"reloc.CODE"</span> + - relocations <span class="k">for</span> section: <span class="m">3</span> <span class="o">(</span>Code<span class="o">)</span> <span class="o">[</span><span class="m">5</span><span class="o">]</span> + - R_wasm_GLOBAL_INDEX_LEB <span class="nv">offset</span><span class="o">=</span>0x000007<span class="o">(</span><span class="nv">file</span><span class="o">=</span>0x000099<span class="o">)</span> <span class="nv">symbol</span><span class="o">=</span><span class="m">1</span> <env.__stack_pointer> + - R_wasm_GLOBAL_INDEX_LEB <span class="nv">offset</span><span class="o">=</span>0x00001c<span class="o">(</span><span class="nv">file</span><span class="o">=</span>0x0000ae<span class="o">)</span> <span class="nv">symbol</span><span class="o">=</span><span class="m">1</span> <env.__stack_pointer> + - R_wasm_MEMORY_ADDR_SLEB <span class="nv">offset</span><span class="o">=</span>0x000031<span class="o">(</span><span class="nv">file</span><span class="o">=</span>0x0000c3<span class="o">)</span> <span class="nv">symbol</span><span class="o">=</span><span class="m">2</span> <__heap_base> + - R_wasm_FUNCTION_INDEX_LEB <span class="nv">offset</span><span class="o">=</span>0x000073<span class="o">(</span><span class="nv">file</span><span class="o">=</span>0x000105<span class="o">)</span> <span class="nv">symbol</span><span class="o">=</span><span class="m">3</span> <env.call_me_sometime> + - R_wasm_GLOBAL_INDEX_LEB <span class="nv">offset</span><span class="o">=</span>0x000086<span class="o">(</span><span class="nv">file</span><span class="o">=</span>0x000118<span class="o">)</span> <span class="nv">symbol</span><span class="o">=</span><span class="m">1</span> <env.__stack_pointer> +Custom: + - name: <span class="s2">"producers"</span> +</pre></div> +<p>Using nodejs as the host, we check if we can instantiate a <code>WebAssembly</code> object</p> +<pre class="code javascript literal-block"> +<span class="ln"> 0 </span><span class="kd">const</span> <span class="nx">fs</span> <span class="o">=</span> <span class="nx">require</span><span class="p">(</span><span class="s1">'fs'</span><span class="p">);</span> +<span class="ln"> 1 </span> +<span class="ln"> 2 </span><span class="kd">const</span> <span class="nx">imports</span> <span class="o">=</span> <span class="p">{}</span> +<span class="ln"> 3 </span> +<span class="ln"> 4 </span><span class="k">async</span> <span class="kd">function</span> <span class="nx">init</span><span class="p">()</span> <span class="p">{</span> +<span class="ln"> 5 </span> <span class="kd">const</span> <span class="nx">code</span> <span class="o">=</span> <span class="nx">fs</span><span class="p">.</span><span class="nx">readFileSync</span><span class="p">(</span><span class="s1">'./bare.wasm'</span><span class="p">);</span> +<span class="ln"> 6 </span> <span class="kd">const</span> <span class="nx">m</span> <span class="o">=</span> <span class="ow">new</span> <span class="nx">WebAssembly</span><span class="p">.</span><span class="nx">Module</span><span class="p">(</span><span class="nx">code</span><span class="p">);</span> +<span class="ln"> 7 </span> <span class="kd">const</span> <span class="nx">i</span> <span class="o">=</span> <span class="ow">new</span> <span class="nx">WebAssembly</span><span class="p">.</span><span class="nx">Instance</span><span class="p">(</span><span class="nx">m</span><span class="p">,</span> <span class="nx">imports</span><span class="p">);</span> +<span class="ln"> 8 </span><span class="p">}</span> +<span class="ln"> 9 </span><span class="nx">init</span><span class="p">();</span> +</pre> +<p>Running this tells us we are apparently missing a property <code>env</code> in the imports object.</p> +<div class="highlight"><pre><span></span>$ node bare_naive.js +/home/lash/src/tests/wasm/bare/bare_naive.js:8 +const <span class="nv">i</span> <span class="o">=</span> new WebAssembly.Instance<span class="o">(</span>m, imports<span class="o">)</span><span class="p">;</span> + ^ + +TypeError: WebAssembly.Instance<span class="o">()</span>: Import <span class="c1">#0 module="env" error: module is not an object or function</span> +</pre></div> +<p>That seems to match with the <code>Import</code> section in the <code>objdump</code> output above. Let's stick the <em>memory</em> and <em>table</em> in there. <a class="footnote-reference" href="#id5" id="id2">[2]</a></p> +<p>And let's make a bold guess that the callback function <code>call_me_sometime</code> needs to go in there aswell.</p> +<pre class="code javascript literal-block"> +<span class="ln"> 0 </span><span class="kd">const</span> <span class="nx">fs</span> <span class="o">=</span> <span class="nx">require</span><span class="p">(</span><span class="s1">'fs'</span><span class="p">);</span> +<span class="ln"> 1 </span> +<span class="ln"> 2 </span><span class="kd">const</span> <span class="nx">memory</span> <span class="o">=</span> <span class="ow">new</span> <span class="nx">WebAssembly</span><span class="p">.</span><span class="nx">Memory</span><span class="p">({</span><span class="nx">initial</span><span class="o">:</span> <span class="mf">2</span><span class="p">});</span> +<span class="ln"> 3 </span><span class="kd">const</span> <span class="nx">table</span> <span class="o">=</span> <span class="ow">new</span> <span class="nx">WebAssembly</span><span class="p">.</span><span class="nx">Table</span><span class="p">({</span><span class="nx">initial</span><span class="o">:</span> <span class="mf">3</span><span class="p">,</span> <span class="nx">element</span><span class="o">:</span> <span class="s1">'anyfunc'</span><span class="p">});</span> +<span class="ln"> 4 </span><span class="kd">const</span> <span class="nx">importsObj</span> <span class="o">=</span> <span class="p">{</span> +<span class="ln"> 5 </span> <span class="nx">env</span><span class="o">:</span> <span class="p">{</span> +<span class="ln"> 6 </span> <span class="nx">memory</span><span class="o">:</span> <span class="nx">memory</span><span class="p">,</span> +<span class="ln"> 7 </span> <span class="nx">__linear_memory</span><span class="o">:</span> <span class="nx">memory</span><span class="p">,</span> +<span class="ln"> 8 </span> <span class="nx">__indirect_function_table</span><span class="o">:</span> <span class="nx">table</span><span class="p">,</span> +<span class="ln"> 9 </span> <span class="nx">call_me_sometime</span><span class="o">:</span> <span class="p">(</span><span class="nx">n</span><span class="p">)</span> <span class="p">=></span> <span class="p">{</span> +<span class="ln">10 </span> <span class="kd">let</span> <span class="nx">a</span> <span class="o">=</span> <span class="ow">new</span> <span class="nb">Uint8Array</span><span class="p">(</span><span class="nx">memory</span><span class="p">.</span><span class="nx">buffer</span><span class="p">,</span> <span class="nx">n</span><span class="p">,</span> <span class="mf">9</span><span class="p">)</span> +<span class="ln">11 </span> <span class="nx">a</span><span class="p">.</span><span class="nx">set</span><span class="p">([</span><span class="mh">0x66</span><span class="p">,</span> <span class="mh">0x6f</span><span class="p">,</span> <span class="mh">0x6f</span><span class="p">],</span> <span class="mf">0</span><span class="p">);</span> +<span class="ln">12 </span> <span class="nx">console</span><span class="p">.</span><span class="nx">debug</span><span class="p">(</span><span class="s1">'heap is at: '</span> <span class="o">+</span> <span class="nx">n</span><span class="p">);</span> +<span class="ln">13 </span> <span class="nx">console</span><span class="p">.</span><span class="nx">log</span><span class="p">(</span><span class="s1">'heap contains: '</span> <span class="o">+</span> <span class="ow">new</span> <span class="nx">TextDecoder</span><span class="p">().</span><span class="nx">decode</span><span class="p">(</span><span class="nx">a</span><span class="p">));</span> +<span class="ln">14 </span> <span class="p">},</span> +<span class="ln">15 </span> <span class="p">},</span> +<span class="ln">16 </span><span class="p">}</span> +<span class="ln">17 </span> +<span class="ln">18 </span><span class="k">async</span> <span class="kd">function</span> <span class="nx">init</span><span class="p">()</span> <span class="p">{</span> +<span class="ln">19 </span> <span class="kd">const</span> <span class="nx">code</span> <span class="o">=</span> <span class="nx">fs</span><span class="p">.</span><span class="nx">readFileSync</span><span class="p">(</span><span class="s1">'./bare.wasm'</span><span class="p">);</span> +<span class="ln">20 </span> <span class="kd">const</span> <span class="nx">m</span> <span class="o">=</span> <span class="ow">new</span> <span class="nx">WebAssembly</span><span class="p">.</span><span class="nx">Module</span><span class="p">(</span><span class="nx">code</span><span class="p">);</span> +<span class="ln">21 </span> <span class="kd">const</span> <span class="nx">i</span> <span class="o">=</span> <span class="ow">new</span> <span class="nx">WebAssembly</span><span class="p">.</span><span class="nx">Instance</span><span class="p">(</span><span class="nx">m</span><span class="p">,</span> <span class="nx">importsObj</span><span class="p">);</span> +<span class="ln">22 </span> <span class="nx">i</span><span class="p">.</span><span class="nx">exports</span><span class="p">.</span><span class="nx">foo</span><span class="p">();</span> +<span class="ln">23 </span><span class="p">}</span> +<span class="ln">24 </span><span class="nx">init</span><span class="p">();</span> +</pre> +<p>The linker needs a little help from us for this:</p> +<ul class="simple"> +<li>Our callback function will not be available at link time, so we have to <code>--allow-undefined</code> to promise that the host has got this covered.</li> +<li><code>--import-memory</code> and <code>--import-table</code> to enable us to get memory and symbol table from the host.</li> +<li><code>--export="foo"</code> to make sure we only export exactly what we intend to from our <code>wasm</code>.</li> +</ul> +<div class="highlight"><pre><span></span>$ clang --target<span class="o">=</span>wasm32 -nostdlib -nostartfiles -Wl,--no-entry -Wl,--export<span class="o">=</span><span class="s2">"foo"</span> -Wl,--import-memory -Wl,--import-table -Wl,--allow-undefined -o bare.wasm bare.c +</pre></div> +<p>And that should give us:</p> +<div class="highlight"><pre><span></span>$ node bare.js +heap is at: <span class="m">66560</span> +heap contains: foobar +</pre></div> +<p>This way of pointing to memory is of course grossly inadequate <em>and</em> unsafe <em>and</em> ridiculous for any purpose more advanced that this one. So some proper memory management would not be a bad thing.</p> +</div> +<div class="section" id="adding-libc"> +<h2>Adding libc</h2> +<p>And what do you know. In other news since last time I looked at this is the addition of "a libc for WebAssembly programs built on top of WASI system calls." <a class="citation-reference" href="#wasi-libc" id="id3">[wasi-libc]</a>. Let's see if we can add a slightly less manual way of handling memory with <code>malloc</code> and <code>memcpy</code></p> +<pre class="code c literal-block"> +<span class="ln"> 0 </span><span class="cp">#ifdef HAVE_LIBC +</span><span class="ln"> 1 </span><span class="cp">#include</span><span class="w"> </span><span class="cpf"><string.h></span><span class="cp"> +</span><span class="ln"> 2 </span><span class="cp">#include</span><span class="w"> </span><span class="cpf"><stdlib.h></span><span class="cp"> +</span><span class="ln"> 3 </span><span class="cp">#endif +</span><span class="ln"> 4 </span><span class="cp"></span><span class="w"> +</span><span class="ln"> 5 </span><span class="w"></span><span class="k">extern</span><span class="w"> </span><span class="kt">unsigned</span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="n">__heap_base</span><span class="p">;</span><span class="w"> +</span><span class="ln"> 6 </span><span class="w"></span><span class="k">extern</span><span class="w"> </span><span class="kt">void</span><span class="w"> </span><span class="nf">call_me_sometime</span><span class="p">(</span><span class="kt">unsigned</span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="o">*</span><span class="n">b</span><span class="p">);</span><span class="w"> +</span><span class="ln"> 7 </span><span class="w"> +</span><span class="ln"> 8 </span><span class="w"></span><span class="kt">void</span><span class="w"> </span><span class="nf">foo</span><span class="p">()</span><span class="w"> </span><span class="p">{</span><span class="w"> +</span><span class="ln"> 9 </span><span class="w"> +</span><span class="ln">10 </span><span class="w"></span><span class="cp">#ifdef HAVE_LIBC +</span><span class="ln">11 </span><span class="cp"></span><span class="w"> </span><span class="kt">unsigned</span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="o">*</span><span class="n">buf</span><span class="p">;</span><span class="w"> +</span><span class="ln">12 </span><span class="w"> </span><span class="n">buf</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">malloc</span><span class="p">(</span><span class="mi">9</span><span class="p">);</span><span class="w"> +</span><span class="ln">13 </span><span class="w"> </span><span class="n">memcpy</span><span class="p">(</span><span class="n">buf</span><span class="o">+</span><span class="mi">3</span><span class="p">,</span><span class="w"> </span><span class="s">"bazbar"</span><span class="p">,</span><span class="w"> </span><span class="mi">6</span><span class="p">);</span><span class="w"> +</span><span class="ln">14 </span><span class="w"></span><span class="cp">#else +</span><span class="ln">15 </span><span class="cp"></span><span class="w"> </span><span class="kt">unsigned</span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="o">*</span><span class="n">buf</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">(</span><span class="kt">unsigned</span><span class="w"> </span><span class="kt">char</span><span class="o">*</span><span class="p">)</span><span class="o">&</span><span class="n">__heap_base</span><span class="p">;</span><span class="w"> +</span><span class="ln">16 </span><span class="w"> </span><span class="o">*</span><span class="p">(</span><span class="n">buf</span><span class="o">+</span><span class="mi">3</span><span class="p">)</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="sc">'b'</span><span class="p">;</span><span class="w"> +</span><span class="ln">17 </span><span class="w"> </span><span class="o">*</span><span class="p">(</span><span class="n">buf</span><span class="o">+</span><span class="mi">4</span><span class="p">)</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="sc">'a'</span><span class="p">;</span><span class="w"> +</span><span class="ln">18 </span><span class="w"> </span><span class="o">*</span><span class="p">(</span><span class="n">buf</span><span class="o">+</span><span class="mi">5</span><span class="p">)</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="sc">'r'</span><span class="p">;</span><span class="w"> +</span><span class="ln">19 </span><span class="w"></span><span class="cp">#endif +</span><span class="ln">20 </span><span class="cp"></span><span class="w"> </span><span class="n">call_me_sometime</span><span class="p">(</span><span class="n">buf</span><span class="p">);</span><span class="w"> +</span><span class="ln">21 </span><span class="w"> +</span><span class="ln">22 </span><span class="w"></span><span class="cp">#ifdef HAVE_LIBC +</span><span class="ln">23 </span><span class="cp"></span><span class="w"> </span><span class="n">free</span><span class="p">(</span><span class="n">buf</span><span class="p">);</span><span class="w"> +</span><span class="ln">24 </span><span class="w"></span><span class="cp">#endif +</span><span class="ln">25 </span><span class="cp"></span><span class="w"> +</span><span class="ln">26 </span><span class="w"></span><span class="p">}</span> +</pre> +<p>As you see, we need a few more parameters for the compiler and linker at this point. The <code>--target=wasm32-unknown-wasi --sysroot /opt/wasi-libc .. /opt/wasi-libc/lib/wasm32-wasi/libc.a</code> is needed to hook us up with headers and symbols for the libc.</p> +<p>My archlinux puts that sysroot in <code>/opt/wasi-libc</code>, that may of course not be the case elsewhere.</p> +<div class="highlight"><pre><span></span>$ clang -DHAVE_LIBC<span class="o">=</span><span class="m">1</span> --target<span class="o">=</span>wasm32-unknown-wasi --sysroot /opt/wasi-libc -nostdlib -nostartfiles -Wl,--no-entry -Wl,--export<span class="o">=</span><span class="s2">"foo"</span> -Wl,--import-memory -Wl,--import-table -Wl,--allow-undefined -o bare.wasm bare.c /opt/wasi-libc/lib/wasm32-wasi/libc.a +$ wasm-objdump -x bare.wasm + +bare.wasm: file format wasm 0x1 + +Section Details: + +Type<span class="o">[</span><span class="m">3</span><span class="o">]</span>: + - type<span class="o">[</span><span class="m">0</span><span class="o">]</span> <span class="o">(</span>i32<span class="o">)</span> -> nil + - type<span class="o">[</span><span class="m">1</span><span class="o">]</span> <span class="o">()</span> -> nil + - type<span class="o">[</span><span class="m">2</span><span class="o">]</span> <span class="o">(</span>i32<span class="o">)</span> -> i32 +Import<span class="o">[</span><span class="m">3</span><span class="o">]</span>: + - memory<span class="o">[</span><span class="m">0</span><span class="o">]</span> pages: <span class="nv">initial</span><span class="o">=</span><span class="m">2</span> <- env.memory + - table<span class="o">[</span><span class="m">0</span><span class="o">]</span> <span class="nv">type</span><span class="o">=</span>funcref <span class="nv">initial</span><span class="o">=</span><span class="m">1</span> <- env.__indirect_function_table + - func<span class="o">[</span><span class="m">0</span><span class="o">]</span> <span class="nv">sig</span><span class="o">=</span><span class="m">0</span> <call_me_sometime> <- env.call_me_sometime +Function<span class="o">[</span><span class="m">7</span><span class="o">]</span>: + - func<span class="o">[</span><span class="m">1</span><span class="o">]</span> <span class="nv">sig</span><span class="o">=</span><span class="m">1</span> <foo> + - func<span class="o">[</span><span class="m">2</span><span class="o">]</span> <span class="nv">sig</span><span class="o">=</span><span class="m">2</span> <malloc> + - func<span class="o">[</span><span class="m">3</span><span class="o">]</span> <span class="nv">sig</span><span class="o">=</span><span class="m">2</span> <dlmalloc> + - func<span class="o">[</span><span class="m">4</span><span class="o">]</span> <span class="nv">sig</span><span class="o">=</span><span class="m">0</span> <free> + - func<span class="o">[</span><span class="m">5</span><span class="o">]</span> <span class="nv">sig</span><span class="o">=</span><span class="m">0</span> <dlfree> + - func<span class="o">[</span><span class="m">6</span><span class="o">]</span> <span class="nv">sig</span><span class="o">=</span><span class="m">1</span> <abort> + - func<span class="o">[</span><span class="m">7</span><span class="o">]</span> <span class="nv">sig</span><span class="o">=</span><span class="m">2</span> <sbrk> +Global<span class="o">[</span><span class="m">2</span><span class="o">]</span>: + - global<span class="o">[</span><span class="m">0</span><span class="o">]</span> i32 <span class="nv">mutable</span><span class="o">=</span><span class="m">1</span> - init <span class="nv">i32</span><span class="o">=</span><span class="m">67072</span> + - global<span class="o">[</span><span class="m">1</span><span class="o">]</span> i32 <span class="nv">mutable</span><span class="o">=</span><span class="m">0</span> <__heap_base> - init <span class="nv">i32</span><span class="o">=</span><span class="m">67072</span> +Export<span class="o">[</span><span class="m">2</span><span class="o">]</span>: + - func<span class="o">[</span><span class="m">1</span><span class="o">]</span> <foo> -> <span class="s2">"foo"</span> + - global<span class="o">[</span><span class="m">1</span><span class="o">]</span> -> <span class="s2">"__heap_base"</span> +Code<span class="o">[</span><span class="m">7</span><span class="o">]</span>: + - func<span class="o">[</span><span class="m">1</span><span class="o">]</span> <span class="nv">size</span><span class="o">=</span><span class="m">171</span> <foo> + - func<span class="o">[</span><span class="m">2</span><span class="o">]</span> <span class="nv">size</span><span class="o">=</span><span class="m">10</span> <malloc> + - func<span class="o">[</span><span class="m">3</span><span class="o">]</span> <span class="nv">size</span><span class="o">=</span><span class="m">6984</span> <dlmalloc> + - func<span class="o">[</span><span class="m">4</span><span class="o">]</span> <span class="nv">size</span><span class="o">=</span><span class="m">10</span> <free> + - func<span class="o">[</span><span class="m">5</span><span class="o">]</span> <span class="nv">size</span><span class="o">=</span><span class="m">1908</span> <dlfree> + - func<span class="o">[</span><span class="m">6</span><span class="o">]</span> <span class="nv">size</span><span class="o">=</span><span class="m">4</span> <abort> + - func<span class="o">[</span><span class="m">7</span><span class="o">]</span> <span class="nv">size</span><span class="o">=</span><span class="m">78</span> <sbrk> +Data<span class="o">[</span><span class="m">2</span><span class="o">]</span>: + - segment<span class="o">[</span><span class="m">0</span><span class="o">]</span> <span class="nv">memory</span><span class="o">=</span><span class="m">0</span> <span class="nv">size</span><span class="o">=</span><span class="m">7</span> - init <span class="nv">i32</span><span class="o">=</span><span class="m">1024</span> + - <span class="m">0000400</span>: <span class="m">6261</span> 7a62 <span class="m">6172</span> <span class="m">00</span> bazbar. + - segment<span class="o">[</span><span class="m">1</span><span class="o">]</span> <span class="nv">memory</span><span class="o">=</span><span class="m">0</span> <span class="nv">size</span><span class="o">=</span><span class="m">500</span> - init <span class="nv">i32</span><span class="o">=</span><span class="m">1032</span> + - <span class="m">0000408</span>: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - <span class="m">0000418</span>: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - <span class="m">0000428</span>: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - <span class="m">0000438</span>: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - <span class="m">0000448</span>: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - <span class="m">0000458</span>: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - <span class="m">0000468</span>: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - <span class="m">0000478</span>: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - <span class="m">0000488</span>: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - <span class="m">0000498</span>: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - 00004a8: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - 00004b8: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - 00004c8: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - 00004d8: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - 00004e8: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - 00004f8: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - <span class="m">0000508</span>: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - <span class="m">0000518</span>: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - <span class="m">0000528</span>: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - <span class="m">0000538</span>: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - <span class="m">0000548</span>: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - <span class="m">0000558</span>: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - <span class="m">0000568</span>: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - <span class="m">0000578</span>: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - <span class="m">0000588</span>: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - <span class="m">0000598</span>: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - 00005a8: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - 00005b8: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - 00005c8: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - 00005d8: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - 00005e8: <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> <span class="m">0000</span> ................ + - 00005f8: <span class="m">0000</span> <span class="m">0000</span> .... +Custom: + - name: <span class="s2">"name"</span> + - func<span class="o">[</span><span class="m">0</span><span class="o">]</span> <call_me_sometime> + - func<span class="o">[</span><span class="m">1</span><span class="o">]</span> <foo> + - func<span class="o">[</span><span class="m">2</span><span class="o">]</span> <malloc> + - func<span class="o">[</span><span class="m">3</span><span class="o">]</span> <dlmalloc> + - func<span class="o">[</span><span class="m">4</span><span class="o">]</span> <free> + - func<span class="o">[</span><span class="m">5</span><span class="o">]</span> <dlfree> + - func<span class="o">[</span><span class="m">6</span><span class="o">]</span> <abort> + - func<span class="o">[</span><span class="m">7</span><span class="o">]</span> <sbrk> +Custom: + - name: <span class="s2">"producers"</span> +</pre></div> +<p>What luxury. And of course, our <code>bare.wasm</code> file just grew from 350 bytes to 10k...</p> +<p>We don't have to change our <code>javascript</code> code at this point. Simply run again, and get:</p> +<div class="highlight"><pre><span></span>$ node bare.js +heap is at: <span class="m">67088</span> +heap contains: foobazbar +</pre></div> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id4" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id1">[1]</a></td><td><code>__heap_base</code> will be set by default by the wasm environment, and is thus available as an external symbol.</td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id5" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id2">[2]</a></td><td>After linking the memory symbol meeds to be called <code>memory</code> instead of <code>__linear_memory</code> for some reason. Thus we add both here for clarity.</td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils citation" frame="void" id="wasi-libc" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id3">[wasi-libc]</a></td><td><a class="reference external" href="https://github.com/WebAssembly/wasi-libc">https://github.com/WebAssembly/wasi-libc</a></td></tr> +</tbody> +</table> +</blockquote> +</div> + + </div><!-- /.entry-content --> + <footer class="meta"> + </footer> + +</section> + <footer id="contentinfo" class="body"> + <hr/> + <address id="about" class="vcard body"> + Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> + </address><!-- /#about --> + </footer><!-- /#contentinfo --> +</body> +</html> +\ No newline at end of file diff --git a/2918deb778c2f4961cc75fcf66f21c6bbd1c6a848e0a5aebb2e3a9afdac25ce6.asc b/2918deb778c2f4961cc75fcf66f21c6bbd1c6a848e0a5aebb2e3a9afdac25ce6.asc @@ -0,0 +1,11 @@ +-----BEGIN PGP SIGNATURE----- + +iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmHkZ0UACgkQzcvSTdHQ +4AFIYQgAiAtYWkg5t6Uq6X09SfkBsQE58UdEnbIoppr2KN5cmOpDtqf8ZPvspRXh +xaiGPnkfIm6t0XjPWMm5hx5r+rCdnsW5x9W1ZeDWHdq39h7xCFqTif2vyD8scX4k +ib6czeGaCt/PAeDRMDYqwMLYUu0aebPQjeJES7Y2EDrIeNZVbflTYWBahG8uuhND +msy6ltPXR40L/cSJoEeBVY/X9oJbjZbESpvCYNr65EXzyEE3DTNSMyTdQfNaeiei +AdjbS4s3h127sLza18phyHlCeZA1P/+toKTjsNoFRVwxeiloVRZVz2Kc9verco6h +r6NRTAQ3pnTYqaHdqLfv749As/Ybmg== +=UmQu +-----END PGP SIGNATURE----- diff --git a/5f38d7d3ab58689e73a237c1e10bf588e66a50457b4c4fea08fe423bc8dbd460 b/5f38d7d3ab58689e73a237c1e10bf588e66a50457b4c4fea08fe423bc8dbd460 @@ -0,0 +1,291 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <link rel="stylesheet" type="text/css" href="./theme/css/style.css" /> + <link rel="stylesheet" type="text/css" href="./theme/css/syntax.css" /> + <title>man bytes gnu - The NFT token id as URI</title> + <meta charset="utf-8" /> + <meta name="generator" content="Pelican" /> + +</head> + +<body id="index" class="home"> + + <header id="banner" class="body"> + <h1>> <a href="./">man bytes gnu</a><span id="cursor">_</span></h1> + </header> + <nav id="menu"> + <ul> + <li><a href="./category/archiving.html">Archiving</a></li> + <li class="active"><a href="./category/code.html">Code</a></li> + <li><a href="./category/hygiene.html">Hygiene</a></li> + <li><a href="./category/offlining.html">Offlining</a></li> + <li><a href="./pages/identities.html">Identities</a></li> + <li><a href="./pages/shares.html">Shares</a></li> + + <li><a href="/tags.html">tags</a></li> + </ul> + <hr/> + </nav><!-- /#menu --> + <header> + <h1 class="entry-title top-body-title"> + <a href="./nft-tokenid-content.html" rel="bookmark" + title="Permalink to The NFT token id as URI">The NFT token id as URI</a></h1> + + + <div class="category meta"> + Posted +<time class="published" datetime="2021-05-08T19:14:00+02:00"> + Sat 08 May 2021 + </time> +in <a class="category" href="./category/code.html">code</a> + <a href="./tag/nft.html">nft</a> + <a href="./tag/evm.html">evm</a> + <a href="./tag/hash.html">hash</a> + <a href="./tag/key-value-store.html">key-value store</a> + <a href="./tag/decentralized-storage.html">decentralized storage</a> + </div> + + <div class="neighbors meta"> + <ul> + <li> + Previous: <a href="./wasm-c-bare.html"> + wasm and C - The bare necessities + </a> + </li> + | + + <li> + Next: <a href="./village-network-protocol.html"> + A village network protocol + </a> + </li> + </ul> + </div> + <div class="meta"> +<hr/> + </div> +</header> + +<section id="content" class="body"> + <div class="entry-content"> + <p>Let's consider an NFT that works like a badge for participating in development of a software project.</p> +<p>This token is awarded as a proof that the task was completed.</p> +<p>To make things more fun, each NFT should have some unique, immutable content attached to it.</p> +<p>In other words, the properties of this token, once set, should never change.</p> +<p>Nor should they disappear.</p> +<p>So how do we refer to the artwork asset within the token standard?</p> +<div class="section" id="it-was-acceptable-at-the-time"> +<h2>It was acceptable at the time</h2> +<p>The ERC721 standard is not explicit about where the assets that belong with the NFT can be discovered and resolved.</p> +<p>At the time when the standard was adopted by the Ethereum community, there were multiple <em>"[...] Alternatives considered: put all metadata for each asset on the blockchain (too expensive), use URL templates to query metadata parts (URL templates do not work with all URL schemes, especially P2P URLs), multiaddr network address (not mature enough)."</em> Furthermore, they <em>"[...] considered an NFT representing ownership of a house, in this case metadata about the house (image, occupants, etc.) can naturally change."</em> <a class="citation-reference" href="#eip721" id="id1">[EIP721]</a></p> +<p>A "changing house" doesn't sound quite like what we need. And anyway; if we stick a good old web2 URI in there, then that will end up on the great bonfire of dead links before long.</p> +</div> +<div class="section" id="image-schmimage"> +<h2>Image, schmimage</h2> +<p>To be honest, I find the presumption in the optional EIP721 metadata structure to be surprisingly short-sighted. It <em>specifically</em> defines the asset as an image, and at the same time is presupposes that only a <em>single</em> asset file will be used.</p> +<p>We may want to add <em>multiple</em> sources, so this is another obstacle for us.</p> +<p>So how to get around this, while still playing nice with existing implementations out there? Two ideas come to mind:</p> +<ul class="simple"> +<li>Embed a <em>thumbnail</em> as a preview of the artwork using a <code>base64</code> <em>data URI</em> <a class="footnote-reference" href="#id7" id="id2">[1]</a> in the metadata. Stick <code>name</code> and <code>description</code> on it, and the schema is still fulfilled.</li> +<li><em>Extend</em> the structure with a list of <em>attachments</em> that <em>our</em> application layer knows about. Of course, each of these can have the same format as above.</li> +</ul> +<p>In other words:</p> +<pre class="code json literal-block"> +<span class="p">{</span> + <span class="nt">"name"</span><span class="p">:</span> <span class="s2">"Foo"</span><span class="p">,</span> + <span class="nt">"description"</span><span class="p">:</span> <span class="s2">"Foo image"</span><span class="p">,</span> + <span class="nt">"image"</span><span class="p">:</span> <span class="nt">"data:image/gif;base64,R0lGODlhDgARAKEDAAAAAOjqAPP1APDw8CH+EUNyZWF0ZWQgd2l0aCBHSU1QACwAAAAADgARAAACO5wHqXdrClocodbUaGg34qoBHaJtl7VYIfqBZxepb5C6NBvbIlyucO6jPUIK1CMiSCYvDIVS4GhCJoYCADs="</span> + <span class="nt">"attachments"</span><span class="p">:</span> <span class="p">[</span> + <span class="p">{</span> + <span class="nt">"name"</span><span class="p">:</span> <span class="s2">"Bar"</span><span class="p">,</span> + <span class="nt">"description"</span><span class="p">:</span> <span class="s2">"Bar image"</span><span class="p">,</span> + <span class="nt">"image"</span><span class="p">:</span> <span class="s2">"data:image/gif;base64,"</span> + <span class="p">},</span> + <span class="p">{</span> + <span class="nt">"name"</span><span class="p">:</span> <span class="s2">"Baz"</span><span class="p">,</span> + <span class="nt">"description"</span><span class="p">:</span> <span class="s2">"Baz image"</span><span class="p">,</span> + <span class="nt">"image"</span><span class="p">:</span> <span class="s2">"data:image/gif;base64,"</span> + <span class="p">}</span> + <span class="p">]</span> +<span class="p">}</span> +</pre> +</div> +<div class="section" id="mirror-mirror"> +<h2>Mirror, mirror</h2> +<p>Since the asset reference shouldn't change, we can refer to it by its fingerprint or <a class="reference external" href="https://en.wikipedia.org/wiki/Content-addressable_storage">content address</a>. If we define that the resource can be looked up over HTTP by that fingerprint as its basename, then we are free to define and modify whatever list of mirrors for that resource that's valid for any point in time. The application layer would simply try the endpoints one after another.</p> +<p>We take the <code>sha2-256</code> <a class="footnote-reference" href="#id8" id="id3">[2]</a> of the asset reference (the json file above, free of evil whitespace and newlines):</p> +<div class="highlight"><pre><span></span>$ cat reference.json <span class="p">|</span> jq -c -j <span class="p">|</span> sha256sum <span class="p">|</span> awk <span class="s1">'{ print $1; }'</span> +3fdfbfe3b510b69f90cd92618e4c1ec76cf8b9c330bc2da1922acda8f84f9551 +</pre></div> +<p>Imagine we had a mirror list of <a class="reference external" href="https://foo.com">https://foo.com</a> and <a class="reference external" href="https://bar.com/baz/">https://bar.com/baz/</a>. Then our application would try these urls in sequence, stopping at the first that returns a valid result:</p> +<div class="highlight"><pre><span></span>https://foo.com/3fdfbfe3b510b69f90cd92618e4c1ec76cf8b9c330bc2da1922acda8f84f9551 +https://bar.com/baz/3fdfbfe3b510b69f90cd92618e4c1ec76cf8b9c330bc2da1922acda8f84f9551 +</pre></div> +<p>Once we receive the content, all we have to do is hash it ourselves and verify that the sum matches the basename of the URI. If it doesn't the result is of course not valid and we continue down the list, appropriately banning the mischievous server then throrougly harassing its admin.</p> +</div> +<div class="section" id="cast-away"> +<h2>Cast away</h2> +<p>Since our fingerprint is 32 bytes, it fits exactly inside the <code>tokenId</code> (<code>uint256</code>). Let's decide to big-endian numbers when converting (I find them easier to make sense of). In that case our hash from the reference turns into this modest number:</p> +<div class="highlight"><pre><span></span><span class="c1"># python3</span> +<span class="o">>>></span> <span class="n">hx</span> <span class="o">=</span> <span class="nb">bytes</span><span class="o">.</span><span class="n">fromhex</span><span class="p">(</span><span class="s1">'3fdfbfe3b510b69f90cd92618e4c1ec76cf8b9c330bc2da1922acda8f84f9551'</span><span class="p">)</span> +<span class="o">>>></span> <span class="nb">int</span><span class="o">.</span><span class="n">from_bytes</span><span class="p">(</span><span class="n">hx</span><span class="p">,</span> <span class="n">byteorder</span><span class="o">=</span><span class="s1">'big'</span><span class="p">)</span> +<span class="mi">28891040728719892888467057134569335350980764617882743994259054993630416573777</span> +</pre></div> +<p>As long as we're composing the <code>evm</code> inputs ourselves, we don't really have to worry about the integer representation in this particular case. But the interface is defined as an integer type, and other mortals may be using higher level interfaces, we have to be explicit about our choice.</p> +</div> +<div class="section" id="welcoming-mint"> +<h2>Welcoming mint</h2> +<p>Assume we have a method <code>mintTo(address _recipient, uint256 _tokenId)</code> on our NFT contract. The Solidity signature of that contract is <code>edb20b7e</code> <a class="footnote-reference" href="#id9" id="id4">[3]</a>. If I were to mint to myself then the input to the contract would be:</p> +<div class="highlight"><pre><span></span>edb20b7e000000000000000000000000185cbce7650ff7ad3b587e26b2877d95568805e33fdfbfe3b510b69f90cd92618e4c1ec76cf8b9c330bc2da1922acda8f84f9551 +</pre></div> +<p>Broken down:</p> +<div class="highlight"><pre><span></span>signature: edb20b7e +address, zero-padded: 000000000000000000000000185cbce7650ff7ad3b587e26b2877d95568805e3 +token id: 3fdfbfe3b510b69f90cd92618e4c1ec76cf8b9c330bc2da1922acda8f84f9551 +</pre></div> +<p>The corresponding web3.js code would look like:</p> +<div class="highlight"><pre><span></span><span class="kd">const</span> <span class="nx">c</span> <span class="o">=</span> <span class="ow">new</span> <span class="nx">web3</span><span class="p">.</span><span class="nx">eth</span><span class="p">.</span><span class="nx">Contract</span><span class="p">([...],</span> <span class="s1">'0x...'</span><span class="p">);</span> +<span class="nx">c</span><span class="p">.</span><span class="nx">methods</span><span class="p">.</span><span class="nx">mintTo</span><span class="p">(</span><span class="s1">'0x185cbce7650ff7ad3b587e26b2877d95568805e3'</span><span class="p">,</span> <span class="mf">28891040728719892888467057134569335350980764617882743994259054993630416573777</span><span class="p">).</span><span class="nx">call</span><span class="p">();</span> +</pre></div> +<p>To satisfy the <cite>tokenURI</cite> method, we can generate a string that's prefix with sha256 as a "scheme" <a class="footnote-reference" href="#id10" id="id5">[4]</a>. A bit of (unoptimized) solidity helps us out here:</p> +<pre class="code solidity literal-block"> +<span class="k">contract</span><span class="w"> </span><span class="ni">NFT</span><span class="w"> </span><span class="p">{</span><span class="w"> + + </span><span class="kt">uint256</span><span class="p">[]</span><span class="w"> </span>token<span class="p">;</span><span class="w"> + </span><span class="kt">mapping</span><span class="p">(</span><span class="kt">uint256</span><span class="w"> </span><span class="o">=></span><span class="w"> </span><span class="kt">uint256</span><span class="p">)</span><span class="w"> </span>tokenIndex<span class="p">;</span><span class="w"> </span><span class="c1">// map token id to master token array index position +</span><span class="w"> + </span><span class="p">[...]</span><span class="w"> + + </span><span class="kt">function</span><span class="w"> </span><span class="nv">tokenURI</span><span class="p">(</span><span class="kt">uint256</span><span class="w"> </span><span class="nv">_tokenId</span><span class="p">)</span><span class="w"> </span><span class="kt">public</span><span class="w"> </span>pure<span class="w"> </span><span class="kt">returns</span><span class="p">(</span><span class="kt">string</span><span class="w"> </span><span class="nv">memory</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"> + </span><span class="kt">bytes32</span><span class="w"> </span><span class="nv">token_bytes</span><span class="p">;</span><span class="w"> + </span><span class="kt">bytes</span><span class="w"> </span><span class="nv">memory</span><span class="w"> </span>out<span class="p">;</span><span class="w"> + </span><span class="kt">uint8</span><span class="w"> </span><span class="nv">t</span><span class="p">;</span><span class="w"> + </span><span class="kt">uint256</span><span class="w"> </span><span class="nv">c</span><span class="p">;</span><span class="w"> + + </span>token_bytes<span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="kt">bytes32</span><span class="p">(</span>token<span class="p">[</span>tokenIndex<span class="p">[</span>_tokenId<span class="p">]]);</span><span class="w"> + + </span>out<span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="kt">new</span><span class="w"> </span><span class="kt">bytes</span><span class="p">(</span><span class="m-Decimal">64</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="m-Decimal">7</span><span class="p">);</span><span class="w"> + </span>out<span class="p">[</span><span class="m-Decimal">0</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">"s"</span><span class="p">;</span><span class="w"> + </span>out<span class="p">[</span><span class="m-Decimal">1</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">"h"</span><span class="p">;</span><span class="w"> + </span>out<span class="p">[</span><span class="m-Decimal">2</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">"a"</span><span class="p">;</span><span class="w"> + </span>out<span class="p">[</span><span class="m-Decimal">3</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">"2"</span><span class="p">;</span><span class="w"> + </span>out<span class="p">[</span><span class="m-Decimal">4</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">"5"</span><span class="p">;</span><span class="w"> + </span>out<span class="p">[</span><span class="m-Decimal">5</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">"6"</span><span class="p">;</span><span class="w"> + </span>out<span class="p">[</span><span class="m-Decimal">6</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">":"</span><span class="p">;</span><span class="w"> + + </span>c<span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="m-Decimal">7</span><span class="p">;</span><span class="w"> + </span><span class="kt">for</span><span class="w"> </span><span class="p">(</span><span class="kt">uint256</span><span class="w"> </span><span class="nv">i</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="m-Decimal">0</span><span class="p">;</span><span class="w"> </span>i<span class="w"> </span><span class="o"><</span><span class="w"> </span><span class="m-Decimal">32</span><span class="p">;</span><span class="w"> </span>i<span class="o">++</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"> + </span>t<span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">(</span><span class="kt">uint8</span><span class="p">(</span>_data<span class="p">[</span>i<span class="p">])</span><span class="w"> </span><span class="err">&</span><span class="w"> </span><span class="mh">0xf0</span><span class="p">)</span><span class="w"> </span><span class="o">>></span><span class="w"> </span><span class="m-Decimal">4</span><span class="p">;</span><span class="w"> + </span><span class="kt">if</span><span class="w"> </span><span class="p">(</span>t<span class="w"> </span><span class="o"><</span><span class="w"> </span><span class="m-Decimal">10</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"> + </span>out<span class="p">[</span>c<span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span>bytes1<span class="p">(</span>t<span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mh">0x30</span><span class="p">);</span><span class="w"> + </span><span class="p">}</span><span class="w"> </span><span class="kt">else</span><span class="w"> </span><span class="p">{</span><span class="w"> + </span>out<span class="p">[</span>c<span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span>bytes1<span class="p">(</span>t<span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mh">0x57</span><span class="p">);</span><span class="w"> + </span><span class="p">}</span><span class="w"> + </span>t<span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="kt">uint8</span><span class="p">(</span>_data<span class="p">[</span>i<span class="p">])</span><span class="w"> </span><span class="err">&</span><span class="w"> </span><span class="mh">0x0f</span><span class="p">;</span><span class="w"> + </span><span class="kt">if</span><span class="w"> </span><span class="p">(</span>t<span class="w"> </span><span class="o"><</span><span class="w"> </span><span class="m-Decimal">10</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"> + </span>out<span class="p">[</span>c<span class="o">+</span><span class="m-Decimal">1</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span>bytes1<span class="p">(</span>t<span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mh">0x30</span><span class="p">);</span><span class="w"> + </span><span class="p">}</span><span class="w"> </span><span class="kt">else</span><span class="w"> </span><span class="p">{</span><span class="w"> + </span>out<span class="p">[</span>c<span class="o">+</span><span class="m-Decimal">1</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span>bytes1<span class="p">(</span>t<span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mh">0x57</span><span class="p">);</span><span class="w"> + </span><span class="p">}</span><span class="w"> + </span>c<span class="w"> </span><span class="o">+=</span><span class="w"> </span><span class="m-Decimal">2</span><span class="p">;</span><span class="w"> + </span><span class="p">}</span><span class="w"> + </span><span class="kt">return</span><span class="w"> </span><span class="kt">string</span><span class="p">(</span>out<span class="p">);</span><span class="w"> + </span><span class="p">}</span><span class="w"> +</span><span class="p">}</span> +</pre> +<p>This will return <code>sha256:3fdfbfe3b510b69f90cd92618e4c1ec76cf8b9c330bc2da1922acda8f84f9551</code> for <code>tokenId</code> <code>3fdfbfe3b510b69f90cd92618e4c1ec76cf8b9c330bc2da1922acda8f84f9551</code> as input, provided that the <code>tokenId</code> actually exists. That may seem a bit useless at first, but consider the scenario where we want to interface with other NFTs aswell. Or perhaps we are implementing a contract that optionally can support a static web2 URI in storage. By doing it this way, all bases are covered.</p> +</div> +<div class="section" id="decentralized-identifiers"> +<h2>Decentralized identifiers</h2> +<p>Even better would be to add redundancy with autonomous decentralized storage. However, networks like <a class="reference external" href="https://ethswarm.org">Swarm</a> and <a class="reference external" href="https://ipfs.io">IPFS</a> use their own hashing recipes. That means that for every network referenced, we'd have to define an <em>alternative</em> in our reference structure.</p> +<p>Referencing the canonical <code>sha256</code> aswell as the <code>Swarmhash</code> for the same item could then look like this <a class="footnote-reference" href="#id11" id="id6">[5]</a>:</p> +<pre class="code json literal-block"> +<span class="p">{</span> + <span class="nt">"name"</span><span class="p">:</span> <span class="s2">"Foo"</span><span class="p">,</span> + <span class="nt">"description"</span><span class="p">:</span> <span class="s2">"Foo image"</span><span class="p">,</span> + <span class="nt">"image"</span><span class="p">:</span> <span class="nt">"data:image/gif;base64,R0lGODlhDgARAKEDAAAAAOjqAPP1APDw8CH+EUNyZWF0ZWQgd2l0aCBHSU1QACwAAAAADgARAAACO5wHqXdrClocodbUaGg34qoBHaJtl7VYIfqBZxepb5C6NBvbIlyucO6jPUIK1CMiSCYvDIVS4GhCJoYCADs="</span> + <span class="nt">"alternatives"</span><span class="p">:</span> <span class="p">[</span> + <span class="p">{</span> + <span class="nt">"name"</span><span class="p">:</span> <span class="s2">"Foo"</span><span class="p">,</span> + <span class="nt">"description"</span><span class="p">:</span> <span class="s2">"Foo image"</span><span class="p">,</span> + <span class="nt">"image"</span><span class="p">:</span> <span class="s2">"bzz:4b9149ee4550f2d786f9ba6584b79a30ee14ae05ff6e84a0f7c7561a14e3b779"</span> + <span class="p">},</span> + <span class="p">{</span> + <span class="nt">"name"</span><span class="p">:</span> <span class="s2">"Foo"</span><span class="p">,</span> + <span class="nt">"description"</span><span class="p">:</span> <span class="s2">"Foo image"</span><span class="p">,</span> + <span class="nt">"image"</span><span class="p">:</span> <span class="s2">"sha256:d036a4ce7f929b632256225b2bebd81bdd558d3bfe3d96faae61db664708c16f"</span> + <span class="p">}</span> + <span class="p">]</span> +<span class="p">}</span> +</pre> +<hr class="docutils" /> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id7" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id2">[1]</a></td><td>Yes, they are valid URIs actually: <a class="reference external" href="https://www.rfc-archive.org/getrfc.php?rfc=2397">https://www.rfc-archive.org/getrfc.php?rfc=2397</a></td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id8" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id3">[2]</a></td><td>Likely it would be prudent to start using the official <code>sha3</code> instead of <code>sha2</code> these days, also because the <code>sha2</code> hash is not a builtin for <code>evm</code>. But neither is <code>sha3</code>. The <code>keccak256</code> Bitcoin uses, which EVM has inherited, is a pre-cursor to the <code>keccak</code> published as the <em>official</em> <code>sha3</code>. Still, <code>keccak256</code> and <code>sha3</code> is used interchangeably in opcode lists (and previously in <a class="reference external" href="https://docs.soliditylang.org/en/v0.8.0/050-breaking-changes.html#functions">Solidity</a> too). This has caused me quite a fair bit of confusion, I might add. Apart from it being ambiguous, the <code>keccak256</code> tooling is also less common in the wild. Therefore <code>sha2</code> seems like a safer bet for our experiments. It's not broken yet, after all.</td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id9" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id4">[3]</a></td><td>The hex result of <code>keccak256("mintTo(address,uint256)")</code></td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id10" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id5">[4]</a></td><td><em>Data URI</em> is of no use here, because the hash itself is just nondescript binary data. Luckily <code><scheme>:<path></code> is still a valid URI.</td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id11" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id6">[5]</a></td><td>Here the hashes represent the media content itself, not the reference. That's why the <code>sha256</code> one is different than before.</td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils citation" frame="void" id="eip721" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id1">[EIP721]</a></td><td><a class="reference external" href="https://eips.ethereum.org/EIPS/eip-721">https://eips.ethereum.org/EIPS/eip-721</a></td></tr> +</tbody> +</table> +</blockquote> +</div> + + </div><!-- /.entry-content --> + <footer class="meta"> + </footer> + +</section> + <footer id="contentinfo" class="body"> + <hr/> + <address id="about" class="vcard body"> + Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> + </address><!-- /#about --> + </footer><!-- /#contentinfo --> +</body> +</html> +\ No newline at end of file diff --git a/5f38d7d3ab58689e73a237c1e10bf588e66a50457b4c4fea08fe423bc8dbd460.asc b/5f38d7d3ab58689e73a237c1e10bf588e66a50457b4c4fea08fe423bc8dbd460.asc @@ -0,0 +1,11 @@ +-----BEGIN PGP SIGNATURE----- + +iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmHkZ0UACgkQzcvSTdHQ +4AGo8wf+Oe9vfl8BEv36O4lGLFcoX5s2OyBjN/PwTbS/YJga232YqX7x/mAEm5Ul +C7rUOyrtx6RaF4lqDDs2HiWMQX7QVspH8pLYrRfWLKq9/fYeKihM44V91/vj/X/t +1yClp0tEmuBx1g1lpjih1SxOgOjslwf1nOXgeJjlmP7nNxNQQZTBHJFhtEAal2MW +2Ryj1MvMOLbxKcNi4uzUvpbrOmPuZwo9X+x3gOryp4Vrz16yW2iMVJICE44TDetw +CJ6j+udujyTk3faGD/y8u7fMOtCl8olTUHEgGhNPMbJGdHlx/dWJzYoDhGoL2ToL +5wv5WgQVUNNRO/ZerxlmuvheLvhGMA== +=ZBzC +-----END PGP SIGNATURE----- diff --git a/6cd7d22162d74eba04b00bdc004e18907c32ba245e0bf210ccb26e431789b18b b/6cd7d22162d74eba04b00bdc004e18907c32ba245e0bf210ccb26e431789b18b @@ -0,0 +1,328 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <link rel="stylesheet" type="text/css" href="./theme/css/style.css" /> + <link rel="stylesheet" type="text/css" href="./theme/css/syntax.css" /> + <title>man bytes gnu - A village network protocol</title> + <meta charset="utf-8" /> + <meta name="generator" content="Pelican" /> + +</head> + +<body id="index" class="home"> + + <header id="banner" class="body"> + <h1>> <a href="./">man bytes gnu</a><span id="cursor">_</span></h1> + </header> + <nav id="menu"> + <ul> + <li><a href="./category/archiving.html">Archiving</a></li> + <li><a href="./category/code.html">Code</a></li> + <li class="active"><a href="./category/hygiene.html">Hygiene</a></li> + <li><a href="./category/offlining.html">Offlining</a></li> + <li><a href="./pages/identities.html">Identities</a></li> + <li><a href="./pages/shares.html">Shares</a></li> + + <li><a href="/tags.html">tags</a></li> + </ul> + <hr/> + </nav><!-- /#menu --> + <header> + <h1 class="entry-title top-body-title"> + <a href="./village-network-protocol.html" rel="bookmark" + title="Permalink to A village network protocol">A village network protocol</a></h1> + + + <div class="category meta"> + Posted +<time class="published" datetime="2021-05-19T10:40:00+02:00"> + Wed 19 May 2021 + </time> +in <a class="category" href="./category/hygiene.html">hygiene</a> + <a href="./tag/p2p.html">p2p</a> + <a href="./tag/protocol.html">protocol</a> + <a href="./tag/network.html">network</a> + <a href="./tag/crm.html">crm</a> + <a href="./tag/privacy.html">privacy</a> + <a href="./tag/f2f.html">f2f</a> + <a href="./tag/community.html">community</a> + </div> + + <div class="neighbors meta"> + <ul> + <li> + Previous: <a href="./nft-tokenid-content.html"> + The NFT token id as URI + </a> + </li> + | + + <li> + Next: <a href="./docker-offline-3-npm.html"> + Local npm repository + </a> + </li> + </ul> + </div> + <div class="meta"> +<hr/> + </div> +</header> + +<section id="content" class="body"> + <div class="entry-content"> + <p>What would truly peer-to-peer CRM backend look like? And at that, once that respected the right of the "customer" to choose how it wants to represent itself in the context?</p> +<p>Perhaps by Modeling the network on how villages treat strangers and their own, and how you would go about interacting with them.</p> +<p>Here's an initial stab at describing what such a meta-protocol could look like.</p> +<!-- There is no truth. There is only perspective and predictability. --> +<div class="section" id="terms"> +<h2>Terms</h2> +<dl class="docutils"> +<dt>Individual</dt> +<dd>A subjective structure where the thing being described is the thing that described it.</dd> +<dt>User</dt> +<dd>A particular perception of an individual by an entity in the network.</dd> +<dt>Userspec</dt> +<dd>Data structure containing details about a user. May or may not be public.</dd> +<dt>Locals</dt> +<dd>A group of individuals acknowledging each other as part of a group, and who share views on users.</dd> +<dt>Local</dt> +<dd>A member of a group of locals, from the shared perspective of an individual and another local.</dd> +<dt>Localslist</dt> +<dd>The individuals making up a group of locals, from the perspective of a local.</dd> +<dt>Stranger</dt> +<dd>A user as percieved by a group of locals.</dd> +<dt>Strangerlist</dt> +<dd>All users that may speak to the locals, from the perspective of a local. To be shared among locals.</dd> +<dt>Strangerspec</dt> +<dd>Userspec created by a local describing a user. To be shared among locals.</dd> +<dt>Entity</dt> +<dd>An individual or any derived representation of it (user, stranger, local).</dd> +<dt>Peer</dt> +<dd>An entity coupled with connection information</dd> +<dt>Content address</dt> +<dd>SHA256 sum of arbitrary content</dd> +<dt>Key setup</dt> +<dd>May be an individual key, or a specification describing how to generate keys for a particular relation.</dd> +</dl> +</div> +<div class="section" id="roles"> +<h2>Roles</h2> +<div class="section" id="the-individuals"> +<h3>The individuals</h3> +<p>Are free to have their own opinion of what they are.</p> +<p>When interacting with others, they are free to choose what they say about what they are.</p> +<p>An individual initially <em>defines itself</em> in the network.</p> +</div> +<div class="section" id="the-users"> +<h3>The users</h3> +<p>A user is a view of an individual as defined by one or more other individuals.</p> +<p>It may be how an individual presents itself to an particular individual or a group.</p> +<p>It may also be how an individual or a group presents another individual to itself.</p> +</div> +<div class="section" id="the-locals"> +<h3>The locals</h3> +<p>Locals are a group of individuals who wish to share a view of themselves and prejudice about others.</p> +<p>An individual will not be treated as a local unless it and its local counterpart considers it a local.</p> +<p>Locals may acknowledge strangers with or without proof.</p> +<p>If one local changes opinion about a stranger, all locals should change their opinon to the same.</p> +<p>If one local excludes a stranger, all locals should exclude the stranger.</p> +<p>You can't keep secrets in a village. If someone says something to another local, it must assume all locals will get to know what was said.</p> +</div> +<div class="section" id="the-strangers"> +<h3>The strangers</h3> +<p>A stranger is a user that's not considered local by a group of locals.</p> +<p>A stranger may not speak to a local until the local has decided it can.</p> +<p>If a stranger may speak to one local, it may speak all locals.</p> +</div> +</div> +<div class="section" id="identifiers"> +<h2>Identifiers</h2> +<p>The <code>individual_id</code> is the identifier of an individual consists of the content address of some basic public data about the entity registering itself. The <code>individual_id</code> is immutable.</p> +<p>Any local will refer to another local by its <code>individual_id</code>.</p> +<p>A <code>user_id</code> is the content address of an <code>individual_id</code> together with the content address of some basic data describing the perception of the individual.</p> +<p>Locals describe stranger by a pair of <code>individual_id</code> and <code>user_id</code>. Let us call this <code>stranger_id</code>. The <code>stranger_id</code> is immutable.</p> +<p>The data represented by the <code>user_id</code> in a <code>stranger_id</code> is visible to locals, and may or may not be public.</p> +</div> +<div class="section" id="state-description"> +<h2>State description</h2> +<p>A user keeps a state towards other entities consisting of the following content-addresses, in the given order:</p> +<ol class="arabic simple"> +<li>Previous state</li> +<li>Userspec</li> +</ol> +<p>A local keeps a state towards its locals consisting of the following content-addresses, in the given order.</p> +<ol class="arabic simple"> +<li>Previous state</li> +<li>Userspec</li> +<li>Localslist</li> +<li>Locals state</li> +<li>Strangerslist</li> +<li>Strangers state</li> +</ol> +<p>The state is represented by the content address of the concatenation of these content-addresses, in the given order.</p> +<p>Any state change will be signed by the key resolvable through the <code>individual_id</code>.</p> +<div class="section" id="previous-state"> +<h3>Previous state</h3> +<p>The content address of the previous state. Will be <code>0</code> in the first state.</p> +</div> +<div class="section" id="userspec"> +<h3>Userspec</h3> +<p>The individual as it wants to be perceived in the current context. There is no guarantee that the individual will be viewed in this way by others.</p> +</div> +<div class="section" id="localslist"> +<h3>Localslist</h3> +<p>A lexiographically ordered array of <code>individual_id</code> that define the individuals in a group of locals.</p> +<p>All locals must keep this list in the same state.</p> +</div> +<div class="section" id="locals-state"> +<h3>Locals state</h3> +<p>A local's last recorded state of each of the locals, in the same order as <code>Localslist</code>.</p> +<p>The recorded state is the same content address as in the state description section above.</p> +<p>All locals must strive to keep this in the same state.</p> +</div> +<div class="section" id="strangerslist"> +<h3>Strangerslist</h3> +<p>A lexiographically ordered pointer array representing a local's opinion of which users that can communicate with locals. <a class="footnote-reference" href="#id7" id="id1">[1]</a></p> +<p>Combination of <code>stranger_id</code> and a <em>vote</em> on the specific user.</p> +<p>The vote can be in one of three states:</p> +<ul class="simple"> +<li>Value <code>0</code>, meaning local has acknowledged the user, explicitly or by following other locals. We call this state "deferred."</li> +<li>Value <code>-1</code>, meaning a local has vetoed the user. We call this state "vetoed."</li> +<li>Any other value, which is the content-address of the proof accepted by the local. <a class="footnote-reference" href="#id8" id="id2">[2]</a> We call this state "accepted."</li> +</ul> +<p>Typically, the initial vote value is <code>0</code>. The value may change at any time, but may never change back to <code>0</code> once set to another value.</p> +<p>A user that is part of the list, but not vetoed, is referred to as "acknowledged."</p> +<p>The Strangerslist state may be differ between locals, but they should strive to keep it the same.</p> +</div> +<div class="section" id="strangers-state"> +<h3>Strangers state</h3> +<p>The local's version of the current user state of the strangers, in the same order as the <code>Strangerlist</code>.</p> +<p>The Strangers state may differ between locals, but they should strive to keep it the same.</p> +</div> +</div> +<div class="section" id="behavior-of-locals"> +<h2>Behavior of locals</h2> +<p>A local may announce itself as <em>active</em> or <em>passive</em>. This should be part of the <code>Userspec</code> in the locals context.</p> +<p>Specifically, this affects the way it "votes" on strangers. A passive local:</p> +<ul class="simple"> +<li>Will always <em>veto</em> a user if <em>at least one active local</em> has vetoed it.</li> +<li>Will always <em>accept</em> a user if <em>all active locals</em> have accepted it. <a class="footnote-reference" href="#id9" id="id3">[3]</a></li> +</ul> +<p>Typically, a passive local will be a bot, for example an always-online node that guarantees availability of the locals so users can request state changes.</p> +<p>There must be at least two active locals in any group of locals.</p> +<div class="section" id="veto-dissent"> +<h3>Veto dissent</h3> +<p>If a local vetoes a stranger, then all locals should cut communication with that stranger.</p> +<p>Vetoes should be adopted by all locals.</p> +<p>If, however, other locals consider the veto to be issued in error, there are two possible outcomes:</p> +<ol class="arabic simple"> +<li>The vetoing local(s) abandon(s) the veto. <a class="footnote-reference" href="#id10" id="id4">[4]</a></li> +<li>The vetoing individual(s) is/are excluded by the locals.</li> +</ol> +</div> +</div> +<div class="section" id="network"> +<h2>Network</h2> +<p>Any entity needs to define and announce itself as a <code>peer</code> in order to be available on the network.</p> +<p>A "peer" is the coupling of network connection information to an <code>individual_id</code>.</p> +<div class="section" id="discoverability"> +<h3>Discoverability</h3> +<p>Peers are kept in a <a class="reference external" href="https://en.wikipedia.org/wiki/Distributed_hash_table">DHT</a>.</p> +<p>Peers must be discoverable by their locals affiliation. <a class="footnote-reference" href="#id11" id="id5">[5]</a></p> +</div> +<div class="section" id="users"> +<h3>Users</h3> +<p>Users open single-request connections to other users to annonunce, look up and prune peers.</p> +<p>Users may request state changes for themselves, or relay such requests for other users.</p> +</div> +<div class="section" id="locals"> +<h3>Locals</h3> +<p>Locals keep open connections (streams) with each other.</p> +<p>Locals continually exchange state changes.</p> +<p>Locals should be able to audit the chain of state changes of other locals.</p> +</div> +<div class="section" id="between-locals-and-users"> +<h3>Between locals and users</h3> +<p>Strangers <em>acknowledged</em> by locals open single-request connections to a local to announce state changes.</p> +<p>Locals may open single-request connections to a user to announce a change to the state of the <code>Userspec</code> of a stranger. <a class="footnote-reference" href="#id12" id="id6">[6]</a></p> +<p>Individuals are free to deny connections from non-locals, for example if the entity has been sending garbage requests. The implementation of such sanctions is deferred to the application layer.</p> +</div> +</div> +<div class="section" id="encryption"> +<h2>Encryption</h2> +<p>An individual may keep content totally private on the network.</p> +<p>A user may keep different encryption keys for different contexts.</p> +<p>If a stranger shares a key setup with a local, that setup will be shared among all locals. A local will expect to be able to communicate with a strangerr using this key setup.</p> +<p>The stranger must be able to change the key setup towards locals.</p> +<p>The locals may be able to change the key setup towards a stranger.</p> +<p>The protocol should allow anything from simple symmetric key exchanges to complex forward/backward secrecy schemes.</p> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id7" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id1">[1]</a></td><td>Insert in this sorted list will face scaling issues when locals need to acknowledge a lot of users. The lists can be split into <a class="reference external" href="https://en.wikipedia.org/wiki/Bucket_%28computing%29">buckets</a>, either statically (for example two-byte prefixes) or dynamically after crossing a certain threshold. It should not be necessary to store empty buckets.</td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id8" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id2">[2]</a></td><td>This proof may map to what's commonly known as <em>kyc information</em>.</td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id9" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id3">[3]</a></td><td>Perhaps with the further limitation that all active users must have accepted with the <em>same</em> proof. Otherwise, how will the passive local choose which proof to accept.</td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id10" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id4">[4]</a></td><td>Observe that since the vote state cannot change back to <code>0</code>, "cancelling" the veto can only be done by adding proof.</td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id11" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id5">[5]</a></td><td>What exactly uniquely identifies a group of locals on the network in such a way that it cannot be hi-jacked or spoofed still needs to be thought through. Perhaps it can be a combination of a specific name with a signature from the "founders;" the first (two or more) locals in it.</td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id12" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id6">[6]</a></td><td>A use-case here would be where the locals make a change for the user due to an out-of-band request from the user. This is <em>not</em> the <code>Strangerspec</code>; it is the user's current <code>Userspec</code> for the given context. Of course, it's up to the user to accept the change. It may in turn request a new state change to the locals to revert it.</td></tr> +</tbody> +</table> +</blockquote> +</div> + + </div><!-- /.entry-content --> + <footer class="meta"> + </footer> + +</section> + <footer id="contentinfo" class="body"> + <hr/> + <address id="about" class="vcard body"> + Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> + </address><!-- /#about --> + </footer><!-- /#contentinfo --> +</body> +</html> +\ No newline at end of file diff --git a/6cd7d22162d74eba04b00bdc004e18907c32ba245e0bf210ccb26e431789b18b.asc b/6cd7d22162d74eba04b00bdc004e18907c32ba245e0bf210ccb26e431789b18b.asc @@ -0,0 +1,11 @@ +-----BEGIN PGP SIGNATURE----- + +iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmHkZ0UACgkQzcvSTdHQ +4AHHzAf/WsYO1rqEMJwV+R2WmMd+ofNiRWVcUIzwzN544JRNn4tySyh7P6gX72u3 +6FDjt3SqMVdAc8Lg1oxGIIuQMVmUhNZl1tEZjJs2cog2n2waFxL0bJsr09bXWjwS +2Gt538pfZZ8quUckO4Y5Mzk3F3H6MHyoRgxnjuMuBr2tmv2euaqORl8MawL5T3o8 +mDMP0XxNfiqqZiVkGSRigxpicYUHqX42QsnuZoTditAAE1D/GQh6BiT36Ji9miLb +D1C2gFCorThSbs+syhzOODtb8M2M7IxzTwa4RLlkfTU4wvdn5l609UrJwS3HcQWD +6/qKhtWny8Mm01w+Lxc/eHT0aic4yQ== +=Zgwg +-----END PGP SIGNATURE----- diff --git a/a5006a0269c2c925630e27977d97e59941f38432eae7b684a8342b92d655ec31 b/a5006a0269c2c925630e27977d97e59941f38432eae7b684a8342b92d655ec31 @@ -0,0 +1,224 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <link rel="stylesheet" type="text/css" href="./theme/css/style.css" /> + <link rel="stylesheet" type="text/css" href="./theme/css/syntax.css" /> + <title>man bytes gnu - Making sense of Ethereum log bloom filters</title> + <meta charset="utf-8" /> + <meta name="generator" content="Pelican" /> + +</head> + +<body id="index" class="home"> + + <header id="banner" class="body"> + <h1>> <a href="./">man bytes gnu</a><span id="cursor">_</span></h1> + </header> + <nav id="menu"> + <ul> + <li><a href="./category/archiving.html">Archiving</a></li> + <li class="active"><a href="./category/code.html">Code</a></li> + <li><a href="./category/hygiene.html">Hygiene</a></li> + <li><a href="./category/offlining.html">Offlining</a></li> + <li><a href="./pages/identities.html">Identities</a></li> + <li><a href="./pages/shares.html">Shares</a></li> + + <li><a href="/tags.html">tags</a></li> + </ul> + <hr/> + </nav><!-- /#menu --> + <header> + <h1 class="entry-title top-body-title"> + <a href="./eth-log-bloom.html" rel="bookmark" + title="Permalink to Making sense of Ethereum log bloom filters">Making sense of Ethereum log bloom filters</a></h1> + + + <div class="category meta"> + Posted +<time class="published" datetime="2021-06-27T11:17:00+02:00"> + Sun 27 June 2021 + </time> +in <a class="category" href="./category/code.html">code</a> + <a href="./tag/crypto.html">crypto</a> + <a href="./tag/ethereum.html">ethereum</a> + <a href="./tag/bloom-filter.html">bloom filter</a> + </div> + + <div class="neighbors meta"> + <ul> + <li> + Previous: <a href="./docker-offline-3-npm.html"> + Local npm repository + </a> + </li> + | + + <li> + Next: <a href="./internet-up-monitor.html"> + Homemade internet state monitor + </a> + </li> + </ul> + </div> + <div class="meta"> +<hr/> + </div> +</header> + +<section id="content" class="body"> + <div class="entry-content"> + <p>Ethereum blocks and transaction receipts both carry a bloom filter. They provide a shortcut to check whether a particular event of a particular contract was triggered. The filter in the transaction indexes all events that occurred in the transaction. The filter in the block carries all the filter bits of the individual transactions within the block.</p> +<p>Somewhat surprisingly, there are practically no friendly descriptions out there that tell you how to generate filters to match them with. In fact, the <a class="reference external" href="https://holbrook.no/doc/ethereum/yellowpaper.pdf">Ethereum Yellow Paper</a> was the only source I found.</p> +<p><strong>So let's make a friendly description</strong></p> +<p>Before we get to the friendliness, though, an ever so slight dose of brain melt is due:</p> +<div class="section" id="the-formalities"> +<h2>The formalities</h2> +<p>The formal definition of the filter calculation in the <a class="reference external" href="https://holbrook.no/doc/ethereum/yellowpaper.pdf">Ethereum Yellow Paper section 4.3.1</a> is (here cited in painfully inadequate math formatting):</p> +<blockquote> +<div class="formula"> +<i>M</i>(<i>O</i>) ≡ ∨<i>x</i> ∈ {<i>Oa</i>}∪<i>Ot</i>(<i>M</i><sub>3 : 2048</sub>(<i>x</i>)) +</div> +</blockquote> +<p>Where <span class="formula"><i>Oa</i></span> is the contract address and <span class="formula"><i>Ot</i></span> is all of the topics.</p> +<p>In the context of <em>Solidity</em>, "all" of the topics means the actual event signature, along with all the <em>indexed</em> parameters to the event.</p> +<p>The <span class="formula"><i>M</i></span> is the filter bit generator. The filter 2048 bits long, and for each entry that is added, 3 bits need to be set.</p> +<p>Reading further, we learn that the bits to set are the "first three" pairs of bytes of the keccak256 hash <a class="footnote-reference" href="#id4" id="id1">[1]</a>, modulo the length of the filter - 2048.</p> +<p>In the context of hashes, I usually assume "first" means from the left. That happens to be the case here, too. In other words, the "first" bytes are actually the bytes of most significance in big-endian order.</p> +<p>What's less clear, however, is which bits to set in the filter. The paper says:</p> +<blockquote> +<span class="formula">ℬ</span> is the bit reference function such that <span class="formula">ℬ<sub><i>j</i></sub>(<i>x</i>)</span> equals the bit of index <span class="formula"><i>j</i></span> (indexed from 0) in the byte array <span class="formula"><i>x</i></span>.</blockquote> +<p>Turns out after some trial and error that "index 0" means the <em>right</em> end of the byte array. So when setting the bits, we have to count from the right side.</p> +</div> +<div class="section" id="the-friendly-recipe"> +<h2>The friendly recipe</h2> +<p>Now let's try to bring this down to a mortal level:</p> +<ol class="arabic simple"> +<li>Take the keccak256 hash of the contract address</li> +<li>Take the keccak256 hash of the first topic value</li> +<li>Optionally, take the keccak256 hash of the remaining topic values (the indexed parameters of the event) <a class="footnote-reference" href="#id5" id="id2">[2]</a></li> +<li>Instantiate a byte array with length 256</li> +<li>Now, <em>for each</em> of the hashes:<ol class="arabic"> +<li>Get the <em>big-endian</em> integer of the <em>first two</em> bytes of the hash</li> +<li>Calculate the 2048-modulo of that integer <a class="footnote-reference" href="#id6" id="id3">[3]</a></li> +<li><em>Bitwise or</em> the filter bit at the index corresponding to the modulated value, counting from <em>right to left</em> (i.e. the value <span class="formula">0</span> corresponds to the <em>rightmost bit</em>).</li> +<li>Repeat 2 and 3 for the two next pairs of bytes of the hash.</li> +</ol> +</li> +</ol> +</div> +<div class="section" id="the-code"> +<h2>The code</h2> +<p>All that taken into account, we can take a stab at implementing a filter generator:</p> +<pre class="code python literal-block"> +<span class="ln"> 0 </span><span class="c1"># external imports</span> +<span class="ln"> 1 </span><span class="kn">import</span> <span class="nn">sha3</span> +<span class="ln"> 2 </span> +<span class="ln"> 3 </span> +<span class="ln"> 4 </span><span class="k">class</span> <span class="nc">LogBloom</span><span class="p">:</span> +<span class="ln"> 5 </span> +<span class="ln"> 6 </span> <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> +<span class="ln"> 7 </span> <span class="bp">self</span><span class="o">.</span><span class="n">content</span> <span class="o">=</span> <span class="nb">bytearray</span><span class="p">(</span><span class="mi">256</span><span class="p">)</span> +<span class="ln"> 8 </span> +<span class="ln"> 9 </span> +<span class="ln">10 </span> <span class="k">def</span> <span class="nf">add</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">element</span><span class="p">):</span> +<span class="ln">11 </span> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">element</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">):</span> +<span class="ln">12 </span> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'element must be bytes'</span><span class="p">)</span> +<span class="ln">13 </span> <span class="n">h</span> <span class="o">=</span> <span class="n">sha3</span><span class="o">.</span><span class="n">keccak_256</span><span class="p">()</span> +<span class="ln">14 </span> <span class="n">h</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">element</span><span class="p">)</span> +<span class="ln">15 </span> <span class="n">z</span> <span class="o">=</span> <span class="n">h</span><span class="o">.</span><span class="n">digest</span><span class="p">()</span> +<span class="ln">16 </span> +<span class="ln">17 </span> <span class="k">for</span> <span class="n">j</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">3</span><span class="p">):</span> +<span class="ln">18 </span> <span class="n">c</span> <span class="o">=</span> <span class="n">j</span> <span class="o">*</span> <span class="mi">2</span> +<span class="ln">19 </span> <span class="n">v</span> <span class="o">=</span> <span class="nb">int</span><span class="o">.</span><span class="n">from_bytes</span><span class="p">(</span><span class="n">z</span><span class="p">[</span><span class="n">c</span><span class="p">:</span><span class="n">c</span><span class="o">+</span><span class="mi">2</span><span class="p">],</span> <span class="n">byteorder</span><span class="o">=</span><span class="s1">'big'</span><span class="p">)</span> +<span class="ln">20 </span> <span class="n">v</span> <span class="o">&=</span> <span class="mh">0x07ff</span> +<span class="ln">21 </span> <span class="n">m</span> <span class="o">=</span> <span class="mi">255</span> <span class="o">-</span> <span class="nb">int</span><span class="p">(</span><span class="n">v</span> <span class="o">/</span> <span class="mi">8</span><span class="p">)</span> +<span class="ln">22 </span> <span class="n">n</span> <span class="o">=</span> <span class="n">v</span> <span class="o">%</span> <span class="mi">8</span> +<span class="ln">23 </span> <span class="bp">self</span><span class="o">.</span><span class="n">content</span><span class="p">[</span><span class="n">m</span><span class="p">]</span> <span class="o">|=</span> <span class="p">(</span><span class="mi">1</span> <span class="o"><<</span> <span class="n">n</span><span class="p">)</span> +</pre> +<p>Let's say we'd want to check if an Solidity-defined event <code>Foo(uint256,bytes32)</code> emitted by a contract at address <code>0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee</code> exists in a filter.</p> +<p>Using the class above, we would have to do something like this:</p> +<pre class="code python literal-block"> +<span class="c1"># external imports</span> +<span class="kn">import</span> <span class="nn">sha3</span> +<span class="kn">import</span> <span class="nn">some_block_getter</span> + +<span class="c1"># local imports</span> +<span class="kn">from</span> <span class="nn">the_above</span> <span class="kn">import</span> <span class="n">LogBloom</span> + +<span class="c1"># all set bits in our filter must be set in theirs</span> +<span class="k">def</span> <span class="nf">filter_match</span><span class="p">(</span><span class="n">theirs</span><span class="p">,</span> <span class="n">ours</span><span class="p">):</span> + <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">ours</span><span class="p">)):</span> + <span class="k">if</span> <span class="n">ours</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">&</span> <span class="n">theirs</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">!=</span> <span class="n">ours</span><span class="p">[</span><span class="n">i</span><span class="p">]:</span> + <span class="k">return</span> <span class="kc">False</span> + <span class="k">return</span> <span class="kc">True</span> + +<span class="n">fltr</span> <span class="o">=</span> <span class="n">LogBloom</span><span class="p">()</span> + +<span class="c1"># add the contract address to the filter</span> +<span class="n">address</span> <span class="o">=</span> <span class="nb">bytes</span><span class="o">.</span><span class="n">fromhex</span><span class="p">(</span><span class="s1">'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee'</span><span class="p">)</span> +<span class="n">h</span> <span class="o">=</span> <span class="n">sha3</span><span class="o">.</span><span class="n">keccak_256</span><span class="p">()</span> +<span class="n">h</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">address</span><span class="p">)</span> +<span class="n">address_element</span> <span class="o">=</span> <span class="n">h</span><span class="o">.</span><span class="n">digest</span><span class="p">()</span> +<span class="n">fltr</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">address_element</span><span class="p">)</span> + +<span class="c1"># create the topic signature for the event</span> +<span class="n">h</span> <span class="o">=</span> <span class="n">sha3</span><span class="o">.</span><span class="n">keccak_256</span><span class="p">()</span> +<span class="n">h</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="sa">b</span><span class="s1">'Foo(uint256,bytes32)'</span><span class="p">)</span> +<span class="n">topic</span> <span class="o">=</span> <span class="n">h</span><span class="o">.</span><span class="n">digest</span><span class="p">()</span> + +<span class="c1"># add the topic to the filter</span> +<span class="n">h</span> <span class="o">=</span> <span class="n">sha3</span><span class="o">.</span><span class="n">keccak_256</span><span class="p">()</span> +<span class="n">h</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">topic</span><span class="p">)</span> +<span class="n">topic_element</span> <span class="o">=</span> <span class="n">h</span><span class="o">.</span><span class="n">digest</span><span class="p">()</span> +<span class="n">fltr</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">topic_element</span><span class="p">)</span> + +<span class="c1"># get the filter from a block</span> +<span class="n">block</span> <span class="o">=</span> <span class="n">some_block_getter</span><span class="p">()</span> +<span class="n">block_bloom</span> <span class="o">=</span> <span class="nb">bytes</span><span class="o">.</span><span class="n">fromhex</span><span class="p">(</span><span class="n">block</span><span class="p">[</span><span class="s1">'logsBloom'</span><span class="p">][</span><span class="mi">2</span><span class="p">:])</span> <span class="c1"># assumes there is a 0x-prefix</span> + +<span class="c1"># check if it matches</span> +<span class="n">match</span> <span class="o">=</span> <span class="n">filter_match</span><span class="p">(</span><span class="n">block_bloom</span><span class="p">,</span> <span class="n">fltr</span><span class="o">.</span><span class="n">contents</span><span class="p">)</span> +<span class="nb">print</span><span class="p">(</span><span class="n">match</span><span class="p">)</span> +</pre> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id4" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id1">[1]</a></td><td>Remember the keccak hash used in Ethereum/Bitcoin is _not_ the official <cite>sha3</cite> hash. The padding parameter is different. More on that <a class="reference external" href="./nft-tokenid-content.html#id8">here</a></td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id5" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id2">[2]</a></td><td>Solidity supports up to 3 indexed parameters, which maps to the <em>evm</em> opcodes <code>LOGn</code> where <span class="formula"><i>n</i> ∈ {1, 2, 3, 4}</span></td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id6" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id3">[3]</a></td><td>Since <span class="formula">2<sup>11</sup> = 2048</span>, this is the same as zeroing out all bits above bit number 11; <span class="formula"><i>x</i>∧2048</span>. In other words, the result is an <em>11 bit integer</em>.</td></tr> +</tbody> +</table> +</blockquote> +</div> + + </div><!-- /.entry-content --> + <footer class="meta"> + </footer> + +</section> + <footer id="contentinfo" class="body"> + <hr/> + <address id="about" class="vcard body"> + Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> + </address><!-- /#about --> + </footer><!-- /#contentinfo --> +</body> +</html> +\ No newline at end of file diff --git a/a5006a0269c2c925630e27977d97e59941f38432eae7b684a8342b92d655ec31.asc b/a5006a0269c2c925630e27977d97e59941f38432eae7b684a8342b92d655ec31.asc @@ -0,0 +1,11 @@ +-----BEGIN PGP SIGNATURE----- + +iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmHkZ0UACgkQzcvSTdHQ +4AF9BQf/ZHncwvTWYwi6gGdwsVOB2waJDoaq/w/KWTNzqoRf5HSYINZuqhNar3Zs +CNLR1vwx9qxWRLK13+NFMb7qU4DUnvm0EmaYWT8XDQSwzHY/LKzquTOXjROO/9q3 +d7lAxftFiPBM8Mbte1seTfsdTrPZbRcX3aD+bRQhBBVaYAEXq/jdlQHI45Fbh4iA +isrLeAR0O2uYNeSZV5X5QQUuuhFuZZR3t65neuFCrrCFCCd1A8JjkjO3viE5qTJe +Wr5zd0XToUAmN+n1teApyoXIqsCEK9nya7uCpyioEJ9CMOMvEd2wig7yKVBd2Azp +kDaebgp9ivUsznfS/yRIRsPvV/Y9kw== +=ruob +-----END PGP SIGNATURE----- diff --git a/af20d83900b9bea7a7258e13dccec33be38aa849bf2387fcdd4ab6ab1e6069f1 b/af20d83900b9bea7a7258e13dccec33be38aa849bf2387fcdd4ab6ab1e6069f1 @@ -0,0 +1,249 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <link rel="stylesheet" type="text/css" href="./theme/css/style.css" /> + <link rel="stylesheet" type="text/css" href="./theme/css/syntax.css" /> + <title>man bytes gnu - Homemade internet state monitor</title> + <meta charset="utf-8" /> + <meta name="generator" content="Pelican" /> + +</head> + +<body id="index" class="home"> + + <header id="banner" class="body"> + <h1>> <a href="./">man bytes gnu</a><span id="cursor">_</span></h1> + </header> + <nav id="menu"> + <ul> + <li><a href="./category/archiving.html">Archiving</a></li> + <li><a href="./category/code.html">Code</a></li> + <li class="active"><a href="./category/hygiene.html">Hygiene</a></li> + <li><a href="./category/offlining.html">Offlining</a></li> + <li><a href="./pages/identities.html">Identities</a></li> + <li><a href="./pages/shares.html">Shares</a></li> + + <li><a href="/tags.html">tags</a></li> + </ul> + <hr/> + </nav><!-- /#menu --> + <header> + <h1 class="entry-title top-body-title"> + <a href="./internet-up-monitor.html" rel="bookmark" + title="Permalink to Homemade internet state monitor">Homemade internet state monitor</a></h1> + + + <div class="category meta"> + Posted +<time class="published" datetime="2021-07-01T13:59:00+02:00"> + Thu 01 July 2021 + </time> +in <a class="category" href="./category/hygiene.html">hygiene</a> + <a href="./tag/bash.html">bash</a> + <a href="./tag/systemd.html">systemd</a> + <a href="./tag/network.html">network</a> + <a href="./tag/i3wm.html">i3wm</a> + <a href="./tag/tmpfs.html">tmpfs</a> + <a href="./tag/openvpn.html">openvpn</a> + </div> + + <div class="neighbors meta"> + <ul> + <li> + Previous: <a href="./eth-log-bloom.html"> + Making sense of Ethereum log bloom filters + </a> + </li> + | + + <li> + Next: <a href="./backup-rsync-duplicity.html"> + Combining duplicity and rsync + </a> + </li> + </ul> + </div> + <div class="meta"> +<hr/> + </div> +</header> + +<section id="content" class="body"> + <div class="entry-content"> + <p>I use <a class="reference external" href="https://i3wm.org">i3wm</a>. I use it because it has a very low resource footprint, it has great documentation, and it's simple to configure.</p> +<p>Complementing <em>i3wm</em> is the <em>i3status</em> status provider to the <em>i3bar</em>. It provides a good suite of preset components for things like temperatur measurement, disk capacity. It also provides a couple of generic ones, namely "output the contents of a file" or "yes if file exists, no if not."</p> +<div class="section" id="it-s-not-you-it-s-me"> +<h2>It's not you, it's me?</h2> +<div class="admonition warning"> +<p class="first admonition-title">Warning</p> +<p class="last"><strong>Always use a VPN! Always!!</strong></p> +</div> +<p>Now, I <em>always</em> use VPN <em>no matter what</em>, and I configure network manually. That is, I do not use a network manager. For this reason, when network applications start misbehaving, it's sometimes hard to tell whether it is because the host is gone, the local network is flaky, the internet route is flaky, or simply that the connection is gone for good.</p> +<p>Evidently, it would be practical to have <strong>an item in the status bar telling me whether I have access to internet right now or not</strong>.</p> +<p>The "yes if file exists, no if not" option seems to be the right one to go for. Basically, we need to stick an empty file in a runtime file store whenever we detect that internet is available.</p> +</div> +<div class="section" id="ping-rinse-repeat"> +<h2>Ping, rinse, repeat</h2> +<p>There's not much trickery involved here. The check is a ping to a remote location every now and then.</p> +<p>Some caveats to cover to grant it some minimum of intelligence:</p> +<ul class="simple"> +<li>We need a few alternative destinations for the ping, that can take over if others happen to be down or take too long to repond.</li> +<li>To make the script useful for general purpose use, we'd also want to be able to supply a custom list of hosts to ping.</li> +<li>We also want to make sure that when the script is interrupted, it deletes the internet indicator file. That way, when it stops running, the internet indicator will be frozen in the "no" state.</li> +<li>Lastly, it can be useful to log to syslog every time the state of the internet connection (or the script itself) changes.</li> +</ul> +<p>That can translate to a shell script like this;</p> +<pre class="code bash literal-block"> +<span class="ch">#!/bin/bash +</span> +<span class="c1"># this is our runtime file, if - which exists - tells us internet is up +</span><span class="nv">f</span><span class="o">=</span><span class="s2">"/run/user/</span><span class="nv">$UID</span><span class="s2">/probe_up"</span> +rm -f <span class="nv">$f</span> + +up<span class="o">()</span> <span class="o">{</span> + <span class="k">if</span> <span class="o">[</span> ! -f <span class="nv">$f</span> <span class="o">]</span><span class="p">;</span> <span class="k">then</span> + logger -p info <span class="s2">"iup up"</span> + <span class="k">fi</span> + touch <span class="nv">$f</span> +<span class="o">}</span> +down<span class="o">()</span> <span class="o">{</span> + <span class="k">if</span> <span class="o">[</span> -f <span class="nv">$f</span> <span class="o">]</span><span class="p">;</span> <span class="k">then</span> + logger -p info <span class="s2">"iup down"</span> + <span class="k">fi</span> + rm -f <span class="nv">$f</span> +<span class="o">}</span> + +argh<span class="o">()</span> <span class="o">{</span> + logger -p info <span class="s2">"iup die"</span> + rm -f <span class="nv">$f</span> + <span class="nb">exit</span> <span class="m">0</span> +<span class="o">}</span> + +<span class="nv">IUP_DELAY</span><span class="o">=</span><span class="si">${</span><span class="nv">IUP_DELAY</span><span class="k">:-</span><span class="nv">$1</span><span class="si">}</span> +<span class="k">if</span> <span class="o">[</span> -z <span class="nv">$IUP_DELAY</span> <span class="o">]</span><span class="p">;</span> <span class="k">then</span> + <span class="nv">IUP_DELAY</span><span class="o">=</span><span class="m">5</span> +<span class="k">fi</span> +logger -p debug iup started with delay <span class="nv">$IUP_DELAY</span> + +<span class="nv">hosts</span><span class="o">=()</span> +<span class="k">while</span> <span class="nb">read</span> h<span class="p">;</span> <span class="k">do</span> + <span class="nv">hosts</span><span class="o">+=(</span><span class="nv">$h</span><span class="o">)</span> +<span class="k">done</span> < <span class="nv">$HOME</span>/.config/iup <span class="m">2</span>> /dev/null +<span class="k">if</span> <span class="o">[</span> <span class="si">${#</span><span class="nv">hosts</span><span class="p">[@]</span><span class="si">}</span> -eq <span class="s2">"0"</span> <span class="o">]</span><span class="p">;</span> <span class="k">then</span> + <span class="nv">hosts</span><span class="o">=(</span><span class="m">8</span>.8.8.8 <span class="m">151</span>.101.193.67<span class="o">)</span> <span class="c1"># google nameserver and cnn.com +</span> logger -p warn missing hosts input file, using evil default: <span class="si">${</span><span class="nv">hosts</span><span class="p">[@]</span><span class="si">}</span> +<span class="k">fi</span> + +<span class="nb">trap</span> argh SIGINT +<span class="nb">trap</span> argh SIGTERM +<span class="nb">trap</span> argh SIGQUIT + +<span class="k">while</span> true<span class="p">;</span> <span class="k">do</span> + <span class="nv">isup</span><span class="o">=</span><span class="s2">""</span> + <span class="k">for</span> h <span class="k">in</span> <span class="si">${</span><span class="nv">hosts</span><span class="p">[@]</span><span class="si">}</span><span class="p">;</span> <span class="k">do</span> + ping -c1 <span class="nv">$h</span> -w1 -q <span class="p">&</span>> /dev/null + <span class="k">if</span> <span class="o">[</span> <span class="nv">$?</span> -eq <span class="s1">'0'</span> <span class="o">]</span><span class="p">;</span> <span class="k">then</span> + up + <span class="nv">isup</span><span class="o">=</span><span class="m">1</span> + <span class="c1">#logger -p debug "ping success $h" +</span> <span class="nb">break</span> + <span class="k">fi</span> + <span class="k">done</span> + <span class="k">if</span> <span class="o">[</span> -z <span class="nv">$isup</span> <span class="o">]</span><span class="p">;</span> <span class="k">then</span> + down + <span class="k">fi</span> + sleep <span class="nv">$IUP_DELAY</span> +<span class="k">done</span> +</pre> +<p>Since we'll want this to run when network interfaces allegedly are up, it makes sense to link it to systemd and the network target:</p> +<pre class="code ini literal-block"> +<span class="k">[Unit]</span> +<span class="na">Description</span><span class="o">=</span><span class="s">Internet connection poller</span> +<span class="na">After</span><span class="o">=</span><span class="s">multi-user.target</span> + +<span class="k">[Service]</span> +<span class="na">ExecStart</span><span class="o">=</span><span class="s">/usr/local/bin/iup.sh</span> +<span class="na">Restart</span><span class="o">=</span><span class="s">always</span> + +<span class="k">[Install]</span> +<span class="na">WantedBy</span><span class="o">=</span><span class="s">multi-user.target</span> +</pre> +<p>Stick this in <code>~/.config/systemd/user/iup.service</code> and enable and start the service:</p> +<div class="highlight"><pre><span></span>$ systemctl --user <span class="nb">enable</span> iup.service +$ systemctl --user start iup.service +</pre></div> +</div> +<div class="section" id="checking-what-status-our-status-is-in"> +<h2>Checking what status our status is in</h2> +<p>Now, in the i3status configuration, assuming that your system uid is 1000 <a class="footnote-reference" href="#id3" id="id1">[1]</a>, add:</p> +<div class="highlight"><pre><span></span>order += "run_watch INET" + +kj + +run_watch INET { + pidfile = "/run/user/1000/probe_up" +} +</pre></div> +<p>To reload the config on the fly, press <code>mod+r</code> (that's <code>ctrl+r</code> on mine). The result should be a new item in your bar showing <code>INET: yes</code> (most likely yes, anyway, since you're currently reading this).</p> +</div> +<div class="section" id="safety-first-eh-second"> +<h2>Safety first, eh ... second</h2> +<div class="admonition warning"> +<p class="first admonition-title">Warning</p> +<p class="last"><strong>Did I mention that you should always use a VPN?</strong></p> +</div> +<p>The same <code>run_watch</code> trick can be used for VPN.</p> +<p>I use <code>openvpn</code>, and it defines a flag <code>--writepid</code>. If you pass a file path to this parameter, either through the <code>writepid</code> config directiry or on the <code>--writepid</code> flag on the command line, it will write the openvpn pid to that file, and similarly remove it when the openvpn service ends.</p> +<p>However, this raises another issue. Namely the fact that a location is needed for openvpn to write the file to, for which it also has access.</p> +<p>If we want to use the <code>/run</code> directory again, now we also has to make sure that this directory exists.</p> +<div class="section" id="got-the-runs"> +<h3>Got the runs?</h3> +<p>This is exactly what <code>systemd-tmpfiles</code> is for. You can add files to <code>/etc/tmpfiles.d</code> which describe what kind of temporary files or directories to add <a class="footnote-reference" href="#id4" id="id2">[2]</a>.</p> +<p>Create a file <code>/etc/tmpfiles.d/openvpn.conf</code> and add a single line to it:</p> +<div class="highlight"><pre><span></span>d /run/openvpn 0755 openvpn openvpn +</pre></div> +<p>This will ensure that the folder is created at startup.</p> +<p>Now, to create one right away for the current session, run <code>systemd-tmpfiles --create</code>.</p> +<p>Now we have everything we need to add the VPN status to <em>i3status</em> aswell. Provided that we chose <code>/run/openvpn/openvpn.pid</code> as our argument to <code>openvpn --writepid</code>, the setup is more or less the same as before:</p> +<div class="highlight"><pre><span></span>order += "run_watch VPN" + +kj + +run_watch VPN { + pidfile = "/run/openvpn/openvpn.pid" +} +</pre></div> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id3" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id1">[1]</a></td><td>The <code>/run/user/$UID</code> folder will be created on system startup, and will be writable by that user. It provides a tmpfs storage location for processes that run in early stages of boot, before the entire filesystem tree (which may include <code>/var/run</code> because <code>/var</code> may be on a separate partition) has been mounted. More in that here: <a class="reference external" href="https://lwn.net/Articles/436012/">https://lwn.net/Articles/436012/</a></td></tr> +</tbody> +</table> +</blockquote> +<!-- --> +<blockquote> +<table class="docutils footnote" frame="void" id="id4" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id2">[2]</a></td><td>This service has an impressive amount of flexibility, which you can inspect yourself by visiting the <code>tmpfiles.d</code> (and <code>systemd-tmpfiles</code>) manpage.</td></tr> +</tbody> +</table> +</blockquote> +</div> +</div> + + </div><!-- /.entry-content --> + <footer class="meta"> + </footer> + +</section> + <footer id="contentinfo" class="body"> + <hr/> + <address id="about" class="vcard body"> + Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> + </address><!-- /#about --> + </footer><!-- /#contentinfo --> +</body> +</html> +\ No newline at end of file diff --git a/af20d83900b9bea7a7258e13dccec33be38aa849bf2387fcdd4ab6ab1e6069f1.asc b/af20d83900b9bea7a7258e13dccec33be38aa849bf2387fcdd4ab6ab1e6069f1.asc @@ -0,0 +1,11 @@ +-----BEGIN PGP SIGNATURE----- + +iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmHkZ0UACgkQzcvSTdHQ +4AGg9Qf/RzFbohchDoQ47kuELxjZLLJW0PO3zYKRHlafSHJUSSRK0e0p363+WUgU +UbZYy6VSicVwBCPjT8pRz8192vobT+Zs8O3On87ikij3KufovJTfmdRfl6PxR4RC +kZIYCgDrktW6lEUyRM5E7FurouITEGFQahjO/X29KSXiL95Tt995SKYAMCTR/lo9 +0Wmo/ITojndNV/PrndQo55/fREiKtD1E+uElzTsMldPdYi6a0jWGdjwE50fgIkDC +VgfjHRLfIpKc3KDpywSBrqebWd1bKJbSGYDFQquuz/TcVwuH4fsHllw49xNKEmbN +LguBumV4lbxVagZtmdS26pjebsfc9A== +=tWqB +-----END PGP SIGNATURE----- diff --git a/drafts/celery-document-graph.html b/drafts/celery-document-graph.html @@ -1,87 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> -<head> - <link rel="stylesheet" type="text/css" href="../theme/css/style.css" /> - <link rel="stylesheet" type="text/css" href="../theme/css/syntax.css" /> - <title>man bytes gnu - Documenting Celery task chains</title> - <meta charset="utf-8" /> - <meta name="generator" content="Pelican" /> - -</head> - -<body id="index" class="home"> - - <header id="banner" class="body"> - <h1>> <a href="../">man bytes gnu</a><span id="cursor">_</span></h1> - </header> - <nav id="menu"> - <ul> - <li><a href="../category/archiving.html">Archiving</a></li> - <li class="active"><a href="../category/code.html">Code</a></li> - <li><a href="../category/hygiene.html">Hygiene</a></li> - <li><a href="../category/offlining.html">Offlining</a></li> - <li><a href="../pages/identities.html">Identities</a></li> - <li><a href="../pages/shares.html">Shares</a></li> - - <li><a href="/tags.html">tags</a></li> - </ul> - <hr/> - </nav><!-- /#menu --> - <header> - <h1 class="entry-title top-body-title"> - <a href="../drafts/celery-document-graph.html" rel="bookmark" - title="Permalink to Documenting Celery task chains">Documenting Celery task chains</a></h1> - - - <div class="category meta"> - Posted -<time class="published" datetime="2021-04-25T15:00:00+02:00"> - Sun 25 April 2021 - </time> -in <a class="category" href="../category/code.html">code</a> - <a href="../tag/python.html">python</a> - <a href="../tag/microservices.html">microservices</a> - <a href="../tag/celery.html">celery</a> - </div> - - <div class="neighbors meta"> - <ul> - </ul> - </div> - <div class="meta"> -<hr/> - </div> -</header> - -<section id="content" class="body"> - <div class="entry-content"> - <div class="highlight"><pre><span></span><span class="n">current_app</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">update</span><span class="p">({</span> -<span class="s1">'broker_url'</span><span class="p">:</span> <span class="n">broker</span><span class="p">,</span> -<span class="p">})</span> - -<span class="n">result</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'CELERY_RESULT_URL'</span><span class="p">)</span> -<span class="k">if</span> <span class="n">result</span><span class="p">[:</span><span class="mi">4</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'file'</span><span class="p">:</span> - <span class="n">rq</span> <span class="o">=</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">mkdtemp</span><span class="p">()</span> - <span class="n">current_app</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">update</span><span class="p">({</span> - <span class="s1">'result_backend'</span><span class="p">:</span> <span class="s1">'file://</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">rq</span><span class="p">),</span> - <span class="p">})</span> - <span class="n">logg</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s1">'celery backend store dir </span><span class="si">{}</span><span class="s1"> created, will NOT be deleted on shutdown'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">rq</span><span class="p">))</span> -<span class="k">else</span><span class="p">:</span> - <span class="n">current_app</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">update</span><span class="p">({</span> - <span class="s1">'result_backend'</span><span class="p">:</span> <span class="n">result</span><span class="p">,</span> - <span class="p">})</span> -</pre></div> - - </div><!-- /.entry-content --> - <footer class="meta"> - </footer> - -</section> - <footer id="contentinfo" class="body"> - <hr/> - <address id="about" class="vcard body"> - Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> - </address><!-- /#about --> - </footer><!-- /#contentinfo --> -</body> -</html> -\ No newline at end of file diff --git a/drafts/celery-document-graph.sha256 b/drafts/celery-document-graph.sha256 @@ -1 +0,0 @@ -9c19cf6e859d24826ea226adc1957b0366371f742f6c57f366dae84bd7358872 celery-document-graph.html -\ No newline at end of file diff --git a/drafts/clortho.html b/drafts/clortho.html @@ -1,92 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> -<head> - <link rel="stylesheet" type="text/css" href="../theme/css/style.css" /> - <link rel="stylesheet" type="text/css" href="../theme/css/syntax.css" /> - <title>man bytes gnu - Clortho</title> - <meta charset="utf-8" /> - <meta name="generator" content="Pelican" /> - -</head> - -<body id="index" class="home"> - - <header id="banner" class="body"> - <h1>> <a href="../">man bytes gnu</a><span id="cursor">_</span></h1> - </header> - <nav id="menu"> - <ul> - <li><a href="../category/archiving.html">Archiving</a></li> - <li class="active"><a href="../category/code.html">Code</a></li> - <li><a href="../category/hygiene.html">Hygiene</a></li> - <li><a href="../category/offlining.html">Offlining</a></li> - <li><a href="../pages/identities.html">Identities</a></li> - <li><a href="../pages/shares.html">Shares</a></li> - - <li><a href="/tags.html">tags</a></li> - </ul> - <hr/> - </nav><!-- /#menu --> - <header> - <h1 class="entry-title top-body-title"> - <a href="../drafts/clortho.html" rel="bookmark" - title="Permalink to Clortho">Clortho</a></h1> - - - <div class="category meta"> - Posted -<time class="published" datetime="2022-01-12T15:03:00+01:00"> - Wed 12 January 2022 - </time> -in <a class="category" href="../category/code.html">code</a> - <a href="../tag/crypto.html">crypto</a> - <a href="../tag/hash.html">hash</a> - <a href="../tag/sha512.html">sha512</a> - <a href="../tag/bash.html">bash</a> - <a href="../tag/cli.html">cli</a> - </div> - - <div class="neighbors meta"> - <ul> - </ul> - </div> - <div class="meta"> -<hr/> - </div> -</header> - -<section id="content" class="body"> - <div class="entry-content"> - <p>Ever since I started using the <a class="reference external" href="https://www.passwordstore.org/">pass</a> CLI as my password manager, I've found myself putting all sorts of stuff in there; usernames, email, urls, crypto addresses, api keys, you name it.</p> -<p>It makes total sense that some of these items are in there. For example, I store the url to a service together with the password, usually accompanied by the username and the email used <a class="footnote-reference" href="#id2" id="id1">[1]</a>. No password recoveries needed.</p> -<p>However, I've also started putting in things like crypto addresses, or even token smart conrtact addresses in there, it seems less of a good fit. One thing is that it spams the password directory. But another more sinister issues is that it's pretty clear for anyone reading the directory what items you are storing data for.</p> -<div class="section" id="hiding-the-key"> -<h2>Hiding the key</h2> -<p>So what if I want to store key/value pairs, and at the same time I want to hide what I am storing?</p> -<!-- --> -<blockquote> -<table class="docutils footnote" frame="void" id="id2" rules="none"> -<colgroup><col class="label" /><col /></colgroup> -<tbody valign="top"> -<tr><td class="label"><a class="fn-backref" href="#id1">[1]</a></td><td>I use a different email for each service I sign up to, and for every other context I have to leave my email for something.</td></tr> -</tbody> -</table> -</blockquote> -<!-- --> -<blockquote> -</blockquote> -</div> - - </div><!-- /.entry-content --> - <footer class="meta"> - </footer> - -</section> - <footer id="contentinfo" class="body"> - <hr/> - <address id="about" class="vcard body"> - Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> - </address><!-- /#about --> - </footer><!-- /#contentinfo --> -</body> -</html> -\ No newline at end of file diff --git a/drafts/clortho.sha256 b/drafts/clortho.sha256 @@ -1 +0,0 @@ -39dec57f05fb159fc94fd24141905614d81351582f12a0bfc60bf8b136f76a57 clortho.html -\ No newline at end of file diff --git a/drafts/docker-offline-4-debian.html b/drafts/docker-offline-4-debian.html @@ -1,82 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> -<head> - <link rel="stylesheet" type="text/css" href="../theme/css/style.css" /> - <link rel="stylesheet" type="text/css" href="../theme/css/syntax.css" /> - <title>man bytes gnu - Debian repository</title> - <meta charset="utf-8" /> - <meta name="generator" content="Pelican" /> - -</head> - -<body id="index" class="home"> - - <header id="banner" class="body"> - <h1>> <a href="../">man bytes gnu</a><span id="cursor">_</span></h1> - </header> - <nav id="menu"> - <ul> - <li><a href="../category/archiving.html">Archiving</a></li> - <li><a href="../category/code.html">Code</a></li> - <li><a href="../category/hygiene.html">Hygiene</a></li> - <li class="active"><a href="../category/offlining.html">Offlining</a></li> - <li><a href="../pages/identities.html">Identities</a></li> - <li><a href="../pages/shares.html">Shares</a></li> - - <li><a href="/tags.html">tags</a></li> - </ul> - <hr/> - </nav><!-- /#menu --> - <header> - <h1 class="entry-title top-body-title"> - <a href="../drafts/docker-offline-4-debian.html" rel="bookmark" - title="Permalink to Debian repository">Offline Docker: Debian repository</a></h1> - - - <div class="category meta"> - Posted -<time class="published" datetime="2021-06-23T13:14:00+02:00"> - Wed 23 June 2021 - </time> -in <a class="category" href="../category/offlining.html">offlining</a> - <a href="../tag/docker.html">docker</a> - <a href="../tag/networking.html">networking</a> - <a href="../tag/debian.html">debian</a> - </div> - - <div class="neighbors meta"> - <ul> - </ul> - </div> - <div class="entry-series meta"> -Part 4 from the series "Offline Docker" - <ol class="entry-series-parts"> - <li>| <a href="docker-offline-1-routing.html" title="The routing to freedom ">Part 1</a></li> - <li>| <a href="docker-offline-2-python.html" title="Local python repository ">Part 2</a></li> - <li>| <a href="docker-offline-3-npm.html" title="Local npm repository ">Part 3</a></li> - </ol> - </div> - <div class="meta"> -<hr/> - </div> -</header> - -<section id="content" class="body"> - <div class="entry-content"> - <!-- --> -<blockquote> -</blockquote> - - </div><!-- /.entry-content --> - <footer class="meta"> - </footer> - -</section> - <footer id="contentinfo" class="body"> - <hr/> - <address id="about" class="vcard body"> - Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> - </address><!-- /#about --> - </footer><!-- /#contentinfo --> -</body> -</html> -\ No newline at end of file diff --git a/drafts/docker-offline-4-debian.sha256 b/drafts/docker-offline-4-debian.sha256 @@ -1 +0,0 @@ -a567d8d38afd23aa934573a7efe197dbca3584dbe519df7c086d7997845d100a docker-offline-4-debian.html -\ No newline at end of file diff --git a/drafts/docker-offline-5-npm-bundle.html b/drafts/docker-offline-5-npm-bundle.html @@ -1,131 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> -<head> - <link rel="stylesheet" type="text/css" href="../theme/css/style.css" /> - <link rel="stylesheet" type="text/css" href="../theme/css/syntax.css" /> - <title>man bytes gnu - Local npm bundles</title> - <meta charset="utf-8" /> - <meta name="generator" content="Pelican" /> - -</head> - -<body id="index" class="home"> - - <header id="banner" class="body"> - <h1>> <a href="../">man bytes gnu</a><span id="cursor">_</span></h1> - </header> - <nav id="menu"> - <ul> - <li><a href="../category/archiving.html">Archiving</a></li> - <li><a href="../category/code.html">Code</a></li> - <li><a href="../category/hygiene.html">Hygiene</a></li> - <li class="active"><a href="../category/offlining.html">Offlining</a></li> - <li><a href="../pages/identities.html">Identities</a></li> - <li><a href="../pages/shares.html">Shares</a></li> - - <li><a href="/tags.html">tags</a></li> - </ul> - <hr/> - </nav><!-- /#menu --> - <header> - <h1 class="entry-title top-body-title"> - <a href="../drafts/docker-offline-5-npm-bundle.html" rel="bookmark" - title="Permalink to Local npm bundles">Offline Docker: Local npm bundles</a></h1> - - - <div class="category meta"> - Posted -<time class="published" datetime="2021-10-22T15:26:00+02:00"> - Fri 22 October 2021 - </time> -in <a class="category" href="../category/offlining.html">offlining</a> - <a href="../tag/docker.html">docker</a> - <a href="../tag/npm.html">npm</a> - <a href="../tag/nodejs.html">nodejs</a> - <a href="../tag/javascript.html">javascript</a> - <a href="../tag/devops.html">devops</a> - </div> - - <div class="neighbors meta"> - <ul> - </ul> - </div> - <div class="entry-series meta"> -Part 5 from the series "Offline Docker" - <ol class="entry-series-parts"> - <li>| <a href="docker-offline-1-routing.html" title="The routing to freedom ">Part 1</a></li> - <li>| <a href="docker-offline-2-python.html" title="Local python repository ">Part 2</a></li> - <li>| <a href="docker-offline-3-npm.html" title="Local npm repository ">Part 3</a></li> - </ol> - </div> - <div class="meta"> -<hr/> - </div> -</header> - -<section id="content" class="body"> - <div class="entry-content"> - <div class="highlight"><pre><span></span><span class="go"><Directory "/srv/http/npm"></span> -<span class="go"> DirectoryIndex package.json</span> -<span class="go"> RewriteEngine on</span> -<span class="go"> RewriteRule "^(.+)-\/(.+)$" "/npm/$1$2" [R=301]</span> -<span class="go"></Directory></span> -</pre></div> -<pre class="code python literal-block"> -<span class="kn">import</span> <span class="nn">json</span> -<span class="kn">import</span> <span class="nn">os</span> -<span class="kn">import</span> <span class="nn">urllib.request</span> -<span class="kn">import</span> <span class="nn">urllib.parse</span> -<span class="kn">import</span> <span class="nn">logging</span> - -<span class="n">logging</span><span class="o">.</span><span class="n">basicConfig</span><span class="p">(</span><span class="n">level</span><span class="o">=</span><span class="n">logging</span><span class="o">.</span><span class="n">DEBUG</span><span class="p">)</span> -<span class="n">logg</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">()</span> - - -<span class="n">f</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="s1">'package-lock.json'</span><span class="p">,</span> <span class="s1">'r'</span><span class="p">)</span> -<span class="n">j</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">f</span><span class="p">)</span> -<span class="n">f</span><span class="o">.</span><span class="n">close</span><span class="p">()</span> - -<span class="n">script_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">realpath</span><span class="p">(</span><span class="vm">__file__</span><span class="p">))</span> -<span class="n">output_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">script_dir</span><span class="p">,</span> <span class="s1">'_out'</span><span class="p">)</span> - -<span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">j</span><span class="p">[</span><span class="s1">'packages'</span><span class="p">]:</span> - <span class="k">if</span> <span class="n">k</span> <span class="o">==</span> <span class="s1">''</span><span class="p">:</span> - <span class="k">continue</span> - <span class="n">url</span> <span class="o">=</span> <span class="n">j</span><span class="p">[</span><span class="s1">'packages'</span><span class="p">][</span><span class="n">k</span><span class="p">][</span><span class="s1">'resolved'</span><span class="p">]</span> - <span class="n">parts</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">urlparse</span><span class="p">(</span><span class="n">url</span><span class="p">)</span> - <span class="n">output_path_package</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_dir</span><span class="p">,</span> <span class="n">parts</span><span class="o">.</span><span class="n">path</span><span class="p">[</span><span class="mi">1</span><span class="p">:])</span> - <span class="k">try</span><span class="p">:</span> - <span class="n">os</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">output_path_package</span><span class="p">)</span> - <span class="n">logg</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'skipping existing </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">output_path_package</span><span class="p">))</span> - <span class="k">continue</span> - <span class="k">except</span> <span class="ne">FileNotFoundError</span><span class="p">:</span> - <span class="k">pass</span> - <span class="n">output_dir_package</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="n">output_path_package</span><span class="p">)</span> - <span class="c1">#print(output_path_package)</span> - <span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">output_dir_package</span><span class="p">,</span> <span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> - - <span class="n">rq</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">request</span><span class="o">.</span><span class="n">urlopen</span><span class="p">(</span><span class="n">url</span><span class="p">)</span> - <span class="n">c</span> <span class="o">=</span> <span class="n">rq</span><span class="o">.</span><span class="n">read</span><span class="p">()</span> - <span class="n">f</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">output_path_package</span><span class="p">,</span> <span class="s1">'wb'</span><span class="p">)</span> - <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">c</span><span class="p">)</span> - <span class="n">f</span><span class="o">.</span><span class="n">close</span><span class="p">()</span> - <span class="n">logg</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'wrote </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">output_path_package</span><span class="p">))</span> -</pre> -<pre class="code bash literal-block"> -find _out -type f -name <span class="s2">"*gz"</span> -exec rename -v <span class="s2">"/-"</span> <span class="s2">""</span> <span class="o">{}</span> <span class="se">\;</span> -</pre> - - </div><!-- /.entry-content --> - <footer class="meta"> - </footer> - -</section> - <footer id="contentinfo" class="body"> - <hr/> - <address id="about" class="vcard body"> - Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> - </address><!-- /#about --> - </footer><!-- /#contentinfo --> -</body> -</html> -\ No newline at end of file diff --git a/drafts/docker-offline-5-npm-bundle.sha256 b/drafts/docker-offline-5-npm-bundle.sha256 @@ -1 +0,0 @@ -71324ed671b174ad627270e4397da302430b4aad77b1b0479e2bc251808336a7 docker-offline-5-npm-bundle.html -\ No newline at end of file diff --git a/drafts/docker-vpn.html b/drafts/docker-vpn.html @@ -1,102 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> -<head> - <link rel="stylesheet" type="text/css" href="../theme/css/style.css" /> - <link rel="stylesheet" type="text/css" href="../theme/css/syntax.css" /> - <title>man bytes gnu - Using Docker with VPN</title> - <meta charset="utf-8" /> - <meta name="generator" content="Pelican" /> - -</head> - -<body id="index" class="home"> - - <header id="banner" class="body"> - <h1>> <a href="../">man bytes gnu</a><span id="cursor">_</span></h1> - </header> - <nav id="menu"> - <ul> - <li><a href="../category/archiving.html">Archiving</a></li> - <li><a href="../category/code.html">Code</a></li> - <li class="active"><a href="../category/hygiene.html">Hygiene</a></li> - <li><a href="../category/offlining.html">Offlining</a></li> - <li><a href="../pages/identities.html">Identities</a></li> - <li><a href="../pages/shares.html">Shares</a></li> - - <li><a href="/tags.html">tags</a></li> - </ul> - <hr/> - </nav><!-- /#menu --> - <header> - <h1 class="entry-title top-body-title"> - <a href="../drafts/docker-vpn.html" rel="bookmark" - title="Permalink to Using Docker with VPN">Using Docker with VPN</a></h1> - - - <div class="category meta"> - Posted -<time class="published" datetime="2021-04-21T06:53:00+02:00"> - Wed 21 April 2021 - </time> -in <a class="category" href="../category/hygiene.html">hygiene</a> - <a href="../tag/vpn.html">vpn</a> - <a href="../tag/openvpn.html">openvpn</a> - <a href="../tag/iproute.html">iproute</a> - <a href="../tag/docker.html">docker</a> - </div> - - <div class="neighbors meta"> - <ul> - </ul> - </div> - <div class="meta"> -<hr/> - </div> -</header> - -<section id="content" class="body"> - <div class="entry-content"> - <p>Need to route through the tun interface, which Docker doesn't seem to automatically do.</p> -<div class="highlight"><pre><span></span>$ ip route ls -<span class="o">[</span>...<span class="o">]</span> -<span class="m">10</span>.7.3.0/24 dev tun0 proto kernel scope link src <span class="m">10</span>.7.3.6 -</pre></div> -<pre class="literal-block"> -ERROR: could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network -</pre> -<p>The simplest solution I have found is to exchange the gateway redirect from the OpenVPN server <a class="footnote-reference" href="#id2" id="id1">[1]</a> and add the VPN interface as the default route.</p> -<pre class="code bash literal-block"> -<span class="ch">#!/bin/sh -</span> -<span class="nv">default_route_vpn_gateway</span><span class="o">=</span><span class="sb">`</span>ip route <span class="p">|</span> awk <span class="s1">'{if ($1 ~ /^0.0.0.0\/1$/) { print $3; }}'</span><span class="sb">`</span> -<span class="nv">route_vpn_gateway</span><span class="o">=</span><span class="si">${</span><span class="nv">VPN_GATEWAY</span><span class="k">:-</span><span class="nv">$default_route_vpn_gateway</span><span class="si">}</span> -<span class="nb">echo</span> <span class="s2">"Adding default route to </span><span class="nv">$route_vpn_gateway</span><span class="s2"> with /0 mask..."</span> -ip route add default via <span class="nv">$route_vpn_gateway</span> - -<span class="nb">echo</span> <span class="s2">"Removing /1 routes..."</span> -ip route del <span class="m">0</span>.0.0.0/1 via <span class="nv">$route_vpn_gateway</span> -ip route del <span class="m">128</span>.0.0.0/1 via <span class="nv">$route_vpn_gateway</span> -</pre> -<!-- --> -<blockquote> -<table class="docutils footnote" frame="void" id="id2" rules="none"> -<colgroup><col class="label" /><col /></colgroup> -<tbody valign="top"> -<tr><td class="label"><a class="fn-backref" href="#id1">[1]</a></td><td><a class="reference external" href="https://stackoverflow.com/a/62333327">https://stackoverflow.com/a/62333327</a></td></tr> -</tbody> -</table> -</blockquote> - - </div><!-- /.entry-content --> - <footer class="meta"> - </footer> - -</section> - <footer id="contentinfo" class="body"> - <hr/> - <address id="about" class="vcard body"> - Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> - </address><!-- /#about --> - </footer><!-- /#contentinfo --> -</body> -</html> -\ No newline at end of file diff --git a/drafts/docker-vpn.sha256 b/drafts/docker-vpn.sha256 @@ -1 +0,0 @@ -43e8f61826ca6048ee2ed086955a4733d01ba3cc58e2c0f3784d7932f4e2a438 docker-vpn.html -\ No newline at end of file diff --git a/drafts/ipfs-api-get.html b/drafts/ipfs-api-get.html @@ -1,101 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> -<head> - <link rel="stylesheet" type="text/css" href="../theme/css/style.css" /> - <link rel="stylesheet" type="text/css" href="../theme/css/syntax.css" /> - <title>man bytes gnu - Dissecting the IPFS API</title> - <meta charset="utf-8" /> - <meta name="generator" content="Pelican" /> - -</head> - -<body id="index" class="home"> - - <header id="banner" class="body"> - <h1>> <a href="../">man bytes gnu</a><span id="cursor">_</span></h1> - </header> - <nav id="menu"> - <ul> - <li><a href="../category/archiving.html">Archiving</a></li> - <li class="active"><a href="../category/code.html">Code</a></li> - <li><a href="../category/hygiene.html">Hygiene</a></li> - <li><a href="../category/offlining.html">Offlining</a></li> - <li><a href="../pages/identities.html">Identities</a></li> - <li><a href="../pages/shares.html">Shares</a></li> - - <li><a href="/tags.html">tags</a></li> - </ul> - <hr/> - </nav><!-- /#menu --> - <header> - <h1 class="entry-title top-body-title"> - <a href="../drafts/ipfs-api-get.html" rel="bookmark" - title="Permalink to Dissecting the IPFS API">Dissecting the IPFS API</a></h1> - - - <div class="category meta"> - Posted -<time class="published" datetime="2021-05-01T08:40:00+02:00"> - Sat 01 May 2021 - </time> -in <a class="category" href="../category/code.html">code</a> - </div> - - <div class="neighbors meta"> - <ul> - </ul> - </div> - <div class="meta"> -<hr/> - </div> -</header> - -<section id="content" class="body"> - <div class="entry-content"> - <div class="highlight"><pre><span></span>$ curl -X POST <span class="s2">"http://localhost:5001/api/v0/object/get?arg=QmWajb6k7oT9HVTq5hgnGZqRxzwwA2g7D33MFjzd2nVcXw"</span> -<span class="o">{</span><span class="s2">"Links"</span>:<span class="o">[{</span><span class="s2">"Name"</span>:<span class="s2">""</span>,<span class="s2">"Hash"</span>:<span class="s2">"QmZeEX1hbQtj34TPLdbUwbFtQBGAmanTVUB6HuiTcrLrfQ"</span>,<span class="s2">"Size"</span>:262158<span class="o">}</span>,<span class="o">{</span><span class="s2">"Name"</span>:<span class="s2">""</span>,<span class="s2">"Hash"</span>:<span class="s2">"QmWdaV34ArzVJH2Si9CxS9b5SxCnmU1HMuEXNA137XoV4f"</span>,<span class="s2">"Size"</span>:262158<span class="o">}</span>,<span class="o">{</span><span class="s2">"Name"</span>:<span class="s2">""</span>,<span class="s2">"Hash"</span>:<span class="s2">"QmUgSEJDRj7So3x7n8RaUfiLfcDy5CyqY7btJdm6B7z4Ae"</span>,<span class="s2">"Size"</span>:131390<span class="o">}]</span>,<span class="s2">"Data"</span>:<span class="s2">"\u0008\u0002\u0018\ufffd\ufffd( \ufffd\ufffd\u0010 \ufffd\ufffd\u0010 \ufffd\ufffd\u0008"</span><span class="o">}</span> -</pre></div> -<div class="highlight"><pre><span></span>$ curl -X POST <span class="s2">"http://localhost:5001/api/v0/get?arg=QmZeEX1hbQtj34TPLdbUwbFtQBGAmanTVUB6HuiTcrLrfQ"</span> <span class="p">|</span> hexdump -C -n1024 -<span class="m">00000010</span> 4c <span class="m">64</span> <span class="m">62</span> <span class="m">55</span> <span class="m">77</span> <span class="m">62</span> <span class="m">46</span> <span class="m">74</span> <span class="m">51</span> <span class="m">42</span> <span class="m">47</span> <span class="m">41</span> 6d <span class="m">61</span> 6e <span class="m">54</span> <span class="p">|</span>LdbUwbFtQBGAmanT<span class="p">|</span> -<span class="m">00000020</span> <span class="m">56</span> <span class="m">55</span> <span class="m">42</span> <span class="m">36</span> <span class="m">48</span> <span class="m">75</span> <span class="m">69</span> <span class="m">54</span> <span class="m">63</span> <span class="m">72</span> 4c <span class="m">72</span> <span class="m">66</span> <span class="m">51</span> <span class="m">00</span> <span class="m">00</span> <span class="p">|</span>VUB6HuiTcrLrfQ..<span class="p">|</span> -<span class="m">00000030</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="p">|</span>................<span class="p">|</span> -* -<span class="m">00000060</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">36</span> <span class="m">34</span> <span class="m">34</span> <span class="m">00</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="p">|</span>....0000644.0000<span class="p">|</span> -<span class="m">00000070</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">00</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">00</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="p">|</span><span class="m">000</span>.0000000.0000<span class="p">|</span> -<span class="m">00000080</span> <span class="m">31</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">00</span> <span class="m">31</span> <span class="m">34</span> <span class="m">30</span> <span class="m">34</span> <span class="m">36</span> <span class="m">31</span> <span class="m">35</span> <span class="m">33</span> <span class="p">|</span><span class="m">1000000</span>.14046153<span class="p">|</span> -<span class="m">00000090</span> <span class="m">30</span> <span class="m">33</span> <span class="m">36</span> <span class="m">00</span> <span class="m">30</span> <span class="m">31</span> <span class="m">37</span> <span class="m">31</span> <span class="m">37</span> <span class="m">32</span> <span class="m">00</span> <span class="m">20</span> <span class="m">30</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="p">|</span><span class="m">036</span>.017172. <span class="m">0</span>...<span class="p">|</span> -000000a0 <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="p">|</span>................<span class="p">|</span> -* -<span class="m">00000100</span> <span class="m">00</span> <span class="m">75</span> <span class="m">73</span> <span class="m">74</span> <span class="m">61</span> <span class="m">72</span> <span class="m">00</span> <span class="m">30</span> <span class="m">30</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="p">|</span>.ustar.00.......<span class="p">|</span> -<span class="m">00000110</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="p">|</span>................<span class="p">|</span> -* -<span class="m">00000140</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="p">|</span>.........0000000<span class="p">|</span> -<span class="m">00000150</span> <span class="m">00</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">30</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="p">|</span>.0000000........<span class="p">|</span> -<span class="m">00000160</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="p">|</span>................<span class="p">|</span> -* -<span class="m">00000200</span> <span class="m">47</span> <span class="m">49</span> <span class="m">46</span> <span class="m">38</span> <span class="m">39</span> <span class="m">61</span> f4 <span class="m">01</span> 7b <span class="m">01</span> f7 fb <span class="m">00</span> <span class="m">94</span> <span class="m">31</span> <span class="m">17</span> <span class="p">|</span>GIF89a..<span class="o">{</span>.....1.<span class="p">|</span> -<span class="m">00000210</span> ac c7 d8 a8 c6 ec d5 <span class="m">70</span> 4b <span class="m">90</span> 8a <span class="m">71</span> e9 <span class="nb">cd</span> a9 c6 <span class="p">|</span>.......pK..q....<span class="p">|</span> -</pre></div> -<div class="highlight"><pre><span></span>$ curl -X POST <span class="s2">"http://localhost:5001/api/v0/get?arg=QmZeEX1hbQtj34TPLdbUwbFtQBGAmanTVUB6HuiTcrLrfQ"</span> <span class="p">|</span> dd <span class="nv">bs</span><span class="o">=</span><span class="m">512</span> <span class="nv">skip</span><span class="o">=</span><span class="m">1</span> > pat.gif -$ curl -X POST <span class="s2">"http://localhost:5001/api/v0/get?arg=QmWdaV34ArzVJH2Si9CxS9b5SxCnmU1HMuEXNA137XoV4f"</span> <span class="p">|</span> dd <span class="nv">bs</span><span class="o">=</span><span class="m">512</span> <span class="nv">skip</span><span class="o">=</span><span class="m">1</span> >> pat.gif -$ curl -X POST <span class="s2">"http://localhost:5001/api/v0/get?arg=QmUgSEJDRj7So3x7n8RaUfiLfcDy5CyqY7btJdm6B7z4Ae"</span> <span class="p">|</span> dd <span class="nv">bs</span><span class="o">=</span><span class="m">512</span> <span class="nv">skip</span><span class="o">=</span><span class="m">1</span> >> pat.gif -</pre></div> -<div class="highlight"><pre><span></span>$ curl -X POST <span class="s2">"http://localhost:5001/api/v0/cat?arg=QmWajb6k7oT9HVTq5hgnGZqRxzwwA2g7D33MFjzd2nVcXw"</span> -o pat2.gif -$ diff pat2.gif pat.gif -$ <span class="nb">echo</span> <span class="nv">$?</span> -<span class="m">0</span> -</pre></div> - - </div><!-- /.entry-content --> - <footer class="meta"> - </footer> - -</section> - <footer id="contentinfo" class="body"> - <hr/> - <address id="about" class="vcard body"> - Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> - </address><!-- /#about --> - </footer><!-- /#contentinfo --> -</body> -</html> -\ No newline at end of file diff --git a/drafts/ipfs-api-get.sha256 b/drafts/ipfs-api-get.sha256 @@ -1 +0,0 @@ -c00ee4bf6cc06399bc6f237a394955ea165b5307f67a8714f16f58944bb12039 ipfs-api-get.html -\ No newline at end of file diff --git a/drafts/keccak-benchmarks.html b/drafts/keccak-benchmarks.html @@ -1,313 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> -<head> - <link rel="stylesheet" type="text/css" href="../theme/css/style.css" /> - <link rel="stylesheet" type="text/css" href="../theme/css/syntax.css" /> - <title>man bytes gnu - In search of a slim KECCAK dependency</title> - <meta charset="utf-8" /> - <meta name="generator" content="Pelican" /> - -</head> - -<body id="index" class="home"> - - <header id="banner" class="body"> - <h1>> <a href="../">man bytes gnu</a><span id="cursor">_</span></h1> - </header> - <nav id="menu"> - <ul> - <li><a href="../category/archiving.html">Archiving</a></li> - <li class="active"><a href="../category/code.html">Code</a></li> - <li><a href="../category/hygiene.html">Hygiene</a></li> - <li><a href="../category/offlining.html">Offlining</a></li> - <li><a href="../pages/identities.html">Identities</a></li> - <li><a href="../pages/shares.html">Shares</a></li> - - <li><a href="/tags.html">tags</a></li> - </ul> - <hr/> - </nav><!-- /#menu --> - <header> - <h1 class="entry-title top-body-title"> - <a href="../drafts/keccak-benchmarks.html" rel="bookmark" - title="Permalink to In search of a slim KECCAK dependency">In search of a slim KECCAK dependency</a></h1> - - - <div class="category meta"> - Posted -<time class="published" datetime="2021-04-18T15:01:00+02:00"> - Sun 18 April 2021 - </time> -in <a class="category" href="../category/code.html">code</a> - <a href="../tag/crypto.html">crypto</a> - <a href="../tag/hash.html">hash</a> - <a href="../tag/keccak.html">keccak</a> - <a href="../tag/sha3.html">sha3</a> - <a href="../tag/python.html">python</a> - <a href="../tag/c.html">c</a> - </div> - - <div class="neighbors meta"> - <ul> - </ul> - </div> - <div class="meta"> -<hr/> - </div> -</header> - -<section id="content" class="body"> - <div class="entry-content"> - <div class="section" id="implementations"> -<h2>Implementations</h2> -<div class="section" id="xkcp"> -<h3>XKCP</h3> -<ul class="simple"> -<li>Site: <a class="reference external" href="https://keccak.team/keccak_specs_summary.html">https://keccak.team/keccak_specs_summary.html</a></li> -<li>Code: <a class="reference external" href="https://github.com/xkcp/xkcp">https://github.com/xkcp/xkcp</a></li> -<li>Git hash: c438ee7b2736726f629da11b7012cffcf6b84fef</li> -</ul> -<pre class="code c literal-block"> -<span class="ln">0 </span><span class="cp">#include</span><span class="w"> </span><span class="cpf"><XKCP/KeccakHash.h></span><span class="cp"> -</span><span class="ln">1 </span><span class="cp"></span><span class="w"> -</span><span class="ln">2 </span><span class="w"></span><span class="kt">int</span><span class="w"> </span><span class="nf">wrap_keccak</span><span class="p">(</span><span class="k">const</span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="o">*</span><span class="n">data</span><span class="p">,</span><span class="w"> </span><span class="kt">size_t</span><span class="w"> </span><span class="n">bitlen</span><span class="p">,</span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="o">*</span><span class="n">zout</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"> -</span><span class="ln">3 </span><span class="w"> </span><span class="n">Keccak_HashInstance</span><span class="w"> </span><span class="n">instance</span><span class="p">;</span><span class="w"> -</span><span class="ln">4 </span><span class="w"> </span><span class="n">Keccak_HashInitialize</span><span class="p">(</span><span class="o">&</span><span class="n">instance</span><span class="p">,</span><span class="w"> </span><span class="mi">1088</span><span class="p">,</span><span class="w"> </span><span class="mi">512</span><span class="p">,</span><span class="w"> </span><span class="mi">256</span><span class="p">,</span><span class="w"> </span><span class="mh">0x01</span><span class="p">);</span><span class="w"> -</span><span class="ln">5 </span><span class="w"> </span><span class="n">Keccak_HashUpdate</span><span class="p">(</span><span class="o">&</span><span class="n">instance</span><span class="p">,</span><span class="w"> </span><span class="n">data</span><span class="p">,</span><span class="w"> </span><span class="p">(</span><span class="kt">int</span><span class="p">)</span><span class="n">bitlen</span><span class="p">);</span><span class="w"> -</span><span class="ln">6 </span><span class="w"> </span><span class="n">Keccak_HashFinal</span><span class="p">(</span><span class="o">&</span><span class="n">instance</span><span class="p">,</span><span class="w"> </span><span class="n">zout</span><span class="p">);</span><span class="w"> -</span><span class="ln">7 </span><span class="w"></span><span class="p">}</span> -</pre> -<pre class="code c literal-block"> -<span class="ln"> 0 </span><span class="cp">#include</span><span class="w"> </span><span class="cpf"><string.h></span><span class="cp"> -</span><span class="ln"> 1 </span><span class="cp">#include</span><span class="w"> </span><span class="cpf"><stdio.h></span><span class="cp"> -</span><span class="ln"> 2 </span><span class="cp">#include</span><span class="w"> </span><span class="cpf"><stdlib.h></span><span class="cp"> -</span><span class="ln"> 3 </span><span class="cp">#include</span><span class="w"> </span><span class="cpf"><time.h></span><span class="cp"> -</span><span class="ln"> 4 </span><span class="cp"></span><span class="w"> -</span><span class="ln"> 5 </span><span class="w"></span><span class="cp">#include</span><span class="w"> </span><span class="cpf"><XKCP/KeccakHash.h></span><span class="cp"> -</span><span class="ln"> 6 </span><span class="cp"></span><span class="w"> -</span><span class="ln"> 7 </span><span class="w"></span><span class="cp">#ifndef ROUNDS -</span><span class="ln"> 8 </span><span class="cp">#define ROUNDS 100000 -</span><span class="ln"> 9 </span><span class="cp">#endif -</span><span class="ln">10 </span><span class="cp"></span><span class="w"> -</span><span class="ln">11 </span><span class="w"></span><span class="k">extern</span><span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="nf">wrap_keccak</span><span class="p">(</span><span class="k">const</span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="o">*</span><span class="n">data</span><span class="p">,</span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="o">*</span><span class="n">zout</span><span class="p">);</span><span class="w"> -</span><span class="ln">12 </span><span class="w"> -</span><span class="ln">13 </span><span class="w"></span><span class="kt">void</span><span class="w"> </span><span class="nf">main</span><span class="p">()</span><span class="w"> </span><span class="p">{</span><span class="w"> -</span><span class="ln">14 </span><span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="n">rounds</span><span class="p">;</span><span class="w"> -</span><span class="ln">15 </span><span class="w"> </span><span class="kt">long</span><span class="w"> </span><span class="n">delta_sec</span><span class="p">;</span><span class="w"> -</span><span class="ln">16 </span><span class="w"> </span><span class="kt">long</span><span class="w"> </span><span class="n">delta_nsec</span><span class="p">;</span><span class="w"> -</span><span class="ln">17 </span><span class="w"> </span><span class="k">struct</span> <span class="nc">timespec</span><span class="w"> </span><span class="n">start</span><span class="p">;</span><span class="w"> -</span><span class="ln">18 </span><span class="w"> </span><span class="k">struct</span> <span class="nc">timespec</span><span class="w"> </span><span class="n">end</span><span class="p">;</span><span class="w"> -</span><span class="ln">19 </span><span class="w"> -</span><span class="ln">20 </span><span class="w"> </span><span class="n">Keccak_HashInstance</span><span class="w"> </span><span class="n">instance</span><span class="p">;</span><span class="w"> -</span><span class="ln">21 </span><span class="w"> -</span><span class="ln">22 </span><span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="n">i</span><span class="p">;</span><span class="w"> -</span><span class="ln">23 </span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="n">data</span><span class="p">[</span><span class="mi">3</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">{</span><span class="sc">'f'</span><span class="p">,</span><span class="w"> </span><span class="sc">'o'</span><span class="p">,</span><span class="w"> </span><span class="sc">'o'</span><span class="p">};</span><span class="w"> -</span><span class="ln">24 </span><span class="w"> -</span><span class="ln">25 </span><span class="w"> </span><span class="kt">unsigned</span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="n">buf</span><span class="p">[</span><span class="mi">256</span><span class="p">];</span><span class="w"> -</span><span class="ln">26 </span><span class="w"> </span><span class="n">memset</span><span class="p">(</span><span class="n">buf</span><span class="p">,</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"> </span><span class="mi">256</span><span class="p">);</span><span class="w"> -</span><span class="ln">27 </span><span class="w"> -</span><span class="ln">28 </span><span class="w"> </span><span class="k">for</span><span class="w"> </span><span class="p">(</span><span class="n">i</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o"><</span><span class="w"> </span><span class="mi">10000</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="o">++</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"> -</span><span class="ln">29 </span><span class="w"> </span><span class="n">wrap_keccak</span><span class="p">(</span><span class="n">data</span><span class="p">,</span><span class="w"> </span><span class="n">buf</span><span class="p">);</span><span class="w"> -</span><span class="ln">30 </span><span class="w"> </span><span class="p">}</span><span class="w"> -</span><span class="ln">31 </span><span class="w"> -</span><span class="ln">32 </span><span class="w"> </span><span class="n">clock_gettime</span><span class="p">(</span><span class="n">CLOCK_PROCESS_CPUTIME_ID</span><span class="p">,</span><span class="w"> </span><span class="o">&</span><span class="n">start</span><span class="p">);</span><span class="w"> -</span><span class="ln">33 </span><span class="w"> </span><span class="k">for</span><span class="w"> </span><span class="p">(</span><span class="n">i</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o"><</span><span class="w"> </span><span class="n">ROUNDS</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="o">++</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"> -</span><span class="ln">34 </span><span class="w"> </span><span class="n">wrap_keccak</span><span class="p">(</span><span class="n">data</span><span class="p">,</span><span class="w"> </span><span class="n">buf</span><span class="p">);</span><span class="w"> -</span><span class="ln">35 </span><span class="w"> </span><span class="p">}</span><span class="w"> -</span><span class="ln">36 </span><span class="w"> </span><span class="n">clock_gettime</span><span class="p">(</span><span class="n">CLOCK_PROCESS_CPUTIME_ID</span><span class="p">,</span><span class="w"> </span><span class="o">&</span><span class="n">end</span><span class="p">);</span><span class="w"> -</span><span class="ln">37 </span><span class="w"> -</span><span class="ln">38 </span><span class="w"> </span><span class="n">delta_sec</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">end</span><span class="p">.</span><span class="n">tv_sec</span><span class="w"> </span><span class="o">-</span><span class="w"> </span><span class="n">start</span><span class="p">.</span><span class="n">tv_sec</span><span class="p">;</span><span class="w"> -</span><span class="ln">39 </span><span class="w"> </span><span class="n">delta_nsec</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">end</span><span class="p">.</span><span class="n">tv_nsec</span><span class="w"> </span><span class="o">-</span><span class="w"> </span><span class="n">start</span><span class="p">.</span><span class="n">tv_nsec</span><span class="p">;</span><span class="w"> -</span><span class="ln">40 </span><span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="n">delta_nsec</span><span class="w"> </span><span class="o"><</span><span class="w"> </span><span class="mi">0</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"> -</span><span class="ln">41 </span><span class="w"> </span><span class="n">delta_sec</span><span class="o">++</span><span class="p">;</span><span class="w"> -</span><span class="ln">42 </span><span class="w"> </span><span class="n">delta_nsec</span><span class="w"> </span><span class="o">+=</span><span class="w"> </span><span class="mf">1000000000L</span><span class="p">;</span><span class="w"> -</span><span class="ln">43 </span><span class="w"> </span><span class="p">}</span><span class="w"> -</span><span class="ln">44 </span><span class="w"> </span><span class="n">printf</span><span class="p">(</span><span class="s">"%u.%09u</span><span class="se">\n</span><span class="s">"</span><span class="p">,</span><span class="w"> </span><span class="n">delta_sec</span><span class="p">,</span><span class="w"> </span><span class="n">delta_nsec</span><span class="p">);</span><span class="w"> -</span><span class="ln">45 </span><span class="w"> -</span><span class="ln">46 </span><span class="w"> </span><span class="k">for</span><span class="w"> </span><span class="p">(</span><span class="n">i</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o"><</span><span class="w"> </span><span class="mi">32</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="o">++</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"> -</span><span class="ln">47 </span><span class="w"> </span><span class="n">printf</span><span class="p">(</span><span class="s">"%02x"</span><span class="p">,</span><span class="w"> </span><span class="n">buf</span><span class="p">[</span><span class="n">i</span><span class="p">]);</span><span class="w"> -</span><span class="ln">48 </span><span class="w"> </span><span class="p">}</span><span class="w"> -</span><span class="ln">49 </span><span class="w"> </span><span class="n">printf</span><span class="p">(</span><span class="s">"</span><span class="se">\n</span><span class="s">"</span><span class="p">);</span><span class="w"> -</span><span class="ln">50 </span><span class="w"></span><span class="p">}</span> -</pre> -</div> -<div class="section" id="keccak-tiny"> -<h3>keccak_tiny</h3> -<ul class="simple"> -<li>Code: <a class="reference external" href="https://github.com/coruus/keccak-tiny">https://github.com/coruus/keccak-tiny</a></li> -<li>Git hash: 64b6647514212b76ae7bca0dea9b7b197d1d8186</li> -</ul> -<p>Needed to export <cite>hash</cite> function symbol, and add to header file.</p> -</div> -<div class="section" id="tiny-sha3"> -<h3>tiny_sha3</h3> -<ul class="simple"> -<li>Code <a class="footnote-reference" href="#id2" id="id1">[1]</a>: <a class="reference external" href="https://github.com/mjosaarinen/tiny_sha3">https://github.com/mjosaarinen/tiny_sha3</a></li> -<li>Git hash: dcbb3192047c2a721f5f851db591871d428036a9</li> -</ul> -<p>Had to change "padding" value in <tt class="docutils literal">sha3.c</tt></p> -<table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span class="normal">142</span></pre></div></td><td class="code"><div class="highlight"><pre><span></span><span class="w"> </span><span class="n">c</span><span class="o">-></span><span class="n">st</span><span class="p">.</span><span class="n">b</span><span class="p">[</span><span class="n">c</span><span class="o">-></span><span class="n">pt</span><span class="p">]</span><span class="w"> </span><span class="o">^=</span><span class="w"> </span><span class="mh">0x01</span><span class="p">;</span><span class="w"></span> -</pre></div> -</td></tr></table><div class="highlight"><pre><span></span><span class="n">sha3</span><span class="p">(</span><span class="n">data</span><span class="p">,</span><span class="w"> </span><span class="mi">3</span><span class="p">,</span><span class="w"> </span><span class="n">buf</span><span class="p">,</span><span class="w"> </span><span class="mi">32</span><span class="p">);</span><span class="w"></span> -</pre></div> -</div> -<div class="section" id="libkeccak"> -<h3>libkeccak</h3> -<ul class="simple"> -<li>Code: <a class="reference external" href="https://github.com/maandree/libkeccak">https://github.com/maandree/libkeccak</a></li> -<li>Git hash: 718b1a6ea1c44bcf15e55d3c265310e1cd9211fa</li> -</ul> -<div class="highlight"><pre><span></span>libkeccak_state_initialise(&state, &spec); -libkeccak_fast_update(&state, msg, 3); -libkeccak_fast_digest(&state, NULL, 0, 0, NULL, buf); -</pre></div> -</div> -</div> -<div class="section" id="results"> -<h2>Results</h2> -<ul class="simple"> -<li>Input: "foo"</li> -<li>Rounds: 100000</li> -</ul> -<!-- :XKCP generic64: - - Time: 0.041101298 - - Executable size: 37768 - - Archive size: 358558 --> -<!-- :keccak-tiny: - - Time: 0.046095483 - - Executable size: 49400 - - Object size: 35544 --> -<!-- :tiny_sha3 [1]_: - - Time: 0.080156921 - - Executable size: 16577 - - Object size: 6712 --> -<!-- :libkeccak: - - Time: 0.200222898 - - Executable size: 50152 - - Archive size: 78680 --> -<table border="1" class="docutils"> -<colgroup> -<col width="32%" /> -<col width="34%" /> -<col width="34%" /> -</colgroup> -<thead valign="bottom"> -<tr><th class="head">Implementation</th> -<th class="head">Execution time</th> -<th class="head">Executable size</th> -</tr> -</thead> -<tbody valign="top"> -<tr><td>XKCP</td> -<td><strong>0.041101298</strong></td> -<td>37768</td> -</tr> -<tr><td>keccak-tiny</td> -<td>0.046095483</td> -<td>49400</td> -</tr> -<tr><td>tiny_sha3</td> -<td>0.080156921</td> -<td><strong>16577</strong></td> -</tr> -<tr><td>libkeccak</td> -<td>0.200222898</td> -<td>50152</td> -</tr> -</tbody> -</table> -<blockquote> -<table class="docutils footnote" frame="void" id="id2" rules="none"> -<colgroup><col class="label" /><col /></colgroup> -<tbody valign="top"> -<tr><td class="label"><a class="fn-backref" href="#id1">[1]</a></td><td>Author claims it is not suited for production</td></tr> -</tbody> -</table> -</blockquote> -</div> -<div class="section" id="python"> -<h2>Python</h2> -<p>XKCP has a pure-python implementation, but it's hopelessly slow.</p> -<p>Added to <tt class="docutils literal">CompactFIPS202.py</tt>:</p> -<div class="highlight"><pre><span></span><span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s1">'__main__'</span><span class="p">:</span> - <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">10</span><span class="p">):</span> - <span class="n">a</span> <span class="o">=</span> <span class="n">Keccak</span><span class="p">(</span><span class="mi">1088</span><span class="p">,</span> <span class="mi">512</span><span class="p">,</span> <span class="sa">b</span><span class="s1">'foo'</span><span class="p">,</span> <span class="mh">0x01</span><span class="p">,</span> <span class="mi">32</span><span class="p">)</span> - - <span class="k">for</span> <span class="n">j</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span><span class="mi">4</span><span class="p">):</span> - <span class="n">a</span> <span class="o">=</span> <span class="mi">10</span><span class="o">**</span><span class="n">j</span> - <span class="n">start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">clock_gettime_ns</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">CLOCK_PROCESS_CPUTIME_ID</span><span class="p">)</span> - <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">a</span><span class="p">):</span> - <span class="n">b</span> <span class="o">=</span> <span class="n">Keccak</span><span class="p">(</span><span class="mi">1088</span><span class="p">,</span> <span class="mi">512</span><span class="p">,</span> <span class="sa">b</span><span class="s1">'foo'</span><span class="p">,</span> <span class="mh">0x01</span><span class="p">,</span> <span class="mi">32</span><span class="p">)</span> - <span class="n">end_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">clock_gettime_ns</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">CLOCK_PROCESS_CPUTIME_ID</span><span class="p">)</span> - <span class="nb">print</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="p">(</span><span class="n">end_time</span> <span class="o">-</span> <span class="n">start_time</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="mi">10</span><span class="o">**</span><span class="mi">9</span><span class="p">))</span> -</pre></div> -<pre class="literal-block"> -$ python CompactFIPS202.py -10 0.007536721 -100 0.066636149 -1000 0.64519725 -</pre> -<div class="section" id="wrapping-keccak-tiny"> -<h3>Wrapping keccak_tiny</h3> -<p>Implementing <tt class="docutils literal">XKCP</tt> in Python means creating two classes for the state and sponge structs. <tt class="docutils literal">keccak_tiny</tt> is much simpler because it is merely a function call with a couple of char buffers. But with ctypes ends up being very slow, too. <a class="footnote-reference" href="#id4" id="id3">[2]</a></p> -<pre class="code python literal-block"> -<span class="ln"> 0 </span><span class="kn">import</span> <span class="nn">ctypes</span> -<span class="ln"> 1 </span><span class="kn">import</span> <span class="nn">sys</span> -<span class="ln"> 2 </span><span class="kn">import</span> <span class="nn">time</span> -<span class="ln"> 3 </span> -<span class="ln"> 4 </span><span class="n">libkeccak</span> <span class="o">=</span> <span class="n">ctypes</span><span class="o">.</span><span class="n">CDLL</span><span class="p">(</span><span class="s1">'libkeccaktiny.so'</span><span class="p">)</span> -<span class="ln"> 5 </span> -<span class="ln"> 6 </span><span class="n">buf</span> <span class="o">=</span> <span class="n">ctypes</span><span class="o">.</span><span class="n">pointer</span><span class="p">((</span><span class="n">ctypes</span><span class="o">.</span><span class="n">c_char</span> <span class="o">*</span> <span class="mi">256</span><span class="p">)())</span> -<span class="ln"> 7 </span><span class="n">data</span> <span class="o">=</span> <span class="n">ctypes</span><span class="o">.</span><span class="n">create_string_buffer</span><span class="p">(</span><span class="sa">b</span><span class="s1">'foo'</span><span class="p">)</span> -<span class="ln"> 8 </span> -<span class="ln"> 9 </span><span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">10000</span><span class="p">):</span> -<span class="ln">10 </span> <span class="n">libkeccak</span><span class="o">.</span><span class="n">hash</span><span class="p">(</span><span class="n">buf</span><span class="p">,</span> <span class="mi">32</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">200</span><span class="o">-</span><span class="mi">64</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span> -<span class="ln">11 </span> -<span class="ln">12 </span><span class="n">start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">clock_gettime_ns</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">CLOCK_PROCESS_CPUTIME_ID</span><span class="p">)</span> -<span class="ln">13 </span><span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">100000</span><span class="p">):</span> -<span class="ln">14 </span> <span class="n">libkeccak</span><span class="o">.</span><span class="n">hash</span><span class="p">(</span><span class="n">buf</span><span class="p">,</span> <span class="mi">32</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">200</span><span class="o">-</span><span class="mi">64</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span> -<span class="ln">15 </span><span class="n">end_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">clock_gettime_ns</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">CLOCK_PROCESS_CPUTIME_ID</span><span class="p">)</span> -<span class="ln">16 </span> -<span class="ln">17 </span><span class="nb">print</span><span class="p">((</span><span class="n">end_time</span> <span class="o">-</span> <span class="n">start_time</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="mi">10</span><span class="o">**</span><span class="mi">9</span><span class="p">))</span> -<span class="ln">18 </span><span class="nb">print</span><span class="p">(</span><span class="n">buf</span><span class="o">.</span><span class="n">contents</span><span class="o">.</span><span class="n">value</span><span class="o">.</span><span class="n">hex</span><span class="p">())</span> -</pre> -<pre class="literal-block"> -$ gcc -L. -D"memset_s(W,WL,V,OL)=memset(W,V,OL)" -shared keccak-tiny.c -o libkeccaktiny.so -$ LD_LIBRARY_PATH=. python wrap.py -0.95114457 -41b1a0649752af1b28b3dc29a1556eee781e4a4c3a1f7f53f90fa834de098c4d -</pre> -<p>Compare with <tt class="docutils literal">pysha3</tt>:</p> -<div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">sha3</span> -<span class="n">h</span> <span class="o">=</span> <span class="n">sha3</span><span class="o">.</span><span class="n">keccak_256</span><span class="p">()</span> -<span class="n">h</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="sa">b</span><span class="s1">'foo'</span><span class="p">)</span> -<span class="n">b</span> <span class="o">=</span> <span class="n">h</span><span class="o">.</span><span class="n">digest</span><span class="p">()</span> -</pre></div> -<pre class="literal-block"> -0.162829927 -</pre> -<!-- --> -<blockquote> -<table class="docutils footnote" frame="void" id="id4" rules="none"> -<colgroup><col class="label" /><col /></colgroup> -<tbody valign="top"> -<tr><td class="label"><a class="fn-backref" href="#id3">[2]</a></td><td>for some reason when on battery; 1.74030088</td></tr> -</tbody> -</table> -</blockquote> -</div> -</div> - - </div><!-- /.entry-content --> - <footer class="meta"> - </footer> - -</section> - <footer id="contentinfo" class="body"> - <hr/> - <address id="about" class="vcard body"> - Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> - </address><!-- /#about --> - </footer><!-- /#contentinfo --> -</body> -</html> -\ No newline at end of file diff --git a/drafts/keccak-benchmarks.sha256 b/drafts/keccak-benchmarks.sha256 @@ -1 +0,0 @@ -04a880946249d7ed07447320426c765a3b77ec8f39e3f4e6cdf708d9847fde57 keccak-benchmarks.html -\ No newline at end of file diff --git a/drafts/python-pipe-args.html b/drafts/python-pipe-args.html @@ -1,91 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> -<head> - <link rel="stylesheet" type="text/css" href="../theme/css/style.css" /> - <link rel="stylesheet" type="text/css" href="../theme/css/syntax.css" /> - <title>man bytes gnu - Python pipe args</title> - <meta charset="utf-8" /> - <meta name="generator" content="Pelican" /> - -</head> - -<body id="index" class="home"> - - <header id="banner" class="body"> - <h1>> <a href="../">man bytes gnu</a><span id="cursor">_</span></h1> - </header> - <nav id="menu"> - <ul> - <li><a href="../category/archiving.html">Archiving</a></li> - <li class="active"><a href="../category/code.html">Code</a></li> - <li><a href="../category/hygiene.html">Hygiene</a></li> - <li><a href="../category/offlining.html">Offlining</a></li> - <li><a href="../pages/identities.html">Identities</a></li> - <li><a href="../pages/shares.html">Shares</a></li> - - <li><a href="/tags.html">tags</a></li> - </ul> - <hr/> - </nav><!-- /#menu --> - <header> - <h1 class="entry-title top-body-title"> - <a href="../drafts/python-pipe-args.html" rel="bookmark" - title="Permalink to Python pipe args">Python pipe args</a></h1> - - - <div class="category meta"> - Posted -<time class="published" datetime="2021-06-09T22:08:00+02:00"> - Wed 09 June 2021 - </time> -in <a class="category" href="../category/code.html">code</a> - <a href="../tag/python.html">python</a> - <a href="../tag/cli.html">cli</a> - </div> - - <div class="neighbors meta"> - <ul> - </ul> - </div> - <div class="meta"> -<hr/> - </div> -</header> - -<section id="content" class="body"> - <div class="entry-content"> - <pre class="code ini literal-block"> -<span class="na">import sys</span> -<span class="na">import select</span> -<span class="na">import argparse</span> - - <span class="na">def stdincheck():</span> - <span class="na">h</span> <span class="o">=</span> <span class="s">select.select([sys.stdin], [], [], 0) # 0 = poll, noblock</span> - <span class="na">if len(h[0]) > 0:</span> - <span class="na">v</span> <span class="o">=</span> <span class="s">h[0][0].read()</span> - <span class="na">return v.rstrip()</span> - <span class="na">return None</span> - - <span class="na">argparser</span> <span class="o">=</span> <span class="s">argparse.ArgumentParser()</span> - <span class="na">argparser.add_argument('val', nargs</span><span class="o">=</span><span class="s">'?', default=stdincheck(), type=str, help='input val')</span> - <span class="na">args</span> <span class="o">=</span> <span class="s">argparser.parse_args(sys.argv[1:])</span> - - <span class="na">if args.val</span> <span class="o">=</span><span class="s">= None:</span> - <span class="na">argparser.error('value but be specified on stdin or as first positional argument')</span> - - <span class="na">print(args.val)</span> -</pre> - - </div><!-- /.entry-content --> - <footer class="meta"> - </footer> - -</section> - <footer id="contentinfo" class="body"> - <hr/> - <address id="about" class="vcard body"> - Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> - </address><!-- /#about --> - </footer><!-- /#contentinfo --> -</body> -</html> -\ No newline at end of file diff --git a/drafts/python-pipe-args.sha256 b/drafts/python-pipe-args.sha256 @@ -1 +0,0 @@ -10868bed68bd00a555f81ce293f77081a691a73d71ba256e112a6fcbb8a52b27 python-pipe-args.html -\ No newline at end of file diff --git a/drafts/python-virtualenv-interpreter.html b/drafts/python-virtualenv-interpreter.html @@ -1,76 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> -<head> - <link rel="stylesheet" type="text/css" href="../theme/css/style.css" /> - <link rel="stylesheet" type="text/css" href="../theme/css/syntax.css" /> - <title>man bytes gnu - Virtualenv with different pythons</title> - <meta charset="utf-8" /> - <meta name="generator" content="Pelican" /> - -</head> - -<body id="index" class="home"> - - <header id="banner" class="body"> - <h1>> <a href="../">man bytes gnu</a><span id="cursor">_</span></h1> - </header> - <nav id="menu"> - <ul> - <li><a href="../category/archiving.html">Archiving</a></li> - <li class="active"><a href="../category/code.html">Code</a></li> - <li><a href="../category/hygiene.html">Hygiene</a></li> - <li><a href="../category/offlining.html">Offlining</a></li> - <li><a href="../pages/identities.html">Identities</a></li> - <li><a href="../pages/shares.html">Shares</a></li> - - <li><a href="/tags.html">tags</a></li> - </ul> - <hr/> - </nav><!-- /#menu --> - <header> - <h1 class="entry-title top-body-title"> - <a href="../drafts/python-virtualenv-interpreter.html" rel="bookmark" - title="Permalink to Virtualenv with different pythons">Virtualenv with different pythons</a></h1> - - - <div class="category meta"> - Posted -<time class="published" datetime="2021-10-17T15:45:00+02:00"> - Sun 17 October 2021 - </time> -in <a class="category" href="../category/code.html">code</a> - <a href="../tag/docker.html">docker</a> - <a href="../tag/networking.html">networking</a> - <a href="../tag/iptables.html">iptables</a> - <a href="../tag/iproute.html">iproute</a> - </div> - - <div class="neighbors meta"> - <ul> - </ul> - </div> - <div class="meta"> -<hr/> - </div> -</header> - -<section id="content" class="body"> - <div class="entry-content"> - <!-- (.venv) cic-ussd $ export CPPFLAGS="-I/home/lash/src/ext/python/Python-3.8.6/Include -I/home/lash/src/ext/python/Python-3.8.6" -(.venv) cic-ussd $ export CFLAGS="-I/home/lash/src/ext/python/Python-3.8.6/Include -I/home/lash/src/ext/python/Python-3.8.6" -(.venv) cic-ussd $ pip download - -global-option=build_ext - -global-option "-I/home/lash/src/ext/Python-3.8.6/Include" - -index-url https://pypi.org/simple - -extra-index-url http://localhost/python - -no-binary :all: -d deps/ -r requirements.txt -bcrypt uses cffi, fails with missing header files Python.h and pyconfig.h --> - - </div><!-- /.entry-content --> - <footer class="meta"> - </footer> - -</section> - <footer id="contentinfo" class="body"> - <hr/> - <address id="about" class="vcard body"> - Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> - </address><!-- /#about --> - </footer><!-- /#contentinfo --> -</body> -</html> -\ No newline at end of file diff --git a/drafts/python-virtualenv-interpreter.sha256 b/drafts/python-virtualenv-interpreter.sha256 @@ -1 +0,0 @@ -bb7f404e49154a8317c793c21e6de6ae39cc147e039f9f0ff11d15bb15ae441c python-virtualenv-interpreter.html -\ No newline at end of file diff --git a/drafts/qemu-host-fw.html b/drafts/qemu-host-fw.html @@ -1,77 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> -<head> - <link rel="stylesheet" type="text/css" href="../theme/css/style.css" /> - <link rel="stylesheet" type="text/css" href="../theme/css/syntax.css" /> - <title>man bytes gnu - Isolating qemu networking from host</title> - <meta charset="utf-8" /> - <meta name="generator" content="Pelican" /> - -</head> - -<body id="index" class="home"> - - <header id="banner" class="body"> - <h1>> <a href="../">man bytes gnu</a><span id="cursor">_</span></h1> - </header> - <nav id="menu"> - <ul> - <li><a href="../category/archiving.html">Archiving</a></li> - <li><a href="../category/code.html">Code</a></li> - <li><a href="../category/hygiene.html">Hygiene</a></li> - <li class="active"><a href="../category/offlining.html">Offlining</a></li> - <li><a href="../pages/identities.html">Identities</a></li> - <li><a href="../pages/shares.html">Shares</a></li> - - <li><a href="/tags.html">tags</a></li> - </ul> - <hr/> - </nav><!-- /#menu --> - <header> - <h1 class="entry-title top-body-title"> - <a href="../drafts/qemu-host-fw.html" rel="bookmark" - title="Permalink to Isolating qemu networking from host">Isolating qemu networking from host</a></h1> - - - <div class="category meta"> - Posted -<time class="published" datetime="2022-01-02T13:17:55+01:00"> - Sun 02 January 2022 - </time> -in <a class="category" href="../category/offlining.html">offlining</a> - <a href="../tag/qemu.html">qemu</a> - <a href="../tag/network.html">network</a> - <a href="../tag/iptables.html">iptables</a> - </div> - - <div class="neighbors meta"> - <ul> - </ul> - </div> - <div class="meta"> -<hr/> - </div> -</header> - -<section id="content" class="body"> - <div class="entry-content"> - <!-- code-block: bash - -iptables -N QEMU -iptables -A OUTPUT -o lo -m owner - -gid-owner 1009 -j QEMU -iptables -A QEMU -d 127.0.0.1/32 -j LOG -iptables -A QEMU -d 127.0.0.1/32 -j DROP --> - - </div><!-- /.entry-content --> - <footer class="meta"> - </footer> - -</section> - <footer id="contentinfo" class="body"> - <hr/> - <address id="about" class="vcard body"> - Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> - </address><!-- /#about --> - </footer><!-- /#contentinfo --> -</body> -</html> -\ No newline at end of file diff --git a/drafts/qemu-host-fw.sha256 b/drafts/qemu-host-fw.sha256 @@ -1 +0,0 @@ -5fe95895cc160a284b36ad5c34c8536b2dc2130938305333a52b3d5e03d788a1 qemu-host-fw.html -\ No newline at end of file diff --git a/drafts/qemu-raw-partitions.html b/drafts/qemu-raw-partitions.html @@ -1,102 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> -<head> - <link rel="stylesheet" type="text/css" href="../theme/css/style.css" /> - <link rel="stylesheet" type="text/css" href="../theme/css/syntax.css" /> - <title>man bytes gnu - Working with raw qemu partitions</title> - <meta charset="utf-8" /> - <meta name="generator" content="Pelican" /> - -</head> - -<body id="index" class="home"> - - <header id="banner" class="body"> - <h1>> <a href="../">man bytes gnu</a><span id="cursor">_</span></h1> - </header> - <nav id="menu"> - <ul> - <li><a href="../category/archiving.html">Archiving</a></li> - <li><a href="../category/code.html">Code</a></li> - <li><a href="../category/hygiene.html">Hygiene</a></li> - <li class="active"><a href="../category/offlining.html">Offlining</a></li> - <li><a href="../pages/identities.html">Identities</a></li> - <li><a href="../pages/shares.html">Shares</a></li> - - <li><a href="/tags.html">tags</a></li> - </ul> - <hr/> - </nav><!-- /#menu --> - <header> - <h1 class="entry-title top-body-title"> - <a href="../drafts/qemu-raw-partitions.html" rel="bookmark" - title="Permalink to Working with raw qemu partitions">Working with raw qemu partitions</a></h1> - - - <div class="category meta"> - Posted -<time class="published" datetime="2022-01-02T12:09:55+01:00"> - Sun 02 January 2022 - </time> -in <a class="category" href="../category/offlining.html">offlining</a> - <a href="../tag/qemu.html">qemu</a> - <a href="../tag/fdisk.html">fdisk</a> - </div> - - <div class="neighbors meta"> - <ul> - </ul> - </div> - <div class="meta"> -<hr/> - </div> -</header> - -<section id="content" class="body"> - <div class="entry-content"> - <!-- code-block: bash - -$ fdisk -l raw.img -$ qemu-image resize raw.img +2G -$ fdisk -l raw.img -Disk raw.img: 4 GiB, 4294967296 bytes, 8388608 sectors -Units: sectors of 1 * 512 = 512 bytes -Sector size (logical/physical): 512 bytes / 512 bytes -I/O size (minimum/optimal): 512 bytes / 512 bytes -Disklabel type: dos -Disk identifier: 0x79c2484b - -Device Boot Start End Sectors Size Id Type -raw.img1 * 2048 2193407 2191360 1G 83 Linux -raw.img2 2195454 4192255 1996802 975M 5 Extended -raw.img3 4192256 8388607 4196352 2G 83 Linux -raw.img5 2195456 4192255 1996800 975M 82 Linux swap / Solaris - -Partition table entries are not in disk order. -$ mke2fs -E offset=$(4192256*512) raw.img -$ sudo mount -o loop,offset=$(calc 2048*512),sizelimit=$(calc 2191360*512) /mnt/a -$ sudo mount -o loop,offset=$(calc 4192256*512) /mnt/b -$ sudo umount /mnt/b -$ losetup -f -P raw.img -$ lsblk -no NAME,UUID /dev/loop0 -loop0 -├─loop0p1 34b73576-e8c6-4ffb-a503-a9828639b5e9 -├─loop0p2 -├─loop0p3 ff6235cc-2f72-4551-a3d3-73d8df2e21b9 -└─loop0p5 0255cf3e-b10b-4522-8cfe-9ef8e2fc2d6a -$ echo "UUID=ff6235cc-2f72-4551-a3d3-73d8df2e21b9 /usr ext4 errors=remount-ro 0 2" >> /mnt/a/etc/fstab -$ sudo umount /mnt/a --> - - </div><!-- /.entry-content --> - <footer class="meta"> - </footer> - -</section> - <footer id="contentinfo" class="body"> - <hr/> - <address id="about" class="vcard body"> - Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> - </address><!-- /#about --> - </footer><!-- /#contentinfo --> -</body> -</html> -\ No newline at end of file diff --git a/drafts/qemu-raw-partitions.sha256 b/drafts/qemu-raw-partitions.sha256 @@ -1 +0,0 @@ -3c756e008d33eb94c704650e80b729e3a4850b1c57e4cc53e55b39de086c89f8 qemu-raw-partitions.html -\ No newline at end of file diff --git a/drafts/swarm-qemu.html b/drafts/swarm-qemu.html @@ -1,103 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> -<head> - <link rel="stylesheet" type="text/css" href="../theme/css/style.css" /> - <link rel="stylesheet" type="text/css" href="../theme/css/syntax.css" /> - <title>man bytes gnu - Self-sufficient swarm on qemu</title> - <meta charset="utf-8" /> - <meta name="generator" content="Pelican" /> - -</head> - -<body id="index" class="home"> - - <header id="banner" class="body"> - <h1>> <a href="../">man bytes gnu</a><span id="cursor">_</span></h1> - </header> - <nav id="menu"> - <ul> - <li><a href="../category/archiving.html">Archiving</a></li> - <li><a href="../category/code.html">Code</a></li> - <li class="active"><a href="../category/hygiene.html">Hygiene</a></li> - <li><a href="../category/offlining.html">Offlining</a></li> - <li><a href="../pages/identities.html">Identities</a></li> - <li><a href="../pages/shares.html">Shares</a></li> - - <li><a href="/tags.html">tags</a></li> - </ul> - <hr/> - </nav><!-- /#menu --> - <header> - <h1 class="entry-title top-body-title"> - <a href="../drafts/swarm-qemu.html" rel="bookmark" - title="Permalink to Self-sufficient swarm on qemu">Self-sufficient Swarm: Self-sufficient swarm on qemu</a></h1> - - - <div class="category meta"> - Posted -<time class="published" datetime="2021-06-23T12:49:00+02:00"> - Wed 23 June 2021 - </time> -in <a class="category" href="../category/hygiene.html">hygiene</a> - <a href="../tag/swarm.html">swarm</a> - <a href="../tag/golang.html">golang</a> - <a href="../tag/qemu.html">qemu</a> - </div> - - <div class="neighbors meta"> - <ul> - </ul> - </div> - <div class="entry-series meta"> -Part 1 from the series "Self-sufficient Swarm" - <ol class="entry-series-parts"> - </ol> - </div> - <div class="meta"> -<hr/> - </div> -</header> - -<section id="content" class="body"> - <div class="entry-content"> - <div class="section" id="base-debian-qemu-install"> -<h2>base debian qemu install</h2> -<div class="highlight"><pre><span></span>dd <span class="k">if</span><span class="o">=</span>/dev/zero <span class="nv">of</span><span class="o">=</span>debootstrap.bin <span class="nv">count</span><span class="o">=</span><span class="m">10485760</span> <span class="nv">bs</span><span class="o">=</span><span class="m">512</span> -mkfs.ext4 debootstrap.bin -mount -o loop debootstrap.bin /mnt -debootstrap --arch<span class="o">=</span>amd64 buster /mnt -</pre></div> -<div class="highlight"><pre><span></span>qemu-system-x86_64 -cdrom /mnt/iso/debian-10.9.0-amd64-xfce-CD-1.iso -hda debian.bin -hdb clef.bin -m 4g --enable-kvm -virtfs local,path<span class="o">=</span>/mnt,id<span class="o">=</span>foo,readonly<span class="o">=</span>on,mount_tag<span class="o">=</span>bar,security_model<span class="o">=</span>passthrough -</pre></div> -<div class="highlight"><pre><span></span>mount -t 9p -o <span class="nv">trans</span><span class="o">=</span>virtio mount_tag /mnt/resource -oversion<span class="o">=</span>9p2000.L,posixacl,msize<span class="o">=</span><span class="m">104857600</span>,cache<span class="o">=</span>loose -</pre></div> -</div> -<div class="section" id="clef-swarm-provisions"> -<h2>clef/swarm provisions</h2> -</div> -<div class="section" id="ethereum-node-provisions"> -<h2>ethereum node provisions</h2> -</div> -<div class="section" id="xdai-requisites"> -<h2>xdai requisites</h2> -</div> -<div class="section" id="mainchain-requisites"> -<h2>mainchain requisites</h2> -</div> -<div class="section" id="orchestration"> -<h2>orchestration</h2> -</div> - - </div><!-- /.entry-content --> - <footer class="meta"> - </footer> - -</section> - <footer id="contentinfo" class="body"> - <hr/> - <address id="about" class="vcard body"> - Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> - </address><!-- /#about --> - </footer><!-- /#contentinfo --> -</body> -</html> -\ No newline at end of file diff --git a/drafts/swarm-qemu.sha256 b/drafts/swarm-qemu.sha256 @@ -1 +0,0 @@ -3a308c92c4aeca24fe79270bbe7769df46ee37fd39864ceaa78de32ee18221a1 swarm-qemu.html -\ No newline at end of file diff --git a/e37125a796b63784152584d8a08a4399c56f488978c5194cc33cc4af10cdbce7 b/e37125a796b63784152584d8a08a4399c56f488978c5194cc33cc4af10cdbce7 @@ -0,0 +1,216 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <link rel="stylesheet" type="text/css" href="./theme/css/style.css" /> + <link rel="stylesheet" type="text/css" href="./theme/css/syntax.css" /> + <title>man bytes gnu - Local python repository</title> + <meta charset="utf-8" /> + <meta name="generator" content="Pelican" /> + +</head> + +<body id="index" class="home"> + + <header id="banner" class="body"> + <h1>> <a href="./">man bytes gnu</a><span id="cursor">_</span></h1> + </header> + <nav id="menu"> + <ul> + <li><a href="./category/archiving.html">Archiving</a></li> + <li><a href="./category/code.html">Code</a></li> + <li><a href="./category/hygiene.html">Hygiene</a></li> + <li class="active"><a href="./category/offlining.html">Offlining</a></li> + <li><a href="./pages/identities.html">Identities</a></li> + <li><a href="./pages/shares.html">Shares</a></li> + + <li><a href="/tags.html">tags</a></li> + </ul> + <hr/> + </nav><!-- /#menu --> + <header> + <h1 class="entry-title top-body-title"> + <a href="./docker-offline-2-python.html" rel="bookmark" + title="Permalink to Local python repository">Offline Docker: Local python repository</a></h1> + + + <div class="category meta"> + Posted +<time class="published" datetime="2021-04-26T07:55:00+02:00"> + Mon 26 April 2021 + </time> +in <a class="category" href="./category/offlining.html">offlining</a> + <a href="./tag/docker.html">docker</a> + <a href="./tag/python.html">python</a> + <a href="./tag/devops.html">devops</a> + </div> + + <div class="neighbors meta"> + <ul> + <li> + Previous: <a href="./docker-offline-1-routing.html"> + The routing to freedom + </a> + </li> + | + + <li> + Next: <a href="./web-snapshot.html"> + Proving what you link to + </a> + </li> + </ul> + </div> + <div class="entry-series meta"> +Part 2 from the series "Offline Docker" + <ol class="entry-series-parts"> + <li>| <a href="docker-offline-1-routing.html" title="The routing to freedom ">Part 1</a></li> + <li>| <a href="docker-offline-3-npm.html" title="Local npm repository ">Part 3</a></li> + </ol> + </div> + <div class="meta"> +<hr/> + </div> +</header> + +<section id="content" class="body"> + <div class="entry-content"> + <!-- .. CAUTION:: + + This is a purely technical article. You should probably be `this geeky <https://g33k.holbrook.no/c18beedd>`_ to continue reading. --> +<p>In the previous part of this series we were able to connect a Docker network to a virtual interface on our host, neither of which have access to the internet. That means we are ready to host content for the container locally. And we will start out with creating a local Python repository.</p> +<div class="section" id="packaging-the-packages"> +<h2>Packaging the packages</h2> +<p>I'll be so bold as to assume that you are using <tt class="docutils literal">pip</tt> to manage your packages. It gives you not only the option to <em>install</em> packages, but merely <em>download</em> them to storage aswell. So let's do that and try to serve the packages.</p> +<div class="highlight"><pre><span></span>$ pip download faker +<span class="o">[</span>...<span class="o">]</span> +Successfully downloaded faker python-dateutil six text-unidecode +$ ls +Faker-8.1.0-py3-none-any.whl python_dateutil-2.8.1-py2.py3-none-any.whl six-1.15.0-py2.py3-none-any.whl text_unidecode-1.3-py2.py3-none-any.whl +$ python -m RangeHTTPServer +</pre></div> +<div class="highlight"><pre><span></span>$ python -m venv .venv +$ . .venv/bin/activate +<span class="o">(</span>.venv<span class="o">)</span> $ pip install --index http://localhost:8000 faker +Looking <span class="k">in</span> indexes: http://localhost:8000 +ERROR: Could not find a version that satisfies the requirement faker <span class="o">(</span>from versions: none<span class="o">)</span> +ERROR: No matching distribution found <span class="k">for</span> faker +</pre></div> +<p>Dag nabbit. Apparently not that simple. And indeed, if we read up on the spec for <a class="reference external" href="https://www.python.org/dev/peps/pep-0503">PEP 503 spec for Simple Repository API</a>, we learn that we are going to stick those package files under directories named after the packages.</p> +<p>Well, nothing that a bit of bash scripting can't sort out:</p> +<pre class="code bash literal-block"> +<span class="ch">#!/usr/bin/env +</span> +<span class="nv">s</span><span class="o">=</span><span class="nv">$1</span> +<span class="nv">d</span><span class="o">=</span><span class="nv">$2</span> +<span class="k">if</span> <span class="o">[</span> -z <span class="nv">$d</span> <span class="o">]</span><span class="p">;</span> <span class="k">then</span> + ><span class="p">&</span><span class="m">2</span> <span class="nb">echo</span> <span class="s2">"usage: pep503.sh <source_dir> <dest_dir>"</span> + <span class="nb">exit</span> <span class="m">1</span> +<span class="k">fi</span> + +<span class="k">for</span> df <span class="k">in</span> <span class="sb">`</span>find <span class="nv">$s</span> -name <span class="s2">"*.whl"</span> -type f<span class="sb">`</span><span class="p">;</span> <span class="k">do</span> + <span class="nv">f</span><span class="o">=</span><span class="sb">`</span>basename <span class="nv">$df</span><span class="sb">`</span> + <span class="nv">pd</span><span class="o">=</span><span class="sb">`</span><span class="nb">echo</span> <span class="nv">$f</span> <span class="p">|</span> sed -e <span class="s2">"s/^\(.*\)-[[:digit:]]*\.[[:digit:]].*</span>$<span class="s2">/\1/g"</span> <span class="p">|</span> tr <span class="s2">"[:upper:]"</span> <span class="s2">"[:lower:]"</span> <span class="p">|</span> tr <span class="s2">"_"</span> <span class="s2">"-"</span><span class="sb">`</span> + mkdir -v <span class="nv">$d</span>/<span class="nv">$pd</span> + cp -v <span class="nv">$df</span> <span class="nv">$d</span>/<span class="nv">$pd</span>/ +<span class="k">done</span> +<span class="k">for</span> df <span class="k">in</span> <span class="sb">`</span>find <span class="nv">$s</span> -name <span class="s2">"*.tar.gz"</span> -type f<span class="sb">`</span><span class="p">;</span> <span class="k">do</span> + <span class="nv">f</span><span class="o">=</span><span class="sb">`</span>basename <span class="nv">$df</span><span class="sb">`</span> + <span class="nv">pd</span><span class="o">=</span><span class="sb">`</span><span class="nb">echo</span> <span class="nv">$f</span> <span class="p">|</span> sed -e <span class="s2">"s/^\(.*\)-[[:digit:]]*\.[[:digit:]].*</span>$<span class="s2">/\1/g"</span> <span class="p">|</span> tr <span class="s2">"[:upper:]"</span> <span class="s2">"[:lower:]"</span> <span class="p">|</span> tr <span class="s2">"_"</span> <span class="s2">"-"</span><span class="sb">`</span> + mkdir -v <span class="nv">$d</span>/<span class="nv">$pd</span> + cp -v <span class="nv">$df</span> <span class="nv">$d</span>/<span class="nv">$pd</span>/ +<span class="k">done</span> +<span class="k">for</span> df <span class="k">in</span> <span class="sb">`</span>find <span class="nv">$s</span> -name <span class="s2">"*.zip"</span> -type f<span class="sb">`</span><span class="p">;</span> <span class="k">do</span> + <span class="nv">f</span><span class="o">=</span><span class="sb">`</span>basename <span class="nv">$df</span><span class="sb">`</span> + <span class="nv">pd</span><span class="o">=</span><span class="sb">`</span><span class="nb">echo</span> <span class="nv">$f</span> <span class="p">|</span> sed -e <span class="s2">"s/^\(.*\)-[[:digit:]]*\.[[:digit:]].*</span>$<span class="s2">/\1/g"</span> <span class="p">|</span> tr <span class="s2">"[:upper:]"</span> <span class="s2">"[:lower:]"</span> <span class="p">|</span> tr <span class="s2">"_"</span> <span class="s2">"-"</span><span class="sb">`</span> + mkdir -v <span class="nv">$d</span>/<span class="nv">$pd</span> + cp -v <span class="nv">$df</span> <span class="nv">$d</span>/<span class="nv">$pd</span>/ +<span class="k">done</span> +</pre> +<p>Armed with this, let's try again:</p> +<div class="highlight"><pre><span></span>$ sh /home/lash/bin/shell/pep503.sh . packages +$ mkdir packages +$ bash /home/lash/bin/shell/pep503.sh . packages +mkdir: created directory <span class="s1">'packages/text-unidecode'</span> +<span class="s1">'./text_unidecode-1.3-py2.py3-none-any.whl'</span> -> <span class="s1">'packages/text-unidecode/text_unidecode-1.3-py2.py3-none-any.whl'</span> +mkdir: created directory <span class="s1">'packages/six'</span> +<span class="s1">'./six-1.15.0-py2.py3-none-any.whl'</span> -> <span class="s1">'packages/six/six-1.15.0-py2.py3-none-any.whl'</span> +mkdir: created directory <span class="s1">'packages/python-dateutil'</span> +<span class="s1">'./python_dateutil-2.8.1-py2.py3-none-any.whl'</span> -> <span class="s1">'packages/python-dateutil/python_dateutil-2.8.1-py2.py3-none-any.whl'</span> +mkdir: created directory <span class="s1">'packages/faker'</span> +<span class="s1">'./Faker-8.1.0-py3-none-any.whl'</span> -> <span class="s1">'packages/faker/Faker-8.1.0-py3-none-any.whl'</span> +$ find packages/ -type f +packages/faker/Faker-8.1.0-py3-none-any.whl +packages/python-dateutil/python_dateutil-2.8.1-py2.py3-none-any.whl +packages/six/six-1.15.0-py2.py3-none-any.whl +packages/text-unidecode/text_unidecode-1.3-py2.py3-none-any.whl +k $ python -m RangeHTTPServer +</pre></div> +<div class="highlight"><pre><span></span>$ pip install --index http://localhost:8000/packages faker +Looking <span class="k">in</span> indexes: http://localhost:8000/packages +Collecting faker +<span class="o">[</span>...<span class="o">]</span> +Successfully installed faker-8.1.0 python-dateutil-2.8.1 six-1.15.0 text-unidecode-1.3 +</pre></div> +</div> +<div class="section" id="extra-prepping"> +<h2>Extra prepping</h2> +<p>There are some basic packages you will most always need, and which <tt class="docutils literal">pip</tt> often will expect to find in at least one of its available repositories, even regardless of whether its installed or not. If you don't have this on your local offline repository when the internet goes <em>poof</em>, then that will block any build you are trying to do.</p> +<p>Let's make sure we have the packages around all the time:</p> +<div class="highlight"><pre><span></span>$ pip download pip setuptools setuptools-markdown wheel +<span class="o">[</span>...<span class="o">]</span> +$ bash /home/lash/bin/shell/pep503.sh . packages +<span class="o">[</span>...<span class="o">]</span> +</pre></div> +</div> +<div class="section" id="choosing-a-server"> +<h2>Choosing a server</h2> +<p>As I tend to favor the classics, I still use <tt class="docutils literal">Apache Web Server</tt> to host stuff to my local environment. One practical (if not all too safe) thing about it is that it will automatically bind to all interfaces. So to make the repository available, we simply link or add the <tt class="docutils literal">packages</tt> directory to the virtual root, and restart the server e.g. with</p> +<div class="highlight"><pre><span></span>systemctl restart httpd +</pre></div> +<p>Of course you can use any HTTP server you like, as long as you know how to bind it to the virtual interface.</p> +<p>To verify that the service is providing what's needed, simply point your web browser to the location, e.g.</p> +<div class="highlight"><pre><span></span>lynx http://10.1.2.1/packages/ +</pre></div> +</div> +<div class="section" id="serving-the-packages"> +<h2>Serving the packages</h2> +<p>Now that we are all prepped, the next step is to do install packages from within a Docker container.</p> +<p>Let's start with an Archlinux base image with basic python provisions.</p> +<pre class="code docker literal-block"> +<span class="k">FROM</span> <span class="s">archlinux:latest</span> + +<span class="k">RUN</span> pacman -Sy <span class="o">&&</span> <span class="se">\ +</span> pacman -S --noconfirm python python-pip +</pre> +<p>Build and tag it with <tt class="docutils literal">pythonbase</tt>, and then let's add the Dockerfile to test the repository:</p> +<div class="highlight"><pre><span></span><span class="o">[</span>...<span class="o">]</span> + +<span class="k">RUN</span> pip install --index http://10.1.2.1/packages --trusted-host <span class="m">10</span>.1.2.1 faker +</pre></div> +<p>Now, turn internet off (and lights, too, if you'd like some extra suspense), and build the second file.</p> +<div class="highlight"><pre><span></span>$ docker build . +Sending build context to Docker daemon <span class="m">3</span>.072kB +<span class="o">[</span>...<span class="o">]</span> +Successfully installed faker-8.1.0 python-dateutil-2.8.1 six-1.15.0 text-unidecode-1.3 +Removing intermediate container 2c10ffcdf3ad +---> 1ba83bb8e111 +Successfully built 1ba83bb8e111 +</pre></div> +<!-- --> +<blockquote> +</blockquote> +</div> + + </div><!-- /.entry-content --> + <footer class="meta"> + </footer> + +</section> + <footer id="contentinfo" class="body"> + <hr/> + <address id="about" class="vcard body"> + Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> + </address><!-- /#about --> + </footer><!-- /#contentinfo --> +</body> +</html> +\ No newline at end of file diff --git a/e37125a796b63784152584d8a08a4399c56f488978c5194cc33cc4af10cdbce7.asc b/e37125a796b63784152584d8a08a4399c56f488978c5194cc33cc4af10cdbce7.asc @@ -0,0 +1,11 @@ +-----BEGIN PGP SIGNATURE----- + +iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmHkZ0UACgkQzcvSTdHQ +4AGVUAf/d4iK6oYDMKfftDiM4f/YBNMHcvfRWYEQG6+R15IRmsDvNjMPrXDEJhs8 +Tf1C3fQl35z89b7oQaj8zgkoJwA/nFLNlrWHOM5v18QrYqGUOoLNNUQOp551xVT7 +xN42ogUxAvzs5GQSq87mHF0nx6EqISgmxwluDYh764iv75ZQn9jNu5j0Xrm92cvl +vzJrCxOWOMKBDD3SlhgfY5B9M+juQ8kC3KpxqzfTJ65bPlIE3gipuhsTPvYz5Y1j +7MVE1bkc9r0R6Me5opmZ+kO1S8AGTrMPBz6tSkFsF/D2PfuDMn+dDPqWdSIp2YSP +ZTpajl1spzIGhQNyzmq8wZcYDyWFVA== +=YrEh +-----END PGP SIGNATURE----- diff --git a/e728dd8f874e06549c187d3bf96943585aa8c3c4b51666511f400d7191c0f8ac b/e728dd8f874e06549c187d3bf96943585aa8c3c4b51666511f400d7191c0f8ac @@ -0,0 +1,176 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <link rel="stylesheet" type="text/css" href="./theme/css/style.css" /> + <link rel="stylesheet" type="text/css" href="./theme/css/syntax.css" /> + <title>man bytes gnu - The routing to freedom</title> + <meta charset="utf-8" /> + <meta name="generator" content="Pelican" /> + +</head> + +<body id="index" class="home"> + + <header id="banner" class="body"> + <h1>> <a href="./">man bytes gnu</a><span id="cursor">_</span></h1> + </header> + <nav id="menu"> + <ul> + <li><a href="./category/archiving.html">Archiving</a></li> + <li><a href="./category/code.html">Code</a></li> + <li><a href="./category/hygiene.html">Hygiene</a></li> + <li class="active"><a href="./category/offlining.html">Offlining</a></li> + <li><a href="./pages/identities.html">Identities</a></li> + <li><a href="./pages/shares.html">Shares</a></li> + + <li><a href="/tags.html">tags</a></li> + </ul> + <hr/> + </nav><!-- /#menu --> + <header> + <h1 class="entry-title top-body-title"> + <a href="./docker-offline-1-routing.html" rel="bookmark" + title="Permalink to The routing to freedom">Offline Docker: The routing to freedom</a></h1> + + + <div class="category meta"> + Posted +<time class="published" datetime="2021-04-26T07:54:00+02:00"> + Mon 26 April 2021 + </time> +in <a class="category" href="./category/offlining.html">offlining</a> + <a href="./tag/docker.html">docker</a> + <a href="./tag/networking.html">networking</a> + <a href="./tag/iptables.html">iptables</a> + <a href="./tag/iproute.html">iproute</a> + </div> + + <div class="neighbors meta"> + <ul> + + <li> + Next: <a href="./docker-offline-2-python.html"> + Local python repository + </a> + </li> + </ul> + </div> + <div class="entry-series meta"> +Part 1 from the series "Offline Docker" + <ol class="entry-series-parts"> + <li>| <a href="docker-offline-2-python.html" title="Local python repository ">Part 2</a></li> + <li>| <a href="docker-offline-3-npm.html" title="Local npm repository ">Part 3</a></li> + </ol> + </div> + <div class="meta"> +<hr/> + </div> +</header> + +<section id="content" class="body"> + <div class="entry-content"> + <!-- .. CAUTION:: + + This is a purely technical article. You should probably be `this geeky <https://g33k.holbrook.no/8319a926>`_ to continue reading. --> +<p>Five years ago I decided that I wanted to be able to work from anywhere, anytime. Four years ago, that promise was kept. In part.</p> +<p>I do not need to go to an office somewhere. I can work outside in a park if I want to. I can ride on to a new town every day. I only ever need to bring my trusty old <a class="reference external" href="https://www.tuxedocomputers.com/en">Tuxedo Laptop</a> whereever I go.</p> +<p>All of this as true, as long as there is internet available. And, as it turns out, <em>good</em> internet available.</p> +<p>This has become especially obvious to me once I started to work with a project that involves a collection of microservices contained in a Docker environment, which also makes extensive use of custom packages that change frequently alongside the development process. Turns out, every time I want to rebuild my cluster of containers when sitting in the sun in a park, I need my LTE modem to play along. If it doesn't, then a single package that can't be reached will thwart the build.</p> +<p>This does not feel much like freedom after all. So let's see how we can serve all of these locally from our host instead.</p> +<p>First of all, we have to be able to reach our local host from the Docker containers. This is less straightforward than it may seem at first. The most obvious solution is to use the <tt class="docutils literal">host</tt> network driver, but this exposes your <em>whole</em> localhost interface and routes to internet, too. Aside from the security issues that raises, it can also also trick you into assuming that some resources are available when they in fact will not be when you move on to a different environment. What we want is to <em>block</em> access to internet, while <em>choosing</em> which services to let the Docker container use.</p> +<p>Once we have this in place, we want to create local repositories for all the stuff we otherwise need to download. In this particular case, that means a <strong>Docker</strong> repository, a <strong>Python</strong> repository, a <strong>nodejs</strong> repository and a <strong>linux</strong> repository. We'll use <a class="reference external" href="https://archlinux.org">Archlinux</a> for this exercise, because that's been my home environment for the last four years.</p> +<p>In fact, having your own mirror of all these and anything else you base most of your work on is not only a good idea for the purpose of <em>offlining</em> in itself, but wasting on bandwidth for items you've already downloaded hundreds of times is not exactly a nod to climate awareness either. And even more importantly, ensuring <em>availability</em> of software is something we should all participate in, and not merely defer to a couple of git repository giants.</p> +<blockquote> +</blockquote> +<!-- --> +<blockquote> +</blockquote> +<div class="section" id="reaching-the-local-host"> +<h2>Reaching the local host</h2> +<p><em>Local host</em> not <em>localhost</em>, mind you. Which means we need a different interface to connect to. And since we are not wiring up in any physical sense, a virtual interface seems to be the reasonable way to go.</p> +<p>First, let's prepare a base Docker layer with some tools that you should never leave home without.</p> +<div class="section" id="prepare-the-docker-image"> +<h3>Prepare the docker image</h3> +<pre class="code docker literal-block"> +<span class="k">FROM</span> <span class="s">archlinux:latest</span> + +<span class="k">RUN</span> pacman -Sy <span class="o">&&</span> <span class="se">\ +</span> pacman -S --noconfirm gnu-netcat socat inetutils iproute2 +</pre> +<p>Let's build this as an image called <tt class="docutils literal">archbase</tt>. Provided the content above is a file called <tt class="docutils literal">Dockerfile.archbase</tt> in your current directory:</p> +<div class="highlight"><pre><span></span>$ docker build -t archbase -f Dockerfile.archbase . +</pre></div> +</div> +<div class="section" id="set-up-network-interfaces"> +<h3>Set up network interfaces</h3> +<p>Bring up a virtual interface</p> +<div class="highlight"><pre><span></span>$ ip link add foo <span class="nb">type</span> dummy +</pre></div> +<p>Find the subnet of the <tt class="docutils literal"><span class="pre">no-internet</span></tt> Docker network. This network driver is a builtin that provides exactly what it advertises.</p> +<div class="highlight"><pre><span></span>$ docker network inspect no-internet +<span class="o">[</span>...<span class="o">]</span> +<span class="s2">"Config"</span>: <span class="o">[</span> + <span class="o">{</span> + <span class="s2">"Subnet"</span>: <span class="s2">"10.1.1.0/24"</span>, + <span class="s2">"Gateway"</span>: <span class="s2">"10.1.1.1"</span> + <span class="o">}</span> +<span class="o">]</span> +</pre></div> +<p>Assign an IP address to the dummy interface in a <em>different</em> subnet than the one the Docker network uses.</p> +<div class="highlight"><pre><span></span>$ ip addr add <span class="m">10</span>.1.2.1/24 dev foo +</pre></div> +</div> +<div class="section" id="traverse-the-firewall"> +<h3>Traverse the firewall</h3> +<p>Find the bridge used by the Docker container. Look for an ip address that matches the gateway of the docker network config.</p> +<div class="highlight"><pre><span></span>$ ip addr ls +<span class="m">17</span>: br-d4ddb68f9938: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu <span class="m">1500</span> qdisc noqueue state UP group default +link/ether <span class="m">02</span>:42:9c:1a:58:d2 brd ff:ff:ff:ff:ff:ff +inet <span class="m">10</span>.1.1.1/24 brd <span class="m">10</span>.1.1.255 scope global br-d4ddb68f9938 +valid_lft forever preferred_lft forever +inet6 fe80::42:9cff:fe1a:58d2/64 scope link +valid_lft forever preferred_lft forever +<span class="o">[</span>...<span class="o">]</span> +<span class="m">7614</span>: foo: <BROADCAST,NOARP,UP,LOWER_UP> mtu <span class="m">1500</span> qdisc noqueue master br-496e528b928c state UNKNOWN group default qlen <span class="m">1000</span> +link/ether 1a:50:53:2b:96:98 brd ff:ff:ff:ff:ff:ff +inet <span class="m">10</span>.1.3.1/24 scope global foo +valid_lft forever preferred_lft forever +inet6 fe80::1850:53ff:fe2b:9698/64 scope link +valid_lft forever preferred_lft forever +</pre></div> +<p>Add the virtual interface to the bridge.</p> +<div class="highlight"><pre><span></span>$ ip link <span class="nb">set</span> foo master br-d4ddb68f9938 +</pre></div> +<p>Long story short, the previous step will make the traffic from the container reach the <tt class="docutils literal">INPUT</tt> chain in <tt class="docutils literal">iptables</tt>. Now we can make an exception for incoming traffic from the <tt class="docutils literal"><span class="pre">no-internet</span></tt> Docker bridge.</p> +<div class="highlight"><pre><span></span>$ iptables -I INPUT <span class="m">1</span> --source br-d4ddb68f9938 --destination <span class="m">10</span>.1.2.1/24 -j ACCEPT +</pre></div> +</div> +<div class="section" id="verify"> +<h3>Verify</h3> +<p>Provided you don't have any other hurdles in your local <tt class="docutils literal">ìptables</tt> setup, a port on device <tt class="docutils literal">foo</tt> should be reachable from the docker container. We can use socat to check.</p> +<p>On local host:</p> +<div class="highlight"><pre><span></span>$ socat TCP4-LISTEN:8000,bind<span class="o">=</span><span class="m">10</span>.1.2.1,reuseaddr - +</pre></div> +<p>Start the docker container with shell prompt</p> +<div class="highlight"><pre><span></span>$ docker run --network no-internet -it archbase /bin/bash +</pre></div> +<p>The moment of truth</p> +<div class="highlight"><pre><span></span>$ <span class="nb">echo</span> bar <span class="p">|</span> socat - TCP4:10.1.2.1:8000 +</pre></div> +<p>Spoiler: <tt class="docutils literal">bar</tt> should pop up on the local host side.</p> +</div> +</div> + + </div><!-- /.entry-content --> + <footer class="meta"> + </footer> + +</section> + <footer id="contentinfo" class="body"> + <hr/> + <address id="about" class="vcard body"> + Powered by <a title="Pelican is a static site generator" href="https://getpelican.com/">Pelican</a> | <a title="manbytesgnu.com is licensed under Creative Commons Attribution-ShareAlike 4.0 International" href="https://holbrook.no/share/licenses/cc/cc-by-sa-4.0.txt">CC-BY-SA 4.0</a> | <a title="GIT site history since jan 15th 2022" href="https://git.defalsify.org/manbytesgnu_site">Site history</a> | ETH <a href="./images/donate_eth_qr.png" title="Ethereum address for tipping">0x185Cbce7650FF7Ad3B587E26B2877d95568805e3</a> | BTC <a href="./images/donate_btc_qr.png" title="Bitcoin address for tipping">12DnRH9HpJ6cfET2LKHrURn2yZBDfDEwHv</a> + </address><!-- /#about --> + </footer><!-- /#contentinfo --> +</body> +</html> +\ No newline at end of file diff --git a/e728dd8f874e06549c187d3bf96943585aa8c3c4b51666511f400d7191c0f8ac.asc b/e728dd8f874e06549c187d3bf96943585aa8c3c4b51666511f400d7191c0f8ac.asc @@ -0,0 +1,11 @@ +-----BEGIN PGP SIGNATURE----- + +iQEzBAABCAAdFiEEWahEpISsESU9Oj6dzcvSTdHQ4AEFAmHkZ0UACgkQzcvSTdHQ +4AFx4gf/SNbtn5BronWO17cgnzoTVpDCBN1qC06ZaFDh3zqAe90W3SgtEGBw+DSz +pi52Tt2q4SwP0jribRLVMOTZeMiXFDLd1De2+iT4kCdgUCnPBBeuJ9l8kmF8gNTj +lCK/l4ZBUcTiCMAKh9tKywXvsWmvIGVYkvujclJD1zhNc0IvUXFZ7BxrSHm7Grjn +PYDDsYxFr9TwxFr7PRvuM5c4bL9CJvwZPeecu/6gj5QJzLhK1Mf7abhBeRinRz6V +zaFo7JcT7UHWNb/W17e6DE1D2IF3IDxjAu6vqzd7GeyPWEODHRLp25/tYhdkQZ2P +CFPHLx5mkZ/TDswBW7CMKDWtDvGtig== +=in3B +-----END PGP SIGNATURE-----