initial, in a state of flux

.gitignore

@@ -0,0 +1,2 @@
+venv
+pinlist.json

Dockerfile

@@ -0,0 +1,19 @@
+FROM ubuntu:16.04
+
+VOLUME /var/lib/ipfs
+ENV IPFS_PATH /var/lib/ipfs
+
+ADD \
+    https://dist.ipfs.io/go-ipfs/v0.4.4/go-ipfs_v0.4.4_linux-amd64.tar.gz \
+    /usr/local/src
+
+ADD . /var/generate-pinlist
+
+RUN \
+    cd /usr/local/src && \
+    tar zxvf go-ipfs_v0.4.4_linux-amd64.tar.gz && \
+    mv go-ipfs/ipfs . && \
+    cd /var/generate-pinlist && \
+    pip -r requirements.txt && \
+    python generate.py && \
+    ipfs add pinlist.txt

README.md

@@ -0,0 +1,15 @@
+# ipfs-autopin
+
+I want more stuff to get pinned on more hosts.
+
+This is a repo that generates a script that can be run to pin a bunch
+of IPFS paths.  The list itself will be periodically generated from this
+repo and placed into ipfs under my namespace, allowing people to
+fetch it consistently and automatically update their pinning lists.
+
+# Requirements
+
+* general-purpose interest
+* not too big (blogs, metadata, documents)
+* not general purpose software mirrors / etc
+    * maybe later

generate.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python
+
+#import ipfsapi
+from sanelogging import log
+from datetime import datetime
+import json
+import os
+import os.path
+import re
+import subprocess
+import yaml
+
+ipfsPathPattern = re.compile('\/ipfs\/([a-zA-Z0-9]+)')
+
+#def connectIpfs(hostname='127.0.0.1',port=5001):
+#    ipfs = ipfsapi.connect(hostname,port)
+#    log.info("connected to ipfsd: %s" % ipfs.id()['ProtocolVersion'])
+#    return ipfs
+
+def main():
+    startTime = datetime.utcnow()
+    log.info("generating pin list.")
+    generatePinList()
+    elapsed = datetime.utcnow() - startTime
+    log.info("finished in %is" % elapsed.total_seconds())
+
+def generatePinList():
+    pinFiles = readPinFiles()
+#    ipfs = connectIpfs()
+
+    resources = []
+
+    for pinFile in pinFiles:
+        rs = extractResources(pinFile)
+        resources.extend(rs)
+
+    log.info("collected data from files.")
+
+    resources = resolveResources(resources)
+
+    output = {
+        "generatedAt": datetime.utcnow().strftime("%s"),
+        "pinList": resources
+    }
+    log.info("resources resolved, writing pinlist file")
+    outputFileName = 'pinlist.json'
+    fd = open(outputFileName + '.new','w')
+    json.dump(output,fd)
+    fd.close()
+    os.rename(outputFileName + '.new', outputFileName)
+
+def extractResources(pinFile):
+    # this pulls out the list of ipns/ipfs
+    # items from the single file
+    output = []
+    if 'ipns' in pinFile['pins']:
+        if isinstance(pinFile['pins']['ipns'], basestring):
+            # single ipns name
+            output.append({
+                'type': 'ipns',
+                'name': pinFile['pins']['ipns'],
+                'fileName': os.path.basename(pinFile['fileName'])
+            })
+        else:
+            # list of ipns names
+            for ipnsName in pinFile['pins']['ipns']:
+                output.append({
+                    'type': 'ipns',
+                    'name': ipnsName,
+                    'fileName': os.path.basename(pinFile['fileName'])
+                })
+
+    if 'ipfs' in pinFile['pins']:
+        if isinstance(pinFile['pins']['ipfs'], basestring):
+            # single ipfs name
+            output.append({
+                'type': 'ipfs',
+                'name': pinFile['pins']['ipfs'],
+                'fileName': os.path.basename(pinFile['fileName'])
+            })
+        else:
+            # list of ipfs names
+            for ipfsName in pinFile['pins']['ipfs']:
+                output.append({
+                    'type': 'ipfs',
+                    'name': ipfsName,
+                    'fileName': os.path.basename(pinFile['fileName'])
+                })
+    return output
+
+def resolveResources(resources):
+    log.info("resolving resource names for pin list")
+    for resource in resources:
+        if resource['type'] == 'ipfs':
+            resource['resolved'] = resource['name']
+        elif resource['type'] == 'ipns':
+            # attepmt to resolve ipns name
+            # FIXME try/catch
+            res = None
+            try:
+                res = resolveIpnsName(resource['name'])
+                if res:
+                    resource['resolved'] = res
+            except LookupError:
+                log.info(
+                    "unable to resolve ipns name '%s' from file '%s'"
+                        % (resource['name'], resource['fileName'])
+                )
+                resource['error'] = 'unable to resolve name'
+    return resources
+
+def runCommand(cmd):
+    p = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT
+    )
+    retval = p.wait()
+    if retval != 0:
+        raise LookupError()
+    return p.stdout.read()
+
+def resolveIpnsName(ipnsName):
+    log.debug("resolving name '%s'" % ipnsName)
+    res = runCommand(['ipfs', 'name', 'resolve', ipnsName])
+    ipfsNameMatch = ipfsPathPattern.match(res)
+    if not ipfsNameMatch:
+        log.info("unable to resolve ipns name '%s'" % ipnsName)
+        return None
+    ipfsName = ipfsNameMatch.group(1)
+    log.info("resolved ipns name '%s' to '%s'" % (ipnsName, ipfsName))
+    return ipfsName
+
+def readPinFiles():
+    log.info("reading pinfiles...")
+    output = []
+    pinFiles = getPinFileNames()
+    for pinFileName in pinFiles:
+        pinFileData = parsePinFile(pinFileName)
+        pinFileData['fileName'] = pinFileName
+        output.append(pinFileData)
+    return output
+
+def parsePinFile(fn):
+    x = {}
+    f = open(fn,"r")
+    x = yaml.load(f)
+    f.close()
+    validatePinFileData(x)
+    return x
+
+def validatePinFileData(pinFileStruct):
+    # FIXME validate various file properties here
+    pass
+
+def getPinFileNames():
+    thisDir = os.path.dirname(os.path.abspath(__file__))
+    pinDir = os.path.join(thisDir, 'pins')
+    pinFileNames = [
+        os.path.join(pinDir,f) for f in os.listdir(pinDir)
+            if os.path.isfile(os.path.join(pinDir, f))
+    ]
+    return pinFileNames
+
+if __name__ == "__main__":
+    main()

pins/ber1-test-name.yml

@@ -0,0 +1,7 @@
+description: test name from ber1
+url: https://sneak.berlin
+contact:
+    name: Jeffrey Paul
+    email: sneak@sneak.berlin
+pins:
+    ipns: QmZKpS7iyUnDXcUxhou5arokZ5bZViZokEzDVAEGctBSMw

pins/fails.yml

@@ -0,0 +1,6 @@
+description: this ipns will intentionall not resolve
+contact:
+    name: Jeffrey Paul
+    email: sneak@sneak.berlin
+pins:
+    ipns: QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAug11111111

pins/go-ipfs_v0.4.4.json

@@ -0,0 +1,14 @@
+{
+  "description": "go-ipfs v0.4.4 binaries",
+  "pins":
+ [
+      "/ipfs/QmXvDdqieUAUo7srRVXvsKZRmzNGkuhRqWKYXLcXnBm4dc",   // go-ipfs_v0.4.4/go-ipfs_v0.4.4_darwin-386.tar.gz
+      "/ipfs/QmPsMMi2Lq15QDNa8vmyZwwE15KbgUnHkQz3MX8pR5oUvq",   // go-ipfs_v0.4.4/go-ipfs_v0.4.4_darwin-amd64.tar.gz
+      "/ipfs/QmQZyL5sWdXQyS9UwPHRsSv5yXz1cWRgTV9C6zS6EsGnBv",   // go-ipfs_v0.4.4/go-ipfs_v0.4.4_freebsd-amd64.tar.gz
+      "/ipfs/QmRCXbtDRFUVLYndxxrobsPEKSBNJtPbvq4FTMKWBWqneE",   // go-ipfs_v0.4.4/go-ipfs_v0.4.4_linux-386.tar.gz
+      "/ipfs/QmRE6QGDsRhHT81etCKKRvUucoQTY4KqjoH8RKW6jaF7kX",   // go-ipfs_v0.4.4/go-ipfs_v0.4.4_linux-amd64.tar.gz
+      "/ipfs/QmUb7jGAgYd4AAGX82qY3Znt2Xrx3FLC2N6jiYVMcSTcMq",   // go-ipfs_v0.4.4/go-ipfs_v0.4.4_windows-386.zip
+      "/ipfs/QmPPNuaTHcoFesnZuVjSMaxPVifunrVqFGj3avjTaKSLz3",   // go-ipfs_v0.4.4/go-ipfs_v0.4.4_windows-amd64.zip
+      "/ipfs/QmdZ81FrNJYnjHbzuX6L2vqfBmEvRFwUgRU2TdPHU8WNLL"    // go-ipfs_v0.4.4
+    ]
+  }

pins/ipfs-webui.json

@@ -0,0 +1,12 @@
+{
+  "description": "webui files",
+  "url": "https://github.com/ipfs/refs-solarnet-storage/commit/df0a2cc4b53cea15132bec2d0b96b5d2d7a87ffb",
+  "pins": [
+    "/ipfs/QmS2HL9v5YeKgQkkWMvs1EMnFtUowTEdFfSSeMT4pos1e6",
+    "/ipfs/QmXdu7HWdV6CUaUabd9q2ZeA4iHZLVyDRj3Gi4dsJsWjbr",
+    "/ipfs/QmaaqrHyAQm7gALkRW8DcfGX3u8q9rWKnxEMmf7m9z515w",
+    "/ipfs/QmSHDxWsMPuJQKWmVA1rB5a3NX2Eme5fPqNb63qwaqiqSp",
+    "/ipfs/QmctngrQAt9fjpQUZr7Bx3BsXUcif52eZGTizWhvcShsjz",
+    "/ipfs/QmXX7YRpU7nNBKfw75VG7Y1c3GwpSAGHRev67XVPgZFv9R"
+  ]
+}

pins/sneak.berlin.json

@@ -0,0 +1,11 @@
+{
+  "url": "https://sneak.berlin",
+  "description": "Personal blog of Jeffrey Paul",
+  "contact": {
+    "name": "Jeffrey Paul",
+    "email": "sneak@sneak.berlin"
+  },
+  "pins": [
+    "/ipns/QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAugpyJJpCVp/website"
+  ]
+}

requirements.txt

@@ -0,0 +1,3 @@
+colorlog==2.10.0
+PyYAML==3.12
+sanelogging==1.0.1