commit d0296251426a01cf0d5909fb5a7d472a36731600 Author: Jeffrey Paul Date: Mon Jan 9 07:41:08 2017 +0100 initial, in a state of flux diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6a7f154 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +venv +pinlist.json diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6f2eb27 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM ubuntu:16.04 + +VOLUME /var/lib/ipfs +ENV IPFS_PATH /var/lib/ipfs + +ADD \ + https://dist.ipfs.io/go-ipfs/v0.4.4/go-ipfs_v0.4.4_linux-amd64.tar.gz \ + /usr/local/src + +ADD . /var/generate-pinlist + +RUN \ + cd /usr/local/src && \ + tar zxvf go-ipfs_v0.4.4_linux-amd64.tar.gz && \ + mv go-ipfs/ipfs . && \ + cd /var/generate-pinlist && \ + pip -r requirements.txt && \ + python generate.py && \ + ipfs add pinlist.txt diff --git a/README.md b/README.md new file mode 100644 index 0000000..ab8eb1c --- /dev/null +++ b/README.md @@ -0,0 +1,15 @@ +# ipfs-autopin + +I want more stuff to get pinned on more hosts. + +This is a repo that generates a script that can be run to pin a bunch +of IPFS paths. The list itself will be periodically generated from this +repo and placed into ipfs under my namespace, allowing people to +fetch it consistently and automatically update their pinning lists. + +# Requirements + +* general-purpose interest +* not too big (blogs, metadata, documents) +* not general purpose software mirrors / etc + * maybe later diff --git a/generate.py b/generate.py new file mode 100755 index 0000000..f047f10 --- /dev/null +++ b/generate.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python + +#import ipfsapi +from sanelogging import log +from datetime import datetime +import json +import os +import os.path +import re +import subprocess +import yaml + +ipfsPathPattern = re.compile('\/ipfs\/([a-zA-Z0-9]+)') + +#def connectIpfs(hostname='127.0.0.1',port=5001): +# ipfs = ipfsapi.connect(hostname,port) +# log.info("connected to ipfsd: %s" % ipfs.id()['ProtocolVersion']) +# return ipfs + +def main(): + startTime = datetime.utcnow() + log.info("generating pin list.") + generatePinList() + elapsed = datetime.utcnow() - startTime + log.info("finished in %is" % elapsed.total_seconds()) + +def generatePinList(): + pinFiles = readPinFiles() +# ipfs = connectIpfs() + + resources = [] + + for pinFile in pinFiles: + rs = extractResources(pinFile) + resources.extend(rs) + + log.info("collected data from files.") + + resources = resolveResources(resources) + + output = { + "generatedAt": datetime.utcnow().strftime("%s"), + "pinList": resources + } + log.info("resources resolved, writing pinlist file") + outputFileName = 'pinlist.json' + fd = open(outputFileName + '.new','w') + json.dump(output,fd) + fd.close() + os.rename(outputFileName + '.new', outputFileName) + +def extractResources(pinFile): + # this pulls out the list of ipns/ipfs + # items from the single file + output = [] + if 'ipns' in pinFile['pins']: + if isinstance(pinFile['pins']['ipns'], basestring): + # single ipns name + output.append({ + 'type': 'ipns', + 'name': pinFile['pins']['ipns'], + 'fileName': os.path.basename(pinFile['fileName']) + }) + else: + # list of ipns names + for ipnsName in pinFile['pins']['ipns']: + output.append({ + 'type': 'ipns', + 'name': ipnsName, + 'fileName': os.path.basename(pinFile['fileName']) + }) + + if 'ipfs' in pinFile['pins']: + if isinstance(pinFile['pins']['ipfs'], basestring): + # single ipfs name + output.append({ + 'type': 'ipfs', + 'name': pinFile['pins']['ipfs'], + 'fileName': os.path.basename(pinFile['fileName']) + }) + else: + # list of ipfs names + for ipfsName in pinFile['pins']['ipfs']: + output.append({ + 'type': 'ipfs', + 'name': ipfsName, + 'fileName': os.path.basename(pinFile['fileName']) + }) + return output + +def resolveResources(resources): + log.info("resolving resource names for pin list") + for resource in resources: + if resource['type'] == 'ipfs': + resource['resolved'] = resource['name'] + elif resource['type'] == 'ipns': + # attepmt to resolve ipns name + # FIXME try/catch + res = None + try: + res = resolveIpnsName(resource['name']) + if res: + resource['resolved'] = res + except LookupError: + log.info( + "unable to resolve ipns name '%s' from file '%s'" + % (resource['name'], resource['fileName']) + ) + resource['error'] = 'unable to resolve name' + return resources + +def runCommand(cmd): + p = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT + ) + retval = p.wait() + if retval != 0: + raise LookupError() + return p.stdout.read() + +def resolveIpnsName(ipnsName): + log.debug("resolving name '%s'" % ipnsName) + res = runCommand(['ipfs', 'name', 'resolve', ipnsName]) + ipfsNameMatch = ipfsPathPattern.match(res) + if not ipfsNameMatch: + log.info("unable to resolve ipns name '%s'" % ipnsName) + return None + ipfsName = ipfsNameMatch.group(1) + log.info("resolved ipns name '%s' to '%s'" % (ipnsName, ipfsName)) + return ipfsName + +def readPinFiles(): + log.info("reading pinfiles...") + output = [] + pinFiles = getPinFileNames() + for pinFileName in pinFiles: + pinFileData = parsePinFile(pinFileName) + pinFileData['fileName'] = pinFileName + output.append(pinFileData) + return output + +def parsePinFile(fn): + x = {} + f = open(fn,"r") + x = yaml.load(f) + f.close() + validatePinFileData(x) + return x + +def validatePinFileData(pinFileStruct): + # FIXME validate various file properties here + pass + +def getPinFileNames(): + thisDir = os.path.dirname(os.path.abspath(__file__)) + pinDir = os.path.join(thisDir, 'pins') + pinFileNames = [ + os.path.join(pinDir,f) for f in os.listdir(pinDir) + if os.path.isfile(os.path.join(pinDir, f)) + ] + return pinFileNames + +if __name__ == "__main__": + main() diff --git a/pins/ber1-test-name.yml b/pins/ber1-test-name.yml new file mode 100644 index 0000000..072733b --- /dev/null +++ b/pins/ber1-test-name.yml @@ -0,0 +1,7 @@ +description: test name from ber1 +url: https://sneak.berlin +contact: + name: Jeffrey Paul + email: sneak@sneak.berlin +pins: + ipns: QmZKpS7iyUnDXcUxhou5arokZ5bZViZokEzDVAEGctBSMw diff --git a/pins/fails.yml b/pins/fails.yml new file mode 100644 index 0000000..3bf3e69 --- /dev/null +++ b/pins/fails.yml @@ -0,0 +1,6 @@ +description: this ipns will intentionall not resolve +contact: + name: Jeffrey Paul + email: sneak@sneak.berlin +pins: + ipns: QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAug11111111 diff --git a/pins/go-ipfs_v0.4.4.json b/pins/go-ipfs_v0.4.4.json new file mode 100644 index 0000000..4877c96 --- /dev/null +++ b/pins/go-ipfs_v0.4.4.json @@ -0,0 +1,14 @@ +{ + "description": "go-ipfs v0.4.4 binaries", + "pins": + [ + "/ipfs/QmXvDdqieUAUo7srRVXvsKZRmzNGkuhRqWKYXLcXnBm4dc", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_darwin-386.tar.gz + "/ipfs/QmPsMMi2Lq15QDNa8vmyZwwE15KbgUnHkQz3MX8pR5oUvq", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_darwin-amd64.tar.gz + "/ipfs/QmQZyL5sWdXQyS9UwPHRsSv5yXz1cWRgTV9C6zS6EsGnBv", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_freebsd-amd64.tar.gz + "/ipfs/QmRCXbtDRFUVLYndxxrobsPEKSBNJtPbvq4FTMKWBWqneE", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_linux-386.tar.gz + "/ipfs/QmRE6QGDsRhHT81etCKKRvUucoQTY4KqjoH8RKW6jaF7kX", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_linux-amd64.tar.gz + "/ipfs/QmUb7jGAgYd4AAGX82qY3Znt2Xrx3FLC2N6jiYVMcSTcMq", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_windows-386.zip + "/ipfs/QmPPNuaTHcoFesnZuVjSMaxPVifunrVqFGj3avjTaKSLz3", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_windows-amd64.zip + "/ipfs/QmdZ81FrNJYnjHbzuX6L2vqfBmEvRFwUgRU2TdPHU8WNLL" // go-ipfs_v0.4.4 + ] + } diff --git a/pins/ipfs-webui.json b/pins/ipfs-webui.json new file mode 100644 index 0000000..b6dcc3e --- /dev/null +++ b/pins/ipfs-webui.json @@ -0,0 +1,12 @@ +{ + "description": "webui files", + "url": "https://github.com/ipfs/refs-solarnet-storage/commit/df0a2cc4b53cea15132bec2d0b96b5d2d7a87ffb", + "pins": [ + "/ipfs/QmS2HL9v5YeKgQkkWMvs1EMnFtUowTEdFfSSeMT4pos1e6", + "/ipfs/QmXdu7HWdV6CUaUabd9q2ZeA4iHZLVyDRj3Gi4dsJsWjbr", + "/ipfs/QmaaqrHyAQm7gALkRW8DcfGX3u8q9rWKnxEMmf7m9z515w", + "/ipfs/QmSHDxWsMPuJQKWmVA1rB5a3NX2Eme5fPqNb63qwaqiqSp", + "/ipfs/QmctngrQAt9fjpQUZr7Bx3BsXUcif52eZGTizWhvcShsjz", + "/ipfs/QmXX7YRpU7nNBKfw75VG7Y1c3GwpSAGHRev67XVPgZFv9R" + ] +} diff --git a/pins/sneak.berlin.json b/pins/sneak.berlin.json new file mode 100644 index 0000000..5457152 --- /dev/null +++ b/pins/sneak.berlin.json @@ -0,0 +1,11 @@ +{ + "url": "https://sneak.berlin", + "description": "Personal blog of Jeffrey Paul", + "contact": { + "name": "Jeffrey Paul", + "email": "sneak@sneak.berlin" + }, + "pins": [ + "/ipns/QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAugpyJJpCVp/website" + ] +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..51a5c37 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +colorlog==2.10.0 +PyYAML==3.12 +sanelogging==1.0.1