master
Jeffrey Paul 7 years ago
parent d029625142
commit 655026cdbd
  1. 3
      .gitignore
  2. 19
      Dockerfile
  3. 22
      Makefile
  4. 166
      generate.py
  5. 11
      pins/ber1-test-name.json
  6. 7
      pins/ber1-test-name.yml
  7. 10
      pins/fails.json
  8. 6
      pins/fails.yml
  9. 23
      pins/go-ipfs_v0.4.4.json
  10. 4
      pins/ipfs-webui.json
  11. 10
      pins/sneak.berlin.json
  12. 3
      requirements.txt

3
.gitignore vendored

@ -1,2 +1,3 @@
venv venv
pinlist.json pinlist.new
pinlist.txt

@ -1,19 +0,0 @@
FROM ubuntu:16.04
VOLUME /var/lib/ipfs
ENV IPFS_PATH /var/lib/ipfs
ADD \
https://dist.ipfs.io/go-ipfs/v0.4.4/go-ipfs_v0.4.4_linux-amd64.tar.gz \
/usr/local/src
ADD . /var/generate-pinlist
RUN \
cd /usr/local/src && \
tar zxvf go-ipfs_v0.4.4_linux-amd64.tar.gz && \
mv go-ipfs/ipfs . && \
cd /var/generate-pinlist && \
pip -r requirements.txt && \
python generate.py && \
ipfs add pinlist.txt

@ -0,0 +1,22 @@
IPFS_ROOT := $(HOME)/Documents/sneak/my-ipfs-root
PINLIST_DIR := $(IPFS_ROOT)/pinlist
default: deploy
.PHONY: deploy
clean:
rm -f pinlist.txt
deploy: pinlist.txt
mkdir -p $(PINLIST_DIR)
rsync -avP ./pinlist.txt $(PINLIST_DIR)/pinlist.txt
ipfs add -r $(IPFS_ROOT) | \
tail -1 | \
awk -F' ' '{print $$2}' | \
ipfs name publish
pinlist.txt: pins/*.json
jq -r .pins[] $^ > pinlist.new && mv pinlist.new $@
echo "# $$(date -u +%s)" >> $@

@ -1,166 +0,0 @@
#!/usr/bin/env python
#import ipfsapi
from sanelogging import log
from datetime import datetime
import json
import os
import os.path
import re
import subprocess
import yaml
ipfsPathPattern = re.compile('\/ipfs\/([a-zA-Z0-9]+)')
#def connectIpfs(hostname='127.0.0.1',port=5001):
# ipfs = ipfsapi.connect(hostname,port)
# log.info("connected to ipfsd: %s" % ipfs.id()['ProtocolVersion'])
# return ipfs
def main():
startTime = datetime.utcnow()
log.info("generating pin list.")
generatePinList()
elapsed = datetime.utcnow() - startTime
log.info("finished in %is" % elapsed.total_seconds())
def generatePinList():
pinFiles = readPinFiles()
# ipfs = connectIpfs()
resources = []
for pinFile in pinFiles:
rs = extractResources(pinFile)
resources.extend(rs)
log.info("collected data from files.")
resources = resolveResources(resources)
output = {
"generatedAt": datetime.utcnow().strftime("%s"),
"pinList": resources
}
log.info("resources resolved, writing pinlist file")
outputFileName = 'pinlist.json'
fd = open(outputFileName + '.new','w')
json.dump(output,fd)
fd.close()
os.rename(outputFileName + '.new', outputFileName)
def extractResources(pinFile):
# this pulls out the list of ipns/ipfs
# items from the single file
output = []
if 'ipns' in pinFile['pins']:
if isinstance(pinFile['pins']['ipns'], basestring):
# single ipns name
output.append({
'type': 'ipns',
'name': pinFile['pins']['ipns'],
'fileName': os.path.basename(pinFile['fileName'])
})
else:
# list of ipns names
for ipnsName in pinFile['pins']['ipns']:
output.append({
'type': 'ipns',
'name': ipnsName,
'fileName': os.path.basename(pinFile['fileName'])
})
if 'ipfs' in pinFile['pins']:
if isinstance(pinFile['pins']['ipfs'], basestring):
# single ipfs name
output.append({
'type': 'ipfs',
'name': pinFile['pins']['ipfs'],
'fileName': os.path.basename(pinFile['fileName'])
})
else:
# list of ipfs names
for ipfsName in pinFile['pins']['ipfs']:
output.append({
'type': 'ipfs',
'name': ipfsName,
'fileName': os.path.basename(pinFile['fileName'])
})
return output
def resolveResources(resources):
log.info("resolving resource names for pin list")
for resource in resources:
if resource['type'] == 'ipfs':
resource['resolved'] = resource['name']
elif resource['type'] == 'ipns':
# attepmt to resolve ipns name
# FIXME try/catch
res = None
try:
res = resolveIpnsName(resource['name'])
if res:
resource['resolved'] = res
except LookupError:
log.info(
"unable to resolve ipns name '%s' from file '%s'"
% (resource['name'], resource['fileName'])
)
resource['error'] = 'unable to resolve name'
return resources
def runCommand(cmd):
p = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT
)
retval = p.wait()
if retval != 0:
raise LookupError()
return p.stdout.read()
def resolveIpnsName(ipnsName):
log.debug("resolving name '%s'" % ipnsName)
res = runCommand(['ipfs', 'name', 'resolve', ipnsName])
ipfsNameMatch = ipfsPathPattern.match(res)
if not ipfsNameMatch:
log.info("unable to resolve ipns name '%s'" % ipnsName)
return None
ipfsName = ipfsNameMatch.group(1)
log.info("resolved ipns name '%s' to '%s'" % (ipnsName, ipfsName))
return ipfsName
def readPinFiles():
log.info("reading pinfiles...")
output = []
pinFiles = getPinFileNames()
for pinFileName in pinFiles:
pinFileData = parsePinFile(pinFileName)
pinFileData['fileName'] = pinFileName
output.append(pinFileData)
return output
def parsePinFile(fn):
x = {}
f = open(fn,"r")
x = yaml.load(f)
f.close()
validatePinFileData(x)
return x
def validatePinFileData(pinFileStruct):
# FIXME validate various file properties here
pass
def getPinFileNames():
thisDir = os.path.dirname(os.path.abspath(__file__))
pinDir = os.path.join(thisDir, 'pins')
pinFileNames = [
os.path.join(pinDir,f) for f in os.listdir(pinDir)
if os.path.isfile(os.path.join(pinDir, f))
]
return pinFileNames
if __name__ == "__main__":
main()

@ -0,0 +1,11 @@
{
"contact": {
"email": "sneak@sneak.berlin",
"name": "Jeffrey Paul"
},
"description": "test name from ber1 (sneak fileserver)",
"pins": [
"/ipns/QmZKpS7iyUnDXcUxhou5arokZ5bZViZokEzDVAEGctBSMw"
],
"url": "https://sneak.berlin"
}

@ -1,7 +0,0 @@
description: test name from ber1
url: https://sneak.berlin
contact:
name: Jeffrey Paul
email: sneak@sneak.berlin
pins:
ipns: QmZKpS7iyUnDXcUxhou5arokZ5bZViZokEzDVAEGctBSMw

@ -0,0 +1,10 @@
{
"contact": {
"email": "sneak@sneak.berlin",
"name": "Jeffrey Paul"
},
"description": "this ipns will intentionally not resolve for testing",
"pins": [
"/ipns/QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAug11111111"
]
}

@ -1,6 +0,0 @@
description: this ipns will intentionall not resolve
contact:
name: Jeffrey Paul
email: sneak@sneak.berlin
pins:
ipns: QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAug11111111

@ -1,14 +1,13 @@
{ {
"description": "go-ipfs v0.4.4 binaries", "description": "go-ipfs v0.4.4 binaries",
"pins": "pins": [
[ "/ipfs/QmXvDdqieUAUo7srRVXvsKZRmzNGkuhRqWKYXLcXnBm4dc",
"/ipfs/QmXvDdqieUAUo7srRVXvsKZRmzNGkuhRqWKYXLcXnBm4dc", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_darwin-386.tar.gz "/ipfs/QmPsMMi2Lq15QDNa8vmyZwwE15KbgUnHkQz3MX8pR5oUvq",
"/ipfs/QmPsMMi2Lq15QDNa8vmyZwwE15KbgUnHkQz3MX8pR5oUvq", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_darwin-amd64.tar.gz "/ipfs/QmQZyL5sWdXQyS9UwPHRsSv5yXz1cWRgTV9C6zS6EsGnBv",
"/ipfs/QmQZyL5sWdXQyS9UwPHRsSv5yXz1cWRgTV9C6zS6EsGnBv", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_freebsd-amd64.tar.gz "/ipfs/QmRCXbtDRFUVLYndxxrobsPEKSBNJtPbvq4FTMKWBWqneE",
"/ipfs/QmRCXbtDRFUVLYndxxrobsPEKSBNJtPbvq4FTMKWBWqneE", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_linux-386.tar.gz "/ipfs/QmRE6QGDsRhHT81etCKKRvUucoQTY4KqjoH8RKW6jaF7kX",
"/ipfs/QmRE6QGDsRhHT81etCKKRvUucoQTY4KqjoH8RKW6jaF7kX", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_linux-amd64.tar.gz "/ipfs/QmUb7jGAgYd4AAGX82qY3Znt2Xrx3FLC2N6jiYVMcSTcMq",
"/ipfs/QmUb7jGAgYd4AAGX82qY3Znt2Xrx3FLC2N6jiYVMcSTcMq", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_windows-386.zip "/ipfs/QmPPNuaTHcoFesnZuVjSMaxPVifunrVqFGj3avjTaKSLz3",
"/ipfs/QmPPNuaTHcoFesnZuVjSMaxPVifunrVqFGj3avjTaKSLz3", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_windows-amd64.zip "/ipfs/QmdZ81FrNJYnjHbzuX6L2vqfBmEvRFwUgRU2TdPHU8WNLL"
"/ipfs/QmdZ81FrNJYnjHbzuX6L2vqfBmEvRFwUgRU2TdPHU8WNLL" // go-ipfs_v0.4.4 ]
] }
}

@ -1,6 +1,5 @@
{ {
"description": "webui files", "description": "webui files",
"url": "https://github.com/ipfs/refs-solarnet-storage/commit/df0a2cc4b53cea15132bec2d0b96b5d2d7a87ffb",
"pins": [ "pins": [
"/ipfs/QmS2HL9v5YeKgQkkWMvs1EMnFtUowTEdFfSSeMT4pos1e6", "/ipfs/QmS2HL9v5YeKgQkkWMvs1EMnFtUowTEdFfSSeMT4pos1e6",
"/ipfs/QmXdu7HWdV6CUaUabd9q2ZeA4iHZLVyDRj3Gi4dsJsWjbr", "/ipfs/QmXdu7HWdV6CUaUabd9q2ZeA4iHZLVyDRj3Gi4dsJsWjbr",
@ -8,5 +7,6 @@
"/ipfs/QmSHDxWsMPuJQKWmVA1rB5a3NX2Eme5fPqNb63qwaqiqSp", "/ipfs/QmSHDxWsMPuJQKWmVA1rB5a3NX2Eme5fPqNb63qwaqiqSp",
"/ipfs/QmctngrQAt9fjpQUZr7Bx3BsXUcif52eZGTizWhvcShsjz", "/ipfs/QmctngrQAt9fjpQUZr7Bx3BsXUcif52eZGTizWhvcShsjz",
"/ipfs/QmXX7YRpU7nNBKfw75VG7Y1c3GwpSAGHRev67XVPgZFv9R" "/ipfs/QmXX7YRpU7nNBKfw75VG7Y1c3GwpSAGHRev67XVPgZFv9R"
] ],
"url": "https://github.com/ipfs/refs-solarnet-storage/commit/df0a2cc4b53cea15132bec2d0b96b5d2d7a87ffb"
} }

@ -1,11 +1,11 @@
{ {
"url": "https://sneak.berlin",
"description": "Personal blog of Jeffrey Paul",
"contact": { "contact": {
"name": "Jeffrey Paul", "email": "sneak@sneak.berlin",
"email": "sneak@sneak.berlin" "name": "Jeffrey Paul"
}, },
"description": "Personal blog of Jeffrey Paul",
"pins": [ "pins": [
"/ipns/QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAugpyJJpCVp/website" "/ipns/QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAugpyJJpCVp/website"
] ],
"url": "https://sneak.berlin"
} }

@ -1,3 +0,0 @@
colorlog==2.10.0
PyYAML==3.12
sanelogging==1.0.1
Loading…
Cancel
Save