latest
This commit is contained in:
parent
d029625142
commit
655026cdbd
|
|
@ -1,2 +1,3 @@
|
|||
venv
|
||||
pinlist.json
|
||||
pinlist.new
|
||||
pinlist.txt
|
||||
|
|
|
|||
19
Dockerfile
19
Dockerfile
|
|
@ -1,19 +0,0 @@
|
|||
FROM ubuntu:16.04
|
||||
|
||||
VOLUME /var/lib/ipfs
|
||||
ENV IPFS_PATH /var/lib/ipfs
|
||||
|
||||
ADD \
|
||||
https://dist.ipfs.io/go-ipfs/v0.4.4/go-ipfs_v0.4.4_linux-amd64.tar.gz \
|
||||
/usr/local/src
|
||||
|
||||
ADD . /var/generate-pinlist
|
||||
|
||||
RUN \
|
||||
cd /usr/local/src && \
|
||||
tar zxvf go-ipfs_v0.4.4_linux-amd64.tar.gz && \
|
||||
mv go-ipfs/ipfs . && \
|
||||
cd /var/generate-pinlist && \
|
||||
pip -r requirements.txt && \
|
||||
python generate.py && \
|
||||
ipfs add pinlist.txt
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
IPFS_ROOT := $(HOME)/Documents/sneak/my-ipfs-root
|
||||
PINLIST_DIR := $(IPFS_ROOT)/pinlist
|
||||
|
||||
default: deploy
|
||||
|
||||
.PHONY: deploy
|
||||
|
||||
clean:
|
||||
rm -f pinlist.txt
|
||||
|
||||
deploy: pinlist.txt
|
||||
mkdir -p $(PINLIST_DIR)
|
||||
rsync -avP ./pinlist.txt $(PINLIST_DIR)/pinlist.txt
|
||||
ipfs add -r $(IPFS_ROOT) | \
|
||||
tail -1 | \
|
||||
awk -F' ' '{print $$2}' | \
|
||||
ipfs name publish
|
||||
|
||||
|
||||
pinlist.txt: pins/*.json
|
||||
jq -r .pins[] $^ > pinlist.new && mv pinlist.new $@
|
||||
echo "# $$(date -u +%s)" >> $@
|
||||
166
generate.py
166
generate.py
|
|
@ -1,166 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
#import ipfsapi
|
||||
from sanelogging import log
|
||||
from datetime import datetime
|
||||
import json
|
||||
import os
|
||||
import os.path
|
||||
import re
|
||||
import subprocess
|
||||
import yaml
|
||||
|
||||
ipfsPathPattern = re.compile('\/ipfs\/([a-zA-Z0-9]+)')
|
||||
|
||||
#def connectIpfs(hostname='127.0.0.1',port=5001):
|
||||
# ipfs = ipfsapi.connect(hostname,port)
|
||||
# log.info("connected to ipfsd: %s" % ipfs.id()['ProtocolVersion'])
|
||||
# return ipfs
|
||||
|
||||
def main():
|
||||
startTime = datetime.utcnow()
|
||||
log.info("generating pin list.")
|
||||
generatePinList()
|
||||
elapsed = datetime.utcnow() - startTime
|
||||
log.info("finished in %is" % elapsed.total_seconds())
|
||||
|
||||
def generatePinList():
|
||||
pinFiles = readPinFiles()
|
||||
# ipfs = connectIpfs()
|
||||
|
||||
resources = []
|
||||
|
||||
for pinFile in pinFiles:
|
||||
rs = extractResources(pinFile)
|
||||
resources.extend(rs)
|
||||
|
||||
log.info("collected data from files.")
|
||||
|
||||
resources = resolveResources(resources)
|
||||
|
||||
output = {
|
||||
"generatedAt": datetime.utcnow().strftime("%s"),
|
||||
"pinList": resources
|
||||
}
|
||||
log.info("resources resolved, writing pinlist file")
|
||||
outputFileName = 'pinlist.json'
|
||||
fd = open(outputFileName + '.new','w')
|
||||
json.dump(output,fd)
|
||||
fd.close()
|
||||
os.rename(outputFileName + '.new', outputFileName)
|
||||
|
||||
def extractResources(pinFile):
|
||||
# this pulls out the list of ipns/ipfs
|
||||
# items from the single file
|
||||
output = []
|
||||
if 'ipns' in pinFile['pins']:
|
||||
if isinstance(pinFile['pins']['ipns'], basestring):
|
||||
# single ipns name
|
||||
output.append({
|
||||
'type': 'ipns',
|
||||
'name': pinFile['pins']['ipns'],
|
||||
'fileName': os.path.basename(pinFile['fileName'])
|
||||
})
|
||||
else:
|
||||
# list of ipns names
|
||||
for ipnsName in pinFile['pins']['ipns']:
|
||||
output.append({
|
||||
'type': 'ipns',
|
||||
'name': ipnsName,
|
||||
'fileName': os.path.basename(pinFile['fileName'])
|
||||
})
|
||||
|
||||
if 'ipfs' in pinFile['pins']:
|
||||
if isinstance(pinFile['pins']['ipfs'], basestring):
|
||||
# single ipfs name
|
||||
output.append({
|
||||
'type': 'ipfs',
|
||||
'name': pinFile['pins']['ipfs'],
|
||||
'fileName': os.path.basename(pinFile['fileName'])
|
||||
})
|
||||
else:
|
||||
# list of ipfs names
|
||||
for ipfsName in pinFile['pins']['ipfs']:
|
||||
output.append({
|
||||
'type': 'ipfs',
|
||||
'name': ipfsName,
|
||||
'fileName': os.path.basename(pinFile['fileName'])
|
||||
})
|
||||
return output
|
||||
|
||||
def resolveResources(resources):
|
||||
log.info("resolving resource names for pin list")
|
||||
for resource in resources:
|
||||
if resource['type'] == 'ipfs':
|
||||
resource['resolved'] = resource['name']
|
||||
elif resource['type'] == 'ipns':
|
||||
# attepmt to resolve ipns name
|
||||
# FIXME try/catch
|
||||
res = None
|
||||
try:
|
||||
res = resolveIpnsName(resource['name'])
|
||||
if res:
|
||||
resource['resolved'] = res
|
||||
except LookupError:
|
||||
log.info(
|
||||
"unable to resolve ipns name '%s' from file '%s'"
|
||||
% (resource['name'], resource['fileName'])
|
||||
)
|
||||
resource['error'] = 'unable to resolve name'
|
||||
return resources
|
||||
|
||||
def runCommand(cmd):
|
||||
p = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT
|
||||
)
|
||||
retval = p.wait()
|
||||
if retval != 0:
|
||||
raise LookupError()
|
||||
return p.stdout.read()
|
||||
|
||||
def resolveIpnsName(ipnsName):
|
||||
log.debug("resolving name '%s'" % ipnsName)
|
||||
res = runCommand(['ipfs', 'name', 'resolve', ipnsName])
|
||||
ipfsNameMatch = ipfsPathPattern.match(res)
|
||||
if not ipfsNameMatch:
|
||||
log.info("unable to resolve ipns name '%s'" % ipnsName)
|
||||
return None
|
||||
ipfsName = ipfsNameMatch.group(1)
|
||||
log.info("resolved ipns name '%s' to '%s'" % (ipnsName, ipfsName))
|
||||
return ipfsName
|
||||
|
||||
def readPinFiles():
|
||||
log.info("reading pinfiles...")
|
||||
output = []
|
||||
pinFiles = getPinFileNames()
|
||||
for pinFileName in pinFiles:
|
||||
pinFileData = parsePinFile(pinFileName)
|
||||
pinFileData['fileName'] = pinFileName
|
||||
output.append(pinFileData)
|
||||
return output
|
||||
|
||||
def parsePinFile(fn):
|
||||
x = {}
|
||||
f = open(fn,"r")
|
||||
x = yaml.load(f)
|
||||
f.close()
|
||||
validatePinFileData(x)
|
||||
return x
|
||||
|
||||
def validatePinFileData(pinFileStruct):
|
||||
# FIXME validate various file properties here
|
||||
pass
|
||||
|
||||
def getPinFileNames():
|
||||
thisDir = os.path.dirname(os.path.abspath(__file__))
|
||||
pinDir = os.path.join(thisDir, 'pins')
|
||||
pinFileNames = [
|
||||
os.path.join(pinDir,f) for f in os.listdir(pinDir)
|
||||
if os.path.isfile(os.path.join(pinDir, f))
|
||||
]
|
||||
return pinFileNames
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
{
|
||||
"contact": {
|
||||
"email": "sneak@sneak.berlin",
|
||||
"name": "Jeffrey Paul"
|
||||
},
|
||||
"description": "test name from ber1 (sneak fileserver)",
|
||||
"pins": [
|
||||
"/ipns/QmZKpS7iyUnDXcUxhou5arokZ5bZViZokEzDVAEGctBSMw"
|
||||
],
|
||||
"url": "https://sneak.berlin"
|
||||
}
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
description: test name from ber1
|
||||
url: https://sneak.berlin
|
||||
contact:
|
||||
name: Jeffrey Paul
|
||||
email: sneak@sneak.berlin
|
||||
pins:
|
||||
ipns: QmZKpS7iyUnDXcUxhou5arokZ5bZViZokEzDVAEGctBSMw
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"contact": {
|
||||
"email": "sneak@sneak.berlin",
|
||||
"name": "Jeffrey Paul"
|
||||
},
|
||||
"description": "this ipns will intentionally not resolve for testing",
|
||||
"pins": [
|
||||
"/ipns/QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAug11111111"
|
||||
]
|
||||
}
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
description: this ipns will intentionall not resolve
|
||||
contact:
|
||||
name: Jeffrey Paul
|
||||
email: sneak@sneak.berlin
|
||||
pins:
|
||||
ipns: QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAug11111111
|
||||
|
|
@ -1,14 +1,13 @@
|
|||
{
|
||||
"description": "go-ipfs v0.4.4 binaries",
|
||||
"pins":
|
||||
[
|
||||
"/ipfs/QmXvDdqieUAUo7srRVXvsKZRmzNGkuhRqWKYXLcXnBm4dc", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_darwin-386.tar.gz
|
||||
"/ipfs/QmPsMMi2Lq15QDNa8vmyZwwE15KbgUnHkQz3MX8pR5oUvq", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_darwin-amd64.tar.gz
|
||||
"/ipfs/QmQZyL5sWdXQyS9UwPHRsSv5yXz1cWRgTV9C6zS6EsGnBv", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_freebsd-amd64.tar.gz
|
||||
"/ipfs/QmRCXbtDRFUVLYndxxrobsPEKSBNJtPbvq4FTMKWBWqneE", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_linux-386.tar.gz
|
||||
"/ipfs/QmRE6QGDsRhHT81etCKKRvUucoQTY4KqjoH8RKW6jaF7kX", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_linux-amd64.tar.gz
|
||||
"/ipfs/QmUb7jGAgYd4AAGX82qY3Znt2Xrx3FLC2N6jiYVMcSTcMq", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_windows-386.zip
|
||||
"/ipfs/QmPPNuaTHcoFesnZuVjSMaxPVifunrVqFGj3avjTaKSLz3", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_windows-amd64.zip
|
||||
"/ipfs/QmdZ81FrNJYnjHbzuX6L2vqfBmEvRFwUgRU2TdPHU8WNLL" // go-ipfs_v0.4.4
|
||||
]
|
||||
}
|
||||
"pins": [
|
||||
"/ipfs/QmXvDdqieUAUo7srRVXvsKZRmzNGkuhRqWKYXLcXnBm4dc",
|
||||
"/ipfs/QmPsMMi2Lq15QDNa8vmyZwwE15KbgUnHkQz3MX8pR5oUvq",
|
||||
"/ipfs/QmQZyL5sWdXQyS9UwPHRsSv5yXz1cWRgTV9C6zS6EsGnBv",
|
||||
"/ipfs/QmRCXbtDRFUVLYndxxrobsPEKSBNJtPbvq4FTMKWBWqneE",
|
||||
"/ipfs/QmRE6QGDsRhHT81etCKKRvUucoQTY4KqjoH8RKW6jaF7kX",
|
||||
"/ipfs/QmUb7jGAgYd4AAGX82qY3Znt2Xrx3FLC2N6jiYVMcSTcMq",
|
||||
"/ipfs/QmPPNuaTHcoFesnZuVjSMaxPVifunrVqFGj3avjTaKSLz3",
|
||||
"/ipfs/QmdZ81FrNJYnjHbzuX6L2vqfBmEvRFwUgRU2TdPHU8WNLL"
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
{
|
||||
"description": "webui files",
|
||||
"url": "https://github.com/ipfs/refs-solarnet-storage/commit/df0a2cc4b53cea15132bec2d0b96b5d2d7a87ffb",
|
||||
"pins": [
|
||||
"/ipfs/QmS2HL9v5YeKgQkkWMvs1EMnFtUowTEdFfSSeMT4pos1e6",
|
||||
"/ipfs/QmXdu7HWdV6CUaUabd9q2ZeA4iHZLVyDRj3Gi4dsJsWjbr",
|
||||
|
|
@ -8,5 +7,6 @@
|
|||
"/ipfs/QmSHDxWsMPuJQKWmVA1rB5a3NX2Eme5fPqNb63qwaqiqSp",
|
||||
"/ipfs/QmctngrQAt9fjpQUZr7Bx3BsXUcif52eZGTizWhvcShsjz",
|
||||
"/ipfs/QmXX7YRpU7nNBKfw75VG7Y1c3GwpSAGHRev67XVPgZFv9R"
|
||||
]
|
||||
],
|
||||
"url": "https://github.com/ipfs/refs-solarnet-storage/commit/df0a2cc4b53cea15132bec2d0b96b5d2d7a87ffb"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
{
|
||||
"url": "https://sneak.berlin",
|
||||
"description": "Personal blog of Jeffrey Paul",
|
||||
"contact": {
|
||||
"name": "Jeffrey Paul",
|
||||
"email": "sneak@sneak.berlin"
|
||||
"email": "sneak@sneak.berlin",
|
||||
"name": "Jeffrey Paul"
|
||||
},
|
||||
"description": "Personal blog of Jeffrey Paul",
|
||||
"pins": [
|
||||
"/ipns/QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAugpyJJpCVp/website"
|
||||
]
|
||||
],
|
||||
"url": "https://sneak.berlin"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
colorlog==2.10.0
|
||||
PyYAML==3.12
|
||||
sanelogging==1.0.1
|
||||
Loading…
Reference in New Issue