parent
d029625142
commit
655026cdbd
@ -1,2 +1,3 @@ |
||||
venv |
||||
pinlist.json |
||||
pinlist.new |
||||
pinlist.txt |
||||
|
@ -1,19 +0,0 @@ |
||||
FROM ubuntu:16.04 |
||||
|
||||
VOLUME /var/lib/ipfs |
||||
ENV IPFS_PATH /var/lib/ipfs |
||||
|
||||
ADD \ |
||||
https://dist.ipfs.io/go-ipfs/v0.4.4/go-ipfs_v0.4.4_linux-amd64.tar.gz \ |
||||
/usr/local/src |
||||
|
||||
ADD . /var/generate-pinlist |
||||
|
||||
RUN \ |
||||
cd /usr/local/src && \ |
||||
tar zxvf go-ipfs_v0.4.4_linux-amd64.tar.gz && \ |
||||
mv go-ipfs/ipfs . && \ |
||||
cd /var/generate-pinlist && \ |
||||
pip -r requirements.txt && \ |
||||
python generate.py && \ |
||||
ipfs add pinlist.txt |
@ -0,0 +1,22 @@ |
||||
IPFS_ROOT := $(HOME)/Documents/sneak/my-ipfs-root
|
||||
PINLIST_DIR := $(IPFS_ROOT)/pinlist
|
||||
|
||||
default: deploy |
||||
|
||||
.PHONY: deploy |
||||
|
||||
clean: |
||||
rm -f pinlist.txt
|
||||
|
||||
deploy: pinlist.txt |
||||
mkdir -p $(PINLIST_DIR)
|
||||
rsync -avP ./pinlist.txt $(PINLIST_DIR)/pinlist.txt
|
||||
ipfs add -r $(IPFS_ROOT) | \
|
||||
tail -1 | \
|
||||
awk -F' ' '{print $$2}' | \
|
||||
ipfs name publish
|
||||
|
||||
|
||||
pinlist.txt: pins/*.json |
||||
jq -r .pins[] $^ > pinlist.new && mv pinlist.new $@
|
||||
echo "# $$(date -u +%s)" >> $@
|
@ -1,166 +0,0 @@ |
||||
#!/usr/bin/env python |
||||
|
||||
#import ipfsapi |
||||
from sanelogging import log |
||||
from datetime import datetime |
||||
import json |
||||
import os |
||||
import os.path |
||||
import re |
||||
import subprocess |
||||
import yaml |
||||
|
||||
ipfsPathPattern = re.compile('\/ipfs\/([a-zA-Z0-9]+)') |
||||
|
||||
#def connectIpfs(hostname='127.0.0.1',port=5001): |
||||
# ipfs = ipfsapi.connect(hostname,port) |
||||
# log.info("connected to ipfsd: %s" % ipfs.id()['ProtocolVersion']) |
||||
# return ipfs |
||||
|
||||
def main(): |
||||
startTime = datetime.utcnow() |
||||
log.info("generating pin list.") |
||||
generatePinList() |
||||
elapsed = datetime.utcnow() - startTime |
||||
log.info("finished in %is" % elapsed.total_seconds()) |
||||
|
||||
def generatePinList(): |
||||
pinFiles = readPinFiles() |
||||
# ipfs = connectIpfs() |
||||
|
||||
resources = [] |
||||
|
||||
for pinFile in pinFiles: |
||||
rs = extractResources(pinFile) |
||||
resources.extend(rs) |
||||
|
||||
log.info("collected data from files.") |
||||
|
||||
resources = resolveResources(resources) |
||||
|
||||
output = { |
||||
"generatedAt": datetime.utcnow().strftime("%s"), |
||||
"pinList": resources |
||||
} |
||||
log.info("resources resolved, writing pinlist file") |
||||
outputFileName = 'pinlist.json' |
||||
fd = open(outputFileName + '.new','w') |
||||
json.dump(output,fd) |
||||
fd.close() |
||||
os.rename(outputFileName + '.new', outputFileName) |
||||
|
||||
def extractResources(pinFile): |
||||
# this pulls out the list of ipns/ipfs |
||||
# items from the single file |
||||
output = [] |
||||
if 'ipns' in pinFile['pins']: |
||||
if isinstance(pinFile['pins']['ipns'], basestring): |
||||
# single ipns name |
||||
output.append({ |
||||
'type': 'ipns', |
||||
'name': pinFile['pins']['ipns'], |
||||
'fileName': os.path.basename(pinFile['fileName']) |
||||
}) |
||||
else: |
||||
# list of ipns names |
||||
for ipnsName in pinFile['pins']['ipns']: |
||||
output.append({ |
||||
'type': 'ipns', |
||||
'name': ipnsName, |
||||
'fileName': os.path.basename(pinFile['fileName']) |
||||
}) |
||||
|
||||
if 'ipfs' in pinFile['pins']: |
||||
if isinstance(pinFile['pins']['ipfs'], basestring): |
||||
# single ipfs name |
||||
output.append({ |
||||
'type': 'ipfs', |
||||
'name': pinFile['pins']['ipfs'], |
||||
'fileName': os.path.basename(pinFile['fileName']) |
||||
}) |
||||
else: |
||||
# list of ipfs names |
||||
for ipfsName in pinFile['pins']['ipfs']: |
||||
output.append({ |
||||
'type': 'ipfs', |
||||
'name': ipfsName, |
||||
'fileName': os.path.basename(pinFile['fileName']) |
||||
}) |
||||
return output |
||||
|
||||
def resolveResources(resources): |
||||
log.info("resolving resource names for pin list") |
||||
for resource in resources: |
||||
if resource['type'] == 'ipfs': |
||||
resource['resolved'] = resource['name'] |
||||
elif resource['type'] == 'ipns': |
||||
# attepmt to resolve ipns name |
||||
# FIXME try/catch |
||||
res = None |
||||
try: |
||||
res = resolveIpnsName(resource['name']) |
||||
if res: |
||||
resource['resolved'] = res |
||||
except LookupError: |
||||
log.info( |
||||
"unable to resolve ipns name '%s' from file '%s'" |
||||
% (resource['name'], resource['fileName']) |
||||
) |
||||
resource['error'] = 'unable to resolve name' |
||||
return resources |
||||
|
||||
def runCommand(cmd): |
||||
p = subprocess.Popen( |
||||
cmd, |
||||
stdout=subprocess.PIPE, |
||||
stderr=subprocess.STDOUT |
||||
) |
||||
retval = p.wait() |
||||
if retval != 0: |
||||
raise LookupError() |
||||
return p.stdout.read() |
||||
|
||||
def resolveIpnsName(ipnsName): |
||||
log.debug("resolving name '%s'" % ipnsName) |
||||
res = runCommand(['ipfs', 'name', 'resolve', ipnsName]) |
||||
ipfsNameMatch = ipfsPathPattern.match(res) |
||||
if not ipfsNameMatch: |
||||
log.info("unable to resolve ipns name '%s'" % ipnsName) |
||||
return None |
||||
ipfsName = ipfsNameMatch.group(1) |
||||
log.info("resolved ipns name '%s' to '%s'" % (ipnsName, ipfsName)) |
||||
return ipfsName |
||||
|
||||
def readPinFiles(): |
||||
log.info("reading pinfiles...") |
||||
output = [] |
||||
pinFiles = getPinFileNames() |
||||
for pinFileName in pinFiles: |
||||
pinFileData = parsePinFile(pinFileName) |
||||
pinFileData['fileName'] = pinFileName |
||||
output.append(pinFileData) |
||||
return output |
||||
|
||||
def parsePinFile(fn): |
||||
x = {} |
||||
f = open(fn,"r") |
||||
x = yaml.load(f) |
||||
f.close() |
||||
validatePinFileData(x) |
||||
return x |
||||
|
||||
def validatePinFileData(pinFileStruct): |
||||
# FIXME validate various file properties here |
||||
pass |
||||
|
||||
def getPinFileNames(): |
||||
thisDir = os.path.dirname(os.path.abspath(__file__)) |
||||
pinDir = os.path.join(thisDir, 'pins') |
||||
pinFileNames = [ |
||||
os.path.join(pinDir,f) for f in os.listdir(pinDir) |
||||
if os.path.isfile(os.path.join(pinDir, f)) |
||||
] |
||||
return pinFileNames |
||||
|
||||
if __name__ == "__main__": |
||||
main() |
@ -0,0 +1,11 @@ |
||||
{ |
||||
"contact": { |
||||
"email": "sneak@sneak.berlin", |
||||
"name": "Jeffrey Paul" |
||||
}, |
||||
"description": "test name from ber1 (sneak fileserver)", |
||||
"pins": [ |
||||
"/ipns/QmZKpS7iyUnDXcUxhou5arokZ5bZViZokEzDVAEGctBSMw" |
||||
], |
||||
"url": "https://sneak.berlin" |
||||
} |
@ -1,7 +0,0 @@ |
||||
description: test name from ber1 |
||||
url: https://sneak.berlin |
||||
contact: |
||||
name: Jeffrey Paul |
||||
email: sneak@sneak.berlin |
||||
pins: |
||||
ipns: QmZKpS7iyUnDXcUxhou5arokZ5bZViZokEzDVAEGctBSMw |
@ -0,0 +1,10 @@ |
||||
{ |
||||
"contact": { |
||||
"email": "sneak@sneak.berlin", |
||||
"name": "Jeffrey Paul" |
||||
}, |
||||
"description": "this ipns will intentionally not resolve for testing", |
||||
"pins": [ |
||||
"/ipns/QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAug11111111" |
||||
] |
||||
} |
@ -1,6 +0,0 @@ |
||||
description: this ipns will intentionall not resolve |
||||
contact: |
||||
name: Jeffrey Paul |
||||
email: sneak@sneak.berlin |
||||
pins: |
||||
ipns: QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAug11111111 |
@ -1,14 +1,13 @@ |
||||
{ |
||||
"description": "go-ipfs v0.4.4 binaries", |
||||
"pins": |
||||
[ |
||||
"/ipfs/QmXvDdqieUAUo7srRVXvsKZRmzNGkuhRqWKYXLcXnBm4dc", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_darwin-386.tar.gz |
||||
"/ipfs/QmPsMMi2Lq15QDNa8vmyZwwE15KbgUnHkQz3MX8pR5oUvq", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_darwin-amd64.tar.gz |
||||
"/ipfs/QmQZyL5sWdXQyS9UwPHRsSv5yXz1cWRgTV9C6zS6EsGnBv", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_freebsd-amd64.tar.gz |
||||
"/ipfs/QmRCXbtDRFUVLYndxxrobsPEKSBNJtPbvq4FTMKWBWqneE", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_linux-386.tar.gz |
||||
"/ipfs/QmRE6QGDsRhHT81etCKKRvUucoQTY4KqjoH8RKW6jaF7kX", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_linux-amd64.tar.gz |
||||
"/ipfs/QmUb7jGAgYd4AAGX82qY3Znt2Xrx3FLC2N6jiYVMcSTcMq", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_windows-386.zip |
||||
"/ipfs/QmPPNuaTHcoFesnZuVjSMaxPVifunrVqFGj3avjTaKSLz3", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_windows-amd64.zip |
||||
"/ipfs/QmdZ81FrNJYnjHbzuX6L2vqfBmEvRFwUgRU2TdPHU8WNLL" // go-ipfs_v0.4.4 |
||||
] |
||||
} |
||||
"pins": [ |
||||
"/ipfs/QmXvDdqieUAUo7srRVXvsKZRmzNGkuhRqWKYXLcXnBm4dc", |
||||
"/ipfs/QmPsMMi2Lq15QDNa8vmyZwwE15KbgUnHkQz3MX8pR5oUvq", |
||||
"/ipfs/QmQZyL5sWdXQyS9UwPHRsSv5yXz1cWRgTV9C6zS6EsGnBv", |
||||
"/ipfs/QmRCXbtDRFUVLYndxxrobsPEKSBNJtPbvq4FTMKWBWqneE", |
||||
"/ipfs/QmRE6QGDsRhHT81etCKKRvUucoQTY4KqjoH8RKW6jaF7kX", |
||||
"/ipfs/QmUb7jGAgYd4AAGX82qY3Znt2Xrx3FLC2N6jiYVMcSTcMq", |
||||
"/ipfs/QmPPNuaTHcoFesnZuVjSMaxPVifunrVqFGj3avjTaKSLz3", |
||||
"/ipfs/QmdZ81FrNJYnjHbzuX6L2vqfBmEvRFwUgRU2TdPHU8WNLL" |
||||
] |
||||
} |
||||
|
@ -1,11 +1,11 @@ |
||||
{ |
||||
"url": "https://sneak.berlin", |
||||
"description": "Personal blog of Jeffrey Paul", |
||||
"contact": { |
||||
"name": "Jeffrey Paul", |
||||
"email": "sneak@sneak.berlin" |
||||
"email": "sneak@sneak.berlin", |
||||
"name": "Jeffrey Paul" |
||||
}, |
||||
"description": "Personal blog of Jeffrey Paul", |
||||
"pins": [ |
||||
"/ipns/QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAugpyJJpCVp/website" |
||||
] |
||||
], |
||||
"url": "https://sneak.berlin" |
||||
} |
||||
|
@ -1,3 +0,0 @@ |
||||
colorlog==2.10.0 |
||||
PyYAML==3.12 |
||||
sanelogging==1.0.1 |
Loading…
Reference in new issue