parent
d029625142
commit
655026cdbd
@ -1,2 +1,3 @@ |
|||||||
venv |
venv |
||||||
pinlist.json |
pinlist.new |
||||||
|
pinlist.txt |
||||||
|
@ -1,19 +0,0 @@ |
|||||||
FROM ubuntu:16.04 |
|
||||||
|
|
||||||
VOLUME /var/lib/ipfs |
|
||||||
ENV IPFS_PATH /var/lib/ipfs |
|
||||||
|
|
||||||
ADD \ |
|
||||||
https://dist.ipfs.io/go-ipfs/v0.4.4/go-ipfs_v0.4.4_linux-amd64.tar.gz \ |
|
||||||
/usr/local/src |
|
||||||
|
|
||||||
ADD . /var/generate-pinlist |
|
||||||
|
|
||||||
RUN \ |
|
||||||
cd /usr/local/src && \ |
|
||||||
tar zxvf go-ipfs_v0.4.4_linux-amd64.tar.gz && \ |
|
||||||
mv go-ipfs/ipfs . && \ |
|
||||||
cd /var/generate-pinlist && \ |
|
||||||
pip -r requirements.txt && \ |
|
||||||
python generate.py && \ |
|
||||||
ipfs add pinlist.txt |
|
@ -0,0 +1,22 @@ |
|||||||
|
IPFS_ROOT := $(HOME)/Documents/sneak/my-ipfs-root
|
||||||
|
PINLIST_DIR := $(IPFS_ROOT)/pinlist
|
||||||
|
|
||||||
|
default: deploy |
||||||
|
|
||||||
|
.PHONY: deploy |
||||||
|
|
||||||
|
clean: |
||||||
|
rm -f pinlist.txt
|
||||||
|
|
||||||
|
deploy: pinlist.txt |
||||||
|
mkdir -p $(PINLIST_DIR)
|
||||||
|
rsync -avP ./pinlist.txt $(PINLIST_DIR)/pinlist.txt
|
||||||
|
ipfs add -r $(IPFS_ROOT) | \
|
||||||
|
tail -1 | \
|
||||||
|
awk -F' ' '{print $$2}' | \
|
||||||
|
ipfs name publish
|
||||||
|
|
||||||
|
|
||||||
|
pinlist.txt: pins/*.json |
||||||
|
jq -r .pins[] $^ > pinlist.new && mv pinlist.new $@
|
||||||
|
echo "# $$(date -u +%s)" >> $@
|
@ -1,166 +0,0 @@ |
|||||||
#!/usr/bin/env python |
|
||||||
|
|
||||||
#import ipfsapi |
|
||||||
from sanelogging import log |
|
||||||
from datetime import datetime |
|
||||||
import json |
|
||||||
import os |
|
||||||
import os.path |
|
||||||
import re |
|
||||||
import subprocess |
|
||||||
import yaml |
|
||||||
|
|
||||||
ipfsPathPattern = re.compile('\/ipfs\/([a-zA-Z0-9]+)') |
|
||||||
|
|
||||||
#def connectIpfs(hostname='127.0.0.1',port=5001): |
|
||||||
# ipfs = ipfsapi.connect(hostname,port) |
|
||||||
# log.info("connected to ipfsd: %s" % ipfs.id()['ProtocolVersion']) |
|
||||||
# return ipfs |
|
||||||
|
|
||||||
def main(): |
|
||||||
startTime = datetime.utcnow() |
|
||||||
log.info("generating pin list.") |
|
||||||
generatePinList() |
|
||||||
elapsed = datetime.utcnow() - startTime |
|
||||||
log.info("finished in %is" % elapsed.total_seconds()) |
|
||||||
|
|
||||||
def generatePinList(): |
|
||||||
pinFiles = readPinFiles() |
|
||||||
# ipfs = connectIpfs() |
|
||||||
|
|
||||||
resources = [] |
|
||||||
|
|
||||||
for pinFile in pinFiles: |
|
||||||
rs = extractResources(pinFile) |
|
||||||
resources.extend(rs) |
|
||||||
|
|
||||||
log.info("collected data from files.") |
|
||||||
|
|
||||||
resources = resolveResources(resources) |
|
||||||
|
|
||||||
output = { |
|
||||||
"generatedAt": datetime.utcnow().strftime("%s"), |
|
||||||
"pinList": resources |
|
||||||
} |
|
||||||
log.info("resources resolved, writing pinlist file") |
|
||||||
outputFileName = 'pinlist.json' |
|
||||||
fd = open(outputFileName + '.new','w') |
|
||||||
json.dump(output,fd) |
|
||||||
fd.close() |
|
||||||
os.rename(outputFileName + '.new', outputFileName) |
|
||||||
|
|
||||||
def extractResources(pinFile): |
|
||||||
# this pulls out the list of ipns/ipfs |
|
||||||
# items from the single file |
|
||||||
output = [] |
|
||||||
if 'ipns' in pinFile['pins']: |
|
||||||
if isinstance(pinFile['pins']['ipns'], basestring): |
|
||||||
# single ipns name |
|
||||||
output.append({ |
|
||||||
'type': 'ipns', |
|
||||||
'name': pinFile['pins']['ipns'], |
|
||||||
'fileName': os.path.basename(pinFile['fileName']) |
|
||||||
}) |
|
||||||
else: |
|
||||||
# list of ipns names |
|
||||||
for ipnsName in pinFile['pins']['ipns']: |
|
||||||
output.append({ |
|
||||||
'type': 'ipns', |
|
||||||
'name': ipnsName, |
|
||||||
'fileName': os.path.basename(pinFile['fileName']) |
|
||||||
}) |
|
||||||
|
|
||||||
if 'ipfs' in pinFile['pins']: |
|
||||||
if isinstance(pinFile['pins']['ipfs'], basestring): |
|
||||||
# single ipfs name |
|
||||||
output.append({ |
|
||||||
'type': 'ipfs', |
|
||||||
'name': pinFile['pins']['ipfs'], |
|
||||||
'fileName': os.path.basename(pinFile['fileName']) |
|
||||||
}) |
|
||||||
else: |
|
||||||
# list of ipfs names |
|
||||||
for ipfsName in pinFile['pins']['ipfs']: |
|
||||||
output.append({ |
|
||||||
'type': 'ipfs', |
|
||||||
'name': ipfsName, |
|
||||||
'fileName': os.path.basename(pinFile['fileName']) |
|
||||||
}) |
|
||||||
return output |
|
||||||
|
|
||||||
def resolveResources(resources): |
|
||||||
log.info("resolving resource names for pin list") |
|
||||||
for resource in resources: |
|
||||||
if resource['type'] == 'ipfs': |
|
||||||
resource['resolved'] = resource['name'] |
|
||||||
elif resource['type'] == 'ipns': |
|
||||||
# attepmt to resolve ipns name |
|
||||||
# FIXME try/catch |
|
||||||
res = None |
|
||||||
try: |
|
||||||
res = resolveIpnsName(resource['name']) |
|
||||||
if res: |
|
||||||
resource['resolved'] = res |
|
||||||
except LookupError: |
|
||||||
log.info( |
|
||||||
"unable to resolve ipns name '%s' from file '%s'" |
|
||||||
% (resource['name'], resource['fileName']) |
|
||||||
) |
|
||||||
resource['error'] = 'unable to resolve name' |
|
||||||
return resources |
|
||||||
|
|
||||||
def runCommand(cmd): |
|
||||||
p = subprocess.Popen( |
|
||||||
cmd, |
|
||||||
stdout=subprocess.PIPE, |
|
||||||
stderr=subprocess.STDOUT |
|
||||||
) |
|
||||||
retval = p.wait() |
|
||||||
if retval != 0: |
|
||||||
raise LookupError() |
|
||||||
return p.stdout.read() |
|
||||||
|
|
||||||
def resolveIpnsName(ipnsName): |
|
||||||
log.debug("resolving name '%s'" % ipnsName) |
|
||||||
res = runCommand(['ipfs', 'name', 'resolve', ipnsName]) |
|
||||||
ipfsNameMatch = ipfsPathPattern.match(res) |
|
||||||
if not ipfsNameMatch: |
|
||||||
log.info("unable to resolve ipns name '%s'" % ipnsName) |
|
||||||
return None |
|
||||||
ipfsName = ipfsNameMatch.group(1) |
|
||||||
log.info("resolved ipns name '%s' to '%s'" % (ipnsName, ipfsName)) |
|
||||||
return ipfsName |
|
||||||
|
|
||||||
def readPinFiles(): |
|
||||||
log.info("reading pinfiles...") |
|
||||||
output = [] |
|
||||||
pinFiles = getPinFileNames() |
|
||||||
for pinFileName in pinFiles: |
|
||||||
pinFileData = parsePinFile(pinFileName) |
|
||||||
pinFileData['fileName'] = pinFileName |
|
||||||
output.append(pinFileData) |
|
||||||
return output |
|
||||||
|
|
||||||
def parsePinFile(fn): |
|
||||||
x = {} |
|
||||||
f = open(fn,"r") |
|
||||||
x = yaml.load(f) |
|
||||||
f.close() |
|
||||||
validatePinFileData(x) |
|
||||||
return x |
|
||||||
|
|
||||||
def validatePinFileData(pinFileStruct): |
|
||||||
# FIXME validate various file properties here |
|
||||||
pass |
|
||||||
|
|
||||||
def getPinFileNames(): |
|
||||||
thisDir = os.path.dirname(os.path.abspath(__file__)) |
|
||||||
pinDir = os.path.join(thisDir, 'pins') |
|
||||||
pinFileNames = [ |
|
||||||
os.path.join(pinDir,f) for f in os.listdir(pinDir) |
|
||||||
if os.path.isfile(os.path.join(pinDir, f)) |
|
||||||
] |
|
||||||
return pinFileNames |
|
||||||
|
|
||||||
if __name__ == "__main__": |
|
||||||
main() |
|
@ -0,0 +1,11 @@ |
|||||||
|
{ |
||||||
|
"contact": { |
||||||
|
"email": "sneak@sneak.berlin", |
||||||
|
"name": "Jeffrey Paul" |
||||||
|
}, |
||||||
|
"description": "test name from ber1 (sneak fileserver)", |
||||||
|
"pins": [ |
||||||
|
"/ipns/QmZKpS7iyUnDXcUxhou5arokZ5bZViZokEzDVAEGctBSMw" |
||||||
|
], |
||||||
|
"url": "https://sneak.berlin" |
||||||
|
} |
@ -1,7 +0,0 @@ |
|||||||
description: test name from ber1 |
|
||||||
url: https://sneak.berlin |
|
||||||
contact: |
|
||||||
name: Jeffrey Paul |
|
||||||
email: sneak@sneak.berlin |
|
||||||
pins: |
|
||||||
ipns: QmZKpS7iyUnDXcUxhou5arokZ5bZViZokEzDVAEGctBSMw |
|
@ -0,0 +1,10 @@ |
|||||||
|
{ |
||||||
|
"contact": { |
||||||
|
"email": "sneak@sneak.berlin", |
||||||
|
"name": "Jeffrey Paul" |
||||||
|
}, |
||||||
|
"description": "this ipns will intentionally not resolve for testing", |
||||||
|
"pins": [ |
||||||
|
"/ipns/QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAug11111111" |
||||||
|
] |
||||||
|
} |
@ -1,6 +0,0 @@ |
|||||||
description: this ipns will intentionall not resolve |
|
||||||
contact: |
|
||||||
name: Jeffrey Paul |
|
||||||
email: sneak@sneak.berlin |
|
||||||
pins: |
|
||||||
ipns: QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAug11111111 |
|
@ -1,14 +1,13 @@ |
|||||||
{ |
{ |
||||||
"description": "go-ipfs v0.4.4 binaries", |
"description": "go-ipfs v0.4.4 binaries", |
||||||
"pins": |
"pins": [ |
||||||
[ |
"/ipfs/QmXvDdqieUAUo7srRVXvsKZRmzNGkuhRqWKYXLcXnBm4dc", |
||||||
"/ipfs/QmXvDdqieUAUo7srRVXvsKZRmzNGkuhRqWKYXLcXnBm4dc", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_darwin-386.tar.gz |
"/ipfs/QmPsMMi2Lq15QDNa8vmyZwwE15KbgUnHkQz3MX8pR5oUvq", |
||||||
"/ipfs/QmPsMMi2Lq15QDNa8vmyZwwE15KbgUnHkQz3MX8pR5oUvq", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_darwin-amd64.tar.gz |
"/ipfs/QmQZyL5sWdXQyS9UwPHRsSv5yXz1cWRgTV9C6zS6EsGnBv", |
||||||
"/ipfs/QmQZyL5sWdXQyS9UwPHRsSv5yXz1cWRgTV9C6zS6EsGnBv", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_freebsd-amd64.tar.gz |
"/ipfs/QmRCXbtDRFUVLYndxxrobsPEKSBNJtPbvq4FTMKWBWqneE", |
||||||
"/ipfs/QmRCXbtDRFUVLYndxxrobsPEKSBNJtPbvq4FTMKWBWqneE", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_linux-386.tar.gz |
"/ipfs/QmRE6QGDsRhHT81etCKKRvUucoQTY4KqjoH8RKW6jaF7kX", |
||||||
"/ipfs/QmRE6QGDsRhHT81etCKKRvUucoQTY4KqjoH8RKW6jaF7kX", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_linux-amd64.tar.gz |
"/ipfs/QmUb7jGAgYd4AAGX82qY3Znt2Xrx3FLC2N6jiYVMcSTcMq", |
||||||
"/ipfs/QmUb7jGAgYd4AAGX82qY3Znt2Xrx3FLC2N6jiYVMcSTcMq", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_windows-386.zip |
"/ipfs/QmPPNuaTHcoFesnZuVjSMaxPVifunrVqFGj3avjTaKSLz3", |
||||||
"/ipfs/QmPPNuaTHcoFesnZuVjSMaxPVifunrVqFGj3avjTaKSLz3", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_windows-amd64.zip |
"/ipfs/QmdZ81FrNJYnjHbzuX6L2vqfBmEvRFwUgRU2TdPHU8WNLL" |
||||||
"/ipfs/QmdZ81FrNJYnjHbzuX6L2vqfBmEvRFwUgRU2TdPHU8WNLL" // go-ipfs_v0.4.4 |
] |
||||||
] |
} |
||||||
} |
|
||||||
|
@ -1,11 +1,11 @@ |
|||||||
{ |
{ |
||||||
"url": "https://sneak.berlin", |
|
||||||
"description": "Personal blog of Jeffrey Paul", |
|
||||||
"contact": { |
"contact": { |
||||||
"name": "Jeffrey Paul", |
"email": "sneak@sneak.berlin", |
||||||
"email": "sneak@sneak.berlin" |
"name": "Jeffrey Paul" |
||||||
}, |
}, |
||||||
|
"description": "Personal blog of Jeffrey Paul", |
||||||
"pins": [ |
"pins": [ |
||||||
"/ipns/QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAugpyJJpCVp/website" |
"/ipns/QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAugpyJJpCVp/website" |
||||||
] |
], |
||||||
|
"url": "https://sneak.berlin" |
||||||
} |
} |
||||||
|
@ -1,3 +0,0 @@ |
|||||||
colorlog==2.10.0 |
|
||||||
PyYAML==3.12 |
|
||||||
sanelogging==1.0.1 |
|
Loading…
Reference in new issue