latest
This commit is contained in:
parent
d029625142
commit
655026cdbd
3
.gitignore
vendored
3
.gitignore
vendored
@ -1,2 +1,3 @@
|
|||||||
venv
|
venv
|
||||||
pinlist.json
|
pinlist.new
|
||||||
|
pinlist.txt
|
||||||
|
|||||||
19
Dockerfile
19
Dockerfile
@ -1,19 +0,0 @@
|
|||||||
FROM ubuntu:16.04
|
|
||||||
|
|
||||||
VOLUME /var/lib/ipfs
|
|
||||||
ENV IPFS_PATH /var/lib/ipfs
|
|
||||||
|
|
||||||
ADD \
|
|
||||||
https://dist.ipfs.io/go-ipfs/v0.4.4/go-ipfs_v0.4.4_linux-amd64.tar.gz \
|
|
||||||
/usr/local/src
|
|
||||||
|
|
||||||
ADD . /var/generate-pinlist
|
|
||||||
|
|
||||||
RUN \
|
|
||||||
cd /usr/local/src && \
|
|
||||||
tar zxvf go-ipfs_v0.4.4_linux-amd64.tar.gz && \
|
|
||||||
mv go-ipfs/ipfs . && \
|
|
||||||
cd /var/generate-pinlist && \
|
|
||||||
pip -r requirements.txt && \
|
|
||||||
python generate.py && \
|
|
||||||
ipfs add pinlist.txt
|
|
||||||
22
Makefile
Normal file
22
Makefile
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
IPFS_ROOT := $(HOME)/Documents/sneak/my-ipfs-root
|
||||||
|
PINLIST_DIR := $(IPFS_ROOT)/pinlist
|
||||||
|
|
||||||
|
default: deploy
|
||||||
|
|
||||||
|
.PHONY: deploy
|
||||||
|
|
||||||
|
clean:
|
||||||
|
rm -f pinlist.txt
|
||||||
|
|
||||||
|
deploy: pinlist.txt
|
||||||
|
mkdir -p $(PINLIST_DIR)
|
||||||
|
rsync -avP ./pinlist.txt $(PINLIST_DIR)/pinlist.txt
|
||||||
|
ipfs add -r $(IPFS_ROOT) | \
|
||||||
|
tail -1 | \
|
||||||
|
awk -F' ' '{print $$2}' | \
|
||||||
|
ipfs name publish
|
||||||
|
|
||||||
|
|
||||||
|
pinlist.txt: pins/*.json
|
||||||
|
jq -r .pins[] $^ > pinlist.new && mv pinlist.new $@
|
||||||
|
echo "# $$(date -u +%s)" >> $@
|
||||||
166
generate.py
166
generate.py
@ -1,166 +0,0 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
|
|
||||||
#import ipfsapi
|
|
||||||
from sanelogging import log
|
|
||||||
from datetime import datetime
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import os.path
|
|
||||||
import re
|
|
||||||
import subprocess
|
|
||||||
import yaml
|
|
||||||
|
|
||||||
ipfsPathPattern = re.compile('\/ipfs\/([a-zA-Z0-9]+)')
|
|
||||||
|
|
||||||
#def connectIpfs(hostname='127.0.0.1',port=5001):
|
|
||||||
# ipfs = ipfsapi.connect(hostname,port)
|
|
||||||
# log.info("connected to ipfsd: %s" % ipfs.id()['ProtocolVersion'])
|
|
||||||
# return ipfs
|
|
||||||
|
|
||||||
def main():
|
|
||||||
startTime = datetime.utcnow()
|
|
||||||
log.info("generating pin list.")
|
|
||||||
generatePinList()
|
|
||||||
elapsed = datetime.utcnow() - startTime
|
|
||||||
log.info("finished in %is" % elapsed.total_seconds())
|
|
||||||
|
|
||||||
def generatePinList():
|
|
||||||
pinFiles = readPinFiles()
|
|
||||||
# ipfs = connectIpfs()
|
|
||||||
|
|
||||||
resources = []
|
|
||||||
|
|
||||||
for pinFile in pinFiles:
|
|
||||||
rs = extractResources(pinFile)
|
|
||||||
resources.extend(rs)
|
|
||||||
|
|
||||||
log.info("collected data from files.")
|
|
||||||
|
|
||||||
resources = resolveResources(resources)
|
|
||||||
|
|
||||||
output = {
|
|
||||||
"generatedAt": datetime.utcnow().strftime("%s"),
|
|
||||||
"pinList": resources
|
|
||||||
}
|
|
||||||
log.info("resources resolved, writing pinlist file")
|
|
||||||
outputFileName = 'pinlist.json'
|
|
||||||
fd = open(outputFileName + '.new','w')
|
|
||||||
json.dump(output,fd)
|
|
||||||
fd.close()
|
|
||||||
os.rename(outputFileName + '.new', outputFileName)
|
|
||||||
|
|
||||||
def extractResources(pinFile):
|
|
||||||
# this pulls out the list of ipns/ipfs
|
|
||||||
# items from the single file
|
|
||||||
output = []
|
|
||||||
if 'ipns' in pinFile['pins']:
|
|
||||||
if isinstance(pinFile['pins']['ipns'], basestring):
|
|
||||||
# single ipns name
|
|
||||||
output.append({
|
|
||||||
'type': 'ipns',
|
|
||||||
'name': pinFile['pins']['ipns'],
|
|
||||||
'fileName': os.path.basename(pinFile['fileName'])
|
|
||||||
})
|
|
||||||
else:
|
|
||||||
# list of ipns names
|
|
||||||
for ipnsName in pinFile['pins']['ipns']:
|
|
||||||
output.append({
|
|
||||||
'type': 'ipns',
|
|
||||||
'name': ipnsName,
|
|
||||||
'fileName': os.path.basename(pinFile['fileName'])
|
|
||||||
})
|
|
||||||
|
|
||||||
if 'ipfs' in pinFile['pins']:
|
|
||||||
if isinstance(pinFile['pins']['ipfs'], basestring):
|
|
||||||
# single ipfs name
|
|
||||||
output.append({
|
|
||||||
'type': 'ipfs',
|
|
||||||
'name': pinFile['pins']['ipfs'],
|
|
||||||
'fileName': os.path.basename(pinFile['fileName'])
|
|
||||||
})
|
|
||||||
else:
|
|
||||||
# list of ipfs names
|
|
||||||
for ipfsName in pinFile['pins']['ipfs']:
|
|
||||||
output.append({
|
|
||||||
'type': 'ipfs',
|
|
||||||
'name': ipfsName,
|
|
||||||
'fileName': os.path.basename(pinFile['fileName'])
|
|
||||||
})
|
|
||||||
return output
|
|
||||||
|
|
||||||
def resolveResources(resources):
|
|
||||||
log.info("resolving resource names for pin list")
|
|
||||||
for resource in resources:
|
|
||||||
if resource['type'] == 'ipfs':
|
|
||||||
resource['resolved'] = resource['name']
|
|
||||||
elif resource['type'] == 'ipns':
|
|
||||||
# attepmt to resolve ipns name
|
|
||||||
# FIXME try/catch
|
|
||||||
res = None
|
|
||||||
try:
|
|
||||||
res = resolveIpnsName(resource['name'])
|
|
||||||
if res:
|
|
||||||
resource['resolved'] = res
|
|
||||||
except LookupError:
|
|
||||||
log.info(
|
|
||||||
"unable to resolve ipns name '%s' from file '%s'"
|
|
||||||
% (resource['name'], resource['fileName'])
|
|
||||||
)
|
|
||||||
resource['error'] = 'unable to resolve name'
|
|
||||||
return resources
|
|
||||||
|
|
||||||
def runCommand(cmd):
|
|
||||||
p = subprocess.Popen(
|
|
||||||
cmd,
|
|
||||||
stdout=subprocess.PIPE,
|
|
||||||
stderr=subprocess.STDOUT
|
|
||||||
)
|
|
||||||
retval = p.wait()
|
|
||||||
if retval != 0:
|
|
||||||
raise LookupError()
|
|
||||||
return p.stdout.read()
|
|
||||||
|
|
||||||
def resolveIpnsName(ipnsName):
|
|
||||||
log.debug("resolving name '%s'" % ipnsName)
|
|
||||||
res = runCommand(['ipfs', 'name', 'resolve', ipnsName])
|
|
||||||
ipfsNameMatch = ipfsPathPattern.match(res)
|
|
||||||
if not ipfsNameMatch:
|
|
||||||
log.info("unable to resolve ipns name '%s'" % ipnsName)
|
|
||||||
return None
|
|
||||||
ipfsName = ipfsNameMatch.group(1)
|
|
||||||
log.info("resolved ipns name '%s' to '%s'" % (ipnsName, ipfsName))
|
|
||||||
return ipfsName
|
|
||||||
|
|
||||||
def readPinFiles():
|
|
||||||
log.info("reading pinfiles...")
|
|
||||||
output = []
|
|
||||||
pinFiles = getPinFileNames()
|
|
||||||
for pinFileName in pinFiles:
|
|
||||||
pinFileData = parsePinFile(pinFileName)
|
|
||||||
pinFileData['fileName'] = pinFileName
|
|
||||||
output.append(pinFileData)
|
|
||||||
return output
|
|
||||||
|
|
||||||
def parsePinFile(fn):
|
|
||||||
x = {}
|
|
||||||
f = open(fn,"r")
|
|
||||||
x = yaml.load(f)
|
|
||||||
f.close()
|
|
||||||
validatePinFileData(x)
|
|
||||||
return x
|
|
||||||
|
|
||||||
def validatePinFileData(pinFileStruct):
|
|
||||||
# FIXME validate various file properties here
|
|
||||||
pass
|
|
||||||
|
|
||||||
def getPinFileNames():
|
|
||||||
thisDir = os.path.dirname(os.path.abspath(__file__))
|
|
||||||
pinDir = os.path.join(thisDir, 'pins')
|
|
||||||
pinFileNames = [
|
|
||||||
os.path.join(pinDir,f) for f in os.listdir(pinDir)
|
|
||||||
if os.path.isfile(os.path.join(pinDir, f))
|
|
||||||
]
|
|
||||||
return pinFileNames
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
11
pins/ber1-test-name.json
Normal file
11
pins/ber1-test-name.json
Normal file
@ -0,0 +1,11 @@
|
|||||||
|
{
|
||||||
|
"contact": {
|
||||||
|
"email": "sneak@sneak.berlin",
|
||||||
|
"name": "Jeffrey Paul"
|
||||||
|
},
|
||||||
|
"description": "test name from ber1 (sneak fileserver)",
|
||||||
|
"pins": [
|
||||||
|
"/ipns/QmZKpS7iyUnDXcUxhou5arokZ5bZViZokEzDVAEGctBSMw"
|
||||||
|
],
|
||||||
|
"url": "https://sneak.berlin"
|
||||||
|
}
|
||||||
@ -1,7 +0,0 @@
|
|||||||
description: test name from ber1
|
|
||||||
url: https://sneak.berlin
|
|
||||||
contact:
|
|
||||||
name: Jeffrey Paul
|
|
||||||
email: sneak@sneak.berlin
|
|
||||||
pins:
|
|
||||||
ipns: QmZKpS7iyUnDXcUxhou5arokZ5bZViZokEzDVAEGctBSMw
|
|
||||||
10
pins/fails.json
Normal file
10
pins/fails.json
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
{
|
||||||
|
"contact": {
|
||||||
|
"email": "sneak@sneak.berlin",
|
||||||
|
"name": "Jeffrey Paul"
|
||||||
|
},
|
||||||
|
"description": "this ipns will intentionally not resolve for testing",
|
||||||
|
"pins": [
|
||||||
|
"/ipns/QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAug11111111"
|
||||||
|
]
|
||||||
|
}
|
||||||
@ -1,6 +0,0 @@
|
|||||||
description: this ipns will intentionall not resolve
|
|
||||||
contact:
|
|
||||||
name: Jeffrey Paul
|
|
||||||
email: sneak@sneak.berlin
|
|
||||||
pins:
|
|
||||||
ipns: QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAug11111111
|
|
||||||
@ -1,14 +1,13 @@
|
|||||||
{
|
{
|
||||||
"description": "go-ipfs v0.4.4 binaries",
|
"description": "go-ipfs v0.4.4 binaries",
|
||||||
"pins":
|
"pins": [
|
||||||
[
|
"/ipfs/QmXvDdqieUAUo7srRVXvsKZRmzNGkuhRqWKYXLcXnBm4dc",
|
||||||
"/ipfs/QmXvDdqieUAUo7srRVXvsKZRmzNGkuhRqWKYXLcXnBm4dc", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_darwin-386.tar.gz
|
"/ipfs/QmPsMMi2Lq15QDNa8vmyZwwE15KbgUnHkQz3MX8pR5oUvq",
|
||||||
"/ipfs/QmPsMMi2Lq15QDNa8vmyZwwE15KbgUnHkQz3MX8pR5oUvq", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_darwin-amd64.tar.gz
|
"/ipfs/QmQZyL5sWdXQyS9UwPHRsSv5yXz1cWRgTV9C6zS6EsGnBv",
|
||||||
"/ipfs/QmQZyL5sWdXQyS9UwPHRsSv5yXz1cWRgTV9C6zS6EsGnBv", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_freebsd-amd64.tar.gz
|
"/ipfs/QmRCXbtDRFUVLYndxxrobsPEKSBNJtPbvq4FTMKWBWqneE",
|
||||||
"/ipfs/QmRCXbtDRFUVLYndxxrobsPEKSBNJtPbvq4FTMKWBWqneE", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_linux-386.tar.gz
|
"/ipfs/QmRE6QGDsRhHT81etCKKRvUucoQTY4KqjoH8RKW6jaF7kX",
|
||||||
"/ipfs/QmRE6QGDsRhHT81etCKKRvUucoQTY4KqjoH8RKW6jaF7kX", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_linux-amd64.tar.gz
|
"/ipfs/QmUb7jGAgYd4AAGX82qY3Znt2Xrx3FLC2N6jiYVMcSTcMq",
|
||||||
"/ipfs/QmUb7jGAgYd4AAGX82qY3Znt2Xrx3FLC2N6jiYVMcSTcMq", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_windows-386.zip
|
"/ipfs/QmPPNuaTHcoFesnZuVjSMaxPVifunrVqFGj3avjTaKSLz3",
|
||||||
"/ipfs/QmPPNuaTHcoFesnZuVjSMaxPVifunrVqFGj3avjTaKSLz3", // go-ipfs_v0.4.4/go-ipfs_v0.4.4_windows-amd64.zip
|
"/ipfs/QmdZ81FrNJYnjHbzuX6L2vqfBmEvRFwUgRU2TdPHU8WNLL"
|
||||||
"/ipfs/QmdZ81FrNJYnjHbzuX6L2vqfBmEvRFwUgRU2TdPHU8WNLL" // go-ipfs_v0.4.4
|
]
|
||||||
]
|
}
|
||||||
}
|
|
||||||
|
|||||||
@ -1,6 +1,5 @@
|
|||||||
{
|
{
|
||||||
"description": "webui files",
|
"description": "webui files",
|
||||||
"url": "https://github.com/ipfs/refs-solarnet-storage/commit/df0a2cc4b53cea15132bec2d0b96b5d2d7a87ffb",
|
|
||||||
"pins": [
|
"pins": [
|
||||||
"/ipfs/QmS2HL9v5YeKgQkkWMvs1EMnFtUowTEdFfSSeMT4pos1e6",
|
"/ipfs/QmS2HL9v5YeKgQkkWMvs1EMnFtUowTEdFfSSeMT4pos1e6",
|
||||||
"/ipfs/QmXdu7HWdV6CUaUabd9q2ZeA4iHZLVyDRj3Gi4dsJsWjbr",
|
"/ipfs/QmXdu7HWdV6CUaUabd9q2ZeA4iHZLVyDRj3Gi4dsJsWjbr",
|
||||||
@ -8,5 +7,6 @@
|
|||||||
"/ipfs/QmSHDxWsMPuJQKWmVA1rB5a3NX2Eme5fPqNb63qwaqiqSp",
|
"/ipfs/QmSHDxWsMPuJQKWmVA1rB5a3NX2Eme5fPqNb63qwaqiqSp",
|
||||||
"/ipfs/QmctngrQAt9fjpQUZr7Bx3BsXUcif52eZGTizWhvcShsjz",
|
"/ipfs/QmctngrQAt9fjpQUZr7Bx3BsXUcif52eZGTizWhvcShsjz",
|
||||||
"/ipfs/QmXX7YRpU7nNBKfw75VG7Y1c3GwpSAGHRev67XVPgZFv9R"
|
"/ipfs/QmXX7YRpU7nNBKfw75VG7Y1c3GwpSAGHRev67XVPgZFv9R"
|
||||||
]
|
],
|
||||||
|
"url": "https://github.com/ipfs/refs-solarnet-storage/commit/df0a2cc4b53cea15132bec2d0b96b5d2d7a87ffb"
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,11 +1,11 @@
|
|||||||
{
|
{
|
||||||
"url": "https://sneak.berlin",
|
|
||||||
"description": "Personal blog of Jeffrey Paul",
|
|
||||||
"contact": {
|
"contact": {
|
||||||
"name": "Jeffrey Paul",
|
"email": "sneak@sneak.berlin",
|
||||||
"email": "sneak@sneak.berlin"
|
"name": "Jeffrey Paul"
|
||||||
},
|
},
|
||||||
|
"description": "Personal blog of Jeffrey Paul",
|
||||||
"pins": [
|
"pins": [
|
||||||
"/ipns/QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAugpyJJpCVp/website"
|
"/ipns/QmaN64WRYdHBojWFQRLxkdjtX6TEnqnCq8uAugpyJJpCVp/website"
|
||||||
]
|
],
|
||||||
|
"url": "https://sneak.berlin"
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,3 +0,0 @@
|
|||||||
colorlog==2.10.0
|
|
||||||
PyYAML==3.12
|
|
||||||
sanelogging==1.0.1
|
|
||||||
Loading…
Reference in New Issue
Block a user