167 lines
4.8 KiB
Python
Executable File
167 lines
4.8 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
#import ipfsapi
|
|
from sanelogging import log
|
|
from datetime import datetime
|
|
import json
|
|
import os
|
|
import os.path
|
|
import re
|
|
import subprocess
|
|
import yaml
|
|
|
|
ipfsPathPattern = re.compile('\/ipfs\/([a-zA-Z0-9]+)')
|
|
|
|
#def connectIpfs(hostname='127.0.0.1',port=5001):
|
|
# ipfs = ipfsapi.connect(hostname,port)
|
|
# log.info("connected to ipfsd: %s" % ipfs.id()['ProtocolVersion'])
|
|
# return ipfs
|
|
|
|
def main():
|
|
startTime = datetime.utcnow()
|
|
log.info("generating pin list.")
|
|
generatePinList()
|
|
elapsed = datetime.utcnow() - startTime
|
|
log.info("finished in %is" % elapsed.total_seconds())
|
|
|
|
def generatePinList():
|
|
pinFiles = readPinFiles()
|
|
# ipfs = connectIpfs()
|
|
|
|
resources = []
|
|
|
|
for pinFile in pinFiles:
|
|
rs = extractResources(pinFile)
|
|
resources.extend(rs)
|
|
|
|
log.info("collected data from files.")
|
|
|
|
resources = resolveResources(resources)
|
|
|
|
output = {
|
|
"generatedAt": datetime.utcnow().strftime("%s"),
|
|
"pinList": resources
|
|
}
|
|
log.info("resources resolved, writing pinlist file")
|
|
outputFileName = 'pinlist.json'
|
|
fd = open(outputFileName + '.new','w')
|
|
json.dump(output,fd)
|
|
fd.close()
|
|
os.rename(outputFileName + '.new', outputFileName)
|
|
|
|
def extractResources(pinFile):
|
|
# this pulls out the list of ipns/ipfs
|
|
# items from the single file
|
|
output = []
|
|
if 'ipns' in pinFile['pins']:
|
|
if isinstance(pinFile['pins']['ipns'], basestring):
|
|
# single ipns name
|
|
output.append({
|
|
'type': 'ipns',
|
|
'name': pinFile['pins']['ipns'],
|
|
'fileName': os.path.basename(pinFile['fileName'])
|
|
})
|
|
else:
|
|
# list of ipns names
|
|
for ipnsName in pinFile['pins']['ipns']:
|
|
output.append({
|
|
'type': 'ipns',
|
|
'name': ipnsName,
|
|
'fileName': os.path.basename(pinFile['fileName'])
|
|
})
|
|
|
|
if 'ipfs' in pinFile['pins']:
|
|
if isinstance(pinFile['pins']['ipfs'], basestring):
|
|
# single ipfs name
|
|
output.append({
|
|
'type': 'ipfs',
|
|
'name': pinFile['pins']['ipfs'],
|
|
'fileName': os.path.basename(pinFile['fileName'])
|
|
})
|
|
else:
|
|
# list of ipfs names
|
|
for ipfsName in pinFile['pins']['ipfs']:
|
|
output.append({
|
|
'type': 'ipfs',
|
|
'name': ipfsName,
|
|
'fileName': os.path.basename(pinFile['fileName'])
|
|
})
|
|
return output
|
|
|
|
def resolveResources(resources):
|
|
log.info("resolving resource names for pin list")
|
|
for resource in resources:
|
|
if resource['type'] == 'ipfs':
|
|
resource['resolved'] = resource['name']
|
|
elif resource['type'] == 'ipns':
|
|
# attepmt to resolve ipns name
|
|
# FIXME try/catch
|
|
res = None
|
|
try:
|
|
res = resolveIpnsName(resource['name'])
|
|
if res:
|
|
resource['resolved'] = res
|
|
except LookupError:
|
|
log.info(
|
|
"unable to resolve ipns name '%s' from file '%s'"
|
|
% (resource['name'], resource['fileName'])
|
|
)
|
|
resource['error'] = 'unable to resolve name'
|
|
return resources
|
|
|
|
def runCommand(cmd):
|
|
p = subprocess.Popen(
|
|
cmd,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT
|
|
)
|
|
retval = p.wait()
|
|
if retval != 0:
|
|
raise LookupError()
|
|
return p.stdout.read()
|
|
|
|
def resolveIpnsName(ipnsName):
|
|
log.debug("resolving name '%s'" % ipnsName)
|
|
res = runCommand(['ipfs', 'name', 'resolve', ipnsName])
|
|
ipfsNameMatch = ipfsPathPattern.match(res)
|
|
if not ipfsNameMatch:
|
|
log.info("unable to resolve ipns name '%s'" % ipnsName)
|
|
return None
|
|
ipfsName = ipfsNameMatch.group(1)
|
|
log.info("resolved ipns name '%s' to '%s'" % (ipnsName, ipfsName))
|
|
return ipfsName
|
|
|
|
def readPinFiles():
|
|
log.info("reading pinfiles...")
|
|
output = []
|
|
pinFiles = getPinFileNames()
|
|
for pinFileName in pinFiles:
|
|
pinFileData = parsePinFile(pinFileName)
|
|
pinFileData['fileName'] = pinFileName
|
|
output.append(pinFileData)
|
|
return output
|
|
|
|
def parsePinFile(fn):
|
|
x = {}
|
|
f = open(fn,"r")
|
|
x = yaml.load(f)
|
|
f.close()
|
|
validatePinFileData(x)
|
|
return x
|
|
|
|
def validatePinFileData(pinFileStruct):
|
|
# FIXME validate various file properties here
|
|
pass
|
|
|
|
def getPinFileNames():
|
|
thisDir = os.path.dirname(os.path.abspath(__file__))
|
|
pinDir = os.path.join(thisDir, 'pins')
|
|
pinFileNames = [
|
|
os.path.join(pinDir,f) for f in os.listdir(pinDir)
|
|
if os.path.isfile(os.path.join(pinDir, f))
|
|
]
|
|
return pinFileNames
|
|
|
|
if __name__ == "__main__":
|
|
main()
|