new stuff

This commit is contained in:
Jeffrey Paul 2013-03-11 23:22:36 +01:00
parent 3638f91be4
commit a3d1ba6310
3 changed files with 87 additions and 2 deletions

View File

@ -103,9 +103,17 @@ class Website(object):
raise ReachabilityProblem("can't access: '%s'" % self.urlstring()) raise ReachabilityProblem("can't access: '%s'" % self.urlstring())
if self.is_tls(): if self.is_tls():
self._get_cert() self._get_cert()
if self.cert.expiresSoon() or not self.cert.validTime(): if not self.cert.validTime():
raise CertificateProblem( raise CertificateProblem(
"cert for %s is invalid or expires soon: %s" % ( "cert for %s is invalid: %s to %s" % (
self.urlstring(),
self.cert.notBefore(),
self.cert.notAfter()
)
)
if self.cert.expiresSoon():
raise CertificateProblem(
"cert for %s expires soon: %s" % (
self.urlstring(), self.urlstring(),
self.cert.notAfter() self.cert.notAfter()
) )

18
fetchtweets/fetchtweets Executable file
View File

@ -0,0 +1,18 @@
#!/usr/bin/env python
from pprint import pformat
# lol https://gist.github.com/re4k/3878505
APIKEY = '3rJOl1ODzm9yZy63FACdg'
APISECRET = '5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8'
import twitter
api = twitter.Api()
# consumer_key = APIKEY,
# consumer_secret = APISECRET
#)
statuses = api.GetUserTimeline('sneakatdatavibe')
print pformat([s.__dict__ for s in statuses])
#print api.VerifyCredentials()

59
scrapers/amexscraper.py Executable file
View File

@ -0,0 +1,59 @@
#!/Users/sneak/dev/venv-2.7/bin/python
from pprint import pformat
import os
import re
import json
import logging
logging.basicConfig(level=logging.ERROR)
log = logging.getLogger()
from ofxclient.request import Builder
url = 'https://online.americanexpress.com/myca/ofxdl/desktop/' + \
'desktopDownload.do?request_type=nl_ofxdownload'
# this exists because ofxclient is tightly coupled with their "Institution"
# class which shits all over my home directory with caching and
# credential storage that I don't want
class MockAmexInstitution(object):
def __init__(self,user=None,password=None):
self.username = user
self.password = password
self.dsn = {
'url': url,
'org': 'AMEX',
'fid': '3101',
}
class AmexScraper(object):
def __init__(self,*args,**kwargs):
self.user = kwargs.pop('user')
self.password = kwargs.pop('password')
def scrape(self):
i = MockAmexInstitution(
user=self.user,
password=self.password
)
b = Builder(i)
r = b.doQuery(b.acctQuery())
# i could parse the sgml. or i could do this.
c = re.compile(r'<ACCTID>(\d+)', re.MULTILINE)
out = {}
for acctnum in re.findall(c,r):
out[acctnum] = {}
c = re.compile(r'<BALAMT>([\d\.\-]+)', re.MULTILINE)
for acctnum in out.keys():
r = b.doQuery(b.ccQuery(acctnum,'19700101000000'))
out[acctnum]['balance'] = re.findall(c,r)[0]
return out
def main():
s = AmexScraper(
user=os.environ['AMEXUSERNAME'],
password=os.environ['AMEXPASSWORD']
)
print json.dumps(s.scrape())
if __name__=="__main__":
main()