sneak/steem-block-db
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

tweaks

Browse Source
This commit is contained in:
Jeffrey Paul 2018-10-31 04:05:25 -07:00
parent 80033e3f33
commit df7f53e7d6
Signed by: sneak
GPG Key ID: 052443F4DF2A55C2
2 changed files with 26 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
db.go
View File

@ -6,7 +6,8 @@ import log "github.com/sirupsen/logrus"
import "fmt"
import "strconv"
const appPrefix = "steem-block-fetcher"
//steem block fetcher
const appPrefix = "sbf"
type SteemDataStorer interface {
SetCurrentBlockHeight() error
@ -28,7 +29,7 @@ func NewSteemDataStore(dir string) *SteemDataStore {
}
func (self *SteemDataStore) ForceSetCurrentBlockHeight(blockNum BlockNumber) error {
keyname := fmt.Sprintf("%s:global:CurrentBlockHeight", appPrefix)
keyname := fmt.Sprintf("%s.global.CurrentBlockHeight", appPrefix)
value := fmt.Sprintf("%d", blockNum)
return self.kv.Put(&keyname, &value)
}
@ -37,7 +38,7 @@ func (self *SteemDataStore) ForceSetCurrentBlockHeight(blockNum BlockNumber) err
// and updates the memo in the db
func (self *SteemDataStore) SetCurrentBlockHeight() error {
nextVal := self.FindHighestContiguousBlockInDb(self.CurrentBlockHeight())
keyname := fmt.Sprintf("%s:global:CurrentBlockHeight", appPrefix)
keyname := fmt.Sprintf("%s.global.CurrentBlockHeight", appPrefix)
value := fmt.Sprintf("%d", nextVal)
log.Infof("updating our current highest block in db to %d", nextVal)
return self.kv.Put(&keyname, &value)
@ -51,7 +52,7 @@ func (self *SteemDataStore) FindHighestContiguousBlockInDb(from BlockNumber) Blo
for {
try = BlockNumber(uint64(last) + 1)
keyname = fmt.Sprintf("%s:BlockOps:%d", appPrefix, try)
keyname = fmt.Sprintf("%s.ops_in_block.%d", appPrefix, try)
exists, _ := self.kv.Exists(&keyname)
if exists == false {
log.Debugf("cannot find block %d in db, highest found is %d", try, last)
@ -63,17 +64,19 @@ func (self *SteemDataStore) FindHighestContiguousBlockInDb(from BlockNumber) Blo
}
func (self *SteemDataStore) StoreBlockOps(blockNum BlockNumber, blockOps *[]byte) error {
keyname := fmt.Sprintf("%s:BlockOps:%d", appPrefix, blockNum)
keyname := fmt.Sprintf("%s.ops_in_block.%d", appPrefix, blockNum)
value := string(*blockOps)
return self.kv.Put(&keyname, &value)
}
func (self *SteemDataStore) HaveOpsForBlock(blockNum BlockNumber) bool {
panic("unimplemented")
keyname := fmt.Sprintf("%s.ops_in_block.%d", appPrefix, blockNum)
exists, _ := self.kv.Exists(&keyname)
return exists
}
func (self *SteemDataStore) CurrentBlockHeight() BlockNumber {
keyname := fmt.Sprintf("%s:global:CurrentBlockHeight", appPrefix)
keyname := fmt.Sprintf("%s.global.CurrentBlockHeight", appPrefix)
val, err := self.kv.Get(&keyname)
if err != nil {
// assume this is key not found, initialize key to default

23
main.go
View File

@ -45,7 +45,7 @@ func NewApp(config *appconfig) *App {
func (self *App) init(config *appconfig) {
self.api = NewSteemAPI(steemAPIURL)
self.datastore = NewSteemDataStore("./d")
self.desiredFetchingThreads = 20
self.desiredFetchingThreads = 40
self.currentFetchingThreads = 0
self.lock = &sync.Mutex{}
}
@ -84,12 +84,16 @@ func (self *App) decrFetchers() {
}
func (self *App) spawnNewFetcher(blockNum BlockNumber) {
log.Infof("spawning fetcher for block %d", blockNum)
log.Debugf("spawning fetcher for block %d", blockNum)
go func() {
// this is so hacky, make a queue like a grownup would you
time.Sleep(100 * time.Millisecond)
if self.datastore.HaveOpsForBlock(blockNum) {
log.Infof("already have ops for block %d, not re-fetching", blockNum)
return
}
self.incrFetchers()
self.storeBlockOps(blockNum, self.fetchBlockOps(blockNum))
self.datastore.SetCurrentBlockHeight()
time.Sleep(250 * time.Millisecond)
self.decrFetchers()
}()
@ -113,10 +117,11 @@ func (self *App) mainloop() {
log.Infof("using %d fetching threads", self.desiredFetchingThreads)
for {
self.updateCurrentBlockHeight()
log.Infof("current number of active fetchers: %d", self.numFetchers())
time.Sleep(1500 * time.Millisecond)
self.datastore.SetCurrentBlockHeight()
localHeight := self.datastore.CurrentBlockHeight()
log.Infof("our highest fetched block height is %d", localHeight)
log.Infof("current number of active fetchers: %d", self.numFetchers())
time.Sleep(1 * time.Second)
if localHeight < self.currentNetworkBlockHeight {
// we need to fetch some blocks from the network
avail := self.desiredFetchingThreads - self.numFetchers()
@ -148,12 +153,16 @@ func (self *App) fetchCurrentBlockHeight() BlockNumber {
func (self *App) fetchBlockOps(blockNum BlockNumber) *[]byte {
r, err := self.api.GetOpsInBlock(blockNum)
if err != nil {
panic(err)
// just retry on error
// sloppy, but works
return self.fetchBlockOps(blockNum)
}
bytes, err := json.Marshal(r)
if err != nil {
panic(err)
}
count := len(*r)
log.Infof("got %d operations for block %d", count, blockNum)
return &bytes
//self.datastore.writeBlockOps(blockNum, bytes)
}