steem-block-db/app.go

package main

type FetchedBlock struct {
	blockNumber BlockNumber
	data        []byte
}

type App struct {
	datastore                 SteemDataStorer
	api                       *SteemAPI
	currentNetworkBlockHeight BlockNumber
	currentLocalBlockHeight   BlockNumber
	desiredFetcherThreads     uint
	wantBlocks                []BlockNumber
	fetchingBlocks            []BlockNumber
	fetchedBlocks             *[]FetchedBlock
	lock                      *sync.Mutex
}

type appconfig struct {
	logLevel log.Level
	apiUrl   string
	redisUrl string
}

func NewApp(config *appconfig) *App {
	self := new(App)
	self.init(config)
	return self
}

func (self *App) init(config *appconfig) {
	log.SetLevel(config.logLevel)
	self.api = NewSteemAPI(config.apiUrl)
	self.datastore = NewSteemDataStore(config.redisUrl)
	self.desiredFetcherThreads = config.fetcherThreads
	self.lock = &sync.Mutex{}
}

func (self *App) updateCurrentBlockHeight() {
	h := self.fetchCurrentBlockHeight()
	if h > self.currentNetworkBlockHeight {
		self.lock.Lock()
		defer self.lock.Unlock()
		self.currentNetworkBlockHeight = h
		log.Infof("current block height is now %d", self.currentNetworkBlockHeight)
	}
}

func (self *App) main() {
	log.Infof("steem block data fetcher starting up...")
	self.mainloop()
}

func (self *App) numFetchers() uint {
	self.lock.Lock()
	defer self.lock.Unlock()
	return uint(len(*self.fetchingBlocks))
}

func (self *App) spawnNewFetcher(blockNum BlockNumber) {
	log.Debugf("spawning fetcher for block %d", blockNum)
	go func() {
		// this is so hacky, make a queue like a grownup would you
		time.Sleep(100 * time.Millisecond)
		if self.datastore.HaveOpsForBlock(blockNum) {
			log.Infof("already have ops for block %d, not re-fetching", blockNum)
			return
		}
		//self.incrFetchers()
		self.storeBlockOps(blockNum, self.fetchBlockOps(blockNum))
		//self.decrFetchers()

	}()
}

func (self *App) storeBlockOps(blockNum BlockNumber, blockOps *[]byte) {
	self.datastore.StoreBlockOps(blockNum, blockOps)
}

// note that parallelFetchAndStoreBlocks does not respect the
// limitation on number of desired fetchers, that is for the caller
func (self *App) parallelFetchAndStoreBlocks(start BlockNumber, end BlockNumber) {
	var diff = uint64(end) - uint64(start)
	var i uint64
	for i = 0; i < diff; i++ {
		self.spawnNewFetcher(BlockNumber(uint64(start) + i))
	}
}

func (self *App) populateFetchers() {
}

func (self *App) mainloop() {

	log.Infof("using %d fetching threads", self.desiredFetchingThreads)

	self.currentNetworkBlockHeight = self.fetchCurrentBlockHeight()

	for {
		self.spawnMoreFetchers()
		self.currentNetworkBlockHeight = self.fetchCurrentBlockHeight()
		time.Sleep(1000 * time.Millisecond)
	}
}

func (self *App) spawnMoreFetchers() {
}

/*
	self.updateCurrentBlockHeight()
	log.Infof("current number of active fetchers: %d", self.numFetchers())
	time.Sleep(1500 * time.Millisecond)
	self.datastore.SetCurrentBlockHeight()
	localHeight := self.datastore.CurrentBlockHeight()
	log.Infof("our highest fetched block height is %d", localHeight)
	if localHeight < self.currentNetworkBlockHeight {
		// we need to fetch some blocks from the network
		avail := self.desiredFetchingThreads - self.numFetchers()
		diff := uint64(self.currentNetworkBlockHeight) - uint64(localHeight)
		log.Infof("we need to fetch %d blocks", diff)
		if uint64(diff) > uint64(avail) {
			self.parallelFetchAndStoreBlocks(BlockNumber(uint64(localHeight)+1), BlockNumber(uint64(localHeight)+1+uint64(avail)))
		} else {
			// just spawn fetchers for the blocks we don't have
			// spawning will update the number of running fetchers
			self.parallelFetchAndStoreBlocks(localHeight+1, self.currentNetworkBlockHeight)
		}
	}
	//needFetchers := self.desiredFetchingThreads - self.numFetchers()

*/

func (self *App) fetchCurrentBlockHeight() BlockNumber {
	r, err := self.api.GetDynamicGlobalProperties()
	if err != nil {
		log.Panicf("can't fetch global properties, bailing. err: %s", err)
	}
	if r.LastIrreversibleBlockNum < 100 {
		log.Panicf("can't fetch global properties, bailing")
	}
	return r.LastIrreversibleBlockNum
}

func (self *App) fetchBlockOps(blockNum BlockNumber) *[]byte {
	r, err := self.api.GetOpsInBlock(blockNum)
	if err != nil {
		// just retry on error
		// sloppy, but works
		return self.fetchBlockOps(blockNum)
	}
	bytes, err := json.Marshal(r)
	if err != nil {
		panic(err)
	}
	count := len(*r)
	log.Infof("got %d operations for block %d", count, blockNum)
	return &bytes
	//self.datastore.writeBlockOps(blockNum, bytes)
}