moving to db

badgerdb.go

@@ -1,62 +0,0 @@
-package main
-
-import "log"
-import "io/ioutil"
-import "github.com/dgraph-io/badger"
-
-type KeyValueStore struct {
-	db *badger.DB
-}
-
-func NewKeyValueStore() *KeyValueStore {
-	kv := new(KeyValueStore)
-	kv.init()
-	return kv
-}
-
-func (kv *KeyValueStore) init() {
-	dir, err := ioutil.TempDir("", "badger")
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	opts := badger.DefaultOptions
-	opts.Dir = dir
-	opts.ValueDir = dir
-	kv.db, err = badger.Open(opts)
-	if err != nil {
-		log.Fatal(err)
-	}
-}
-
-func (kv *KeyValueStore) Close() {
-	kv.db.Close()
-}
-
-func (kv *KeyValueStore) Put(key *string, value *string) error {
-	txn := kv.db.NewTransaction(true) // Read-write txn
-	err := txn.Set([]byte(*key), []byte(*value))
-	if err != nil {
-		log.Fatal(err)
-	}
-	err = txn.Commit(nil)
-	if err != nil {
-		log.Fatal(err)
-	}
-	return nil
-}
-
-func (kv *KeyValueStore) Get(key *string) (*string, error) {
-	txn := kv.db.NewTransaction(true) // Read-write txn
-	item, err := txn.Get([]byte(*key))
-	if err != nil {
-		return nil, err
-	}
-
-	val, err := item.ValueCopy(nil)
-	if err != nil {
-		return nil, err
-	}
-	s := string(val)
-	return &s, nil
-}

db.go

@@ -0,0 +1,91 @@
+package main
+
+import "log"
+import "io/ioutil"
+import "github.com/dgraph-io/badger"
+
+// SteemDataStore is the object with which the rest of this tool interacts
+type SteemDataStore struct {
+	kv KeyValueStorer
+}
+
+func NewSteemDataStore(dir string) *SteemDataStore {
+	self := new(SteemDataStore)
+	self.kv = NewBadgerKeyValueStore(dir)
+	return self
+}
+
+func (self *SteemDataStore) StoreBlockOps(blockNum int, blockOps []byte) {
+	panic("not implemented")
+}
+
+// KeyValueStorer is an interface for the backend kv store used by
+// SteemDataStore
+// it could be fs, badgerdb, whatever
+
+type KeyValueStorer interface {
+	Open(string)
+	Get(*string) (*string, error)
+	Put(*string, *string) error
+	Close()
+}
+
+// BadgerKeyValueStore is an object that conforms to KeyValueStorer for use
+// by SteemDataStore to persist Steem data
+
+type BadgerKeyValueStore struct {
+	db *badger.DB
+}
+
+func NewBadgerKeyValueStore(dir string) *BadgerKeyValueStore {
+	kv := new(BadgerKeyValueStore)
+	kv.Open(dir)
+	return kv
+}
+
+func (kv *BadgerKeyValueStore) Open(dir string) {
+	dir, err := ioutil.TempDir("", "badger")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	opts := badger.DefaultOptions
+	opts.Dir = dir
+	opts.ValueDir = dir
+	kv.db, err = badger.Open(opts)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func (kv *BadgerKeyValueStore) Close() {
+	kv.db.Close()
+}
+
+func (kv *BadgerKeyValueStore) Put(key *string, value *string) error {
+	txn := kv.db.NewTransaction(true) // Read-write txn
+	err := txn.Set([]byte(*key), []byte(*value))
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = txn.Commit(nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return nil
+}
+
+func (kv *BadgerKeyValueStore) Get(key *string) (*string, error) {
+	txn := kv.db.NewTransaction(true) // Read-write txn
+	item, err := txn.Get([]byte(*key))
+	if err != nil {
+		return nil, err
+	}
+
+	val, err := item.ValueCopy(nil)
+	if err != nil {
+		return nil, err
+	}
+	s := string(val)
+	return &s, nil
+}

main.go

@@ -1,21 +1,90 @@
 package main
 
 import (
-	"github.com/davecgh/go-spew/spew"
+	"encoding/json"
+	"fmt"
 	log "github.com/sirupsen/logrus"
+	"github.com/spf13/afero"
 )
 
 const steemAPIURL = "https://api.steemit.com"
 
 func main() {
+	log.SetLevel(log.DebugLevel)
 
-	s := NewSteemAPI(steemAPIURL)
+	var fs = afero.NewBasePathFs(afero.NewOsFs(), "./d")
+	var s = NewSteemAPI(steemAPIURL)
+	var state = NewSteemDataStore("./d")
 
-	r, err := s.GetOpsInBlock(20000000)
+	var endBlock = *currentBlockHeight(s)
+
+	var startBlock = 1
+
+	fetchBlockRange(s, state, startBlock, endBlock)
+}
+
+func fetchBlockRange(s *SteemAPI, fs afero.Fs, startBlock int, endBlock int) *error {
+	log.Debugf("fetching block range %d to %d inclusive", startBlock, endBlock)
+	for i := startBlock; i <= endBlock; i++ {
+		fetchSingleBlock(s, fs, i)
+	}
+	return nil
+}
+
+func fetchSingleBlock(s *SteemAPI, fs afero.Fs, blockNum int) {
+	tmpName := fmt.Sprintf("/blockOps/%d.json.tmp", blockNum)
+	realName := fmt.Sprintf("/blockOps/%d.json", blockNum)
+
+	done, err := afero.Exists(fs, realName)
 
 	if err != nil {
-		log.Fatal(err)
+		panic(err)
 	}
 
-	spew.Dump(r)
+	if done {
+		return
+	}
+
+	r, err := s.GetOpsInBlock(blockNum)
+	if err != nil {
+		panic(err)
+	}
+	bytes, err := json.Marshal(r)
+	if err != nil {
+		panic(err)
+	}
+	err = afero.WriteFile(fs, tmpName, bytes, 0)
+	if err != nil {
+		panic(err)
+	}
+	fs.Rename(tmpName, realName)
+}
+
+func nope() {
+
+	//r2, err := s.GetOpsInBlock(20000000)
+
+	//if err != nil {
+	//	log.Fatal(err)
+	//}
+
+	//spew.Dump(r)
+
+	//bytes, err := json.Marshal(r2)
+
+	//if err != nil {
+	//		fmt.Println(err)
+	//	}
+	//	fmt.Println(string(bytes))
+}
+
+func currentBlockHeight(s *SteemAPI) *int {
+	r, err := s.GetDynamicGlobalProperties()
+	if err != nil {
+		return nil
+	}
+	if r.LastIrreversibleBlockNum > 0 {
+		return &r.LastIrreversibleBlockNum
+	}
+	return nil
 }

steemapi.go

@@ -12,6 +12,9 @@ type SteemAPI struct {
 	log   *log.Logger
 }
 
+var EmptyParams = []string{}
+var EmptyParamsRaw, _ = json.Marshal(EmptyParams)
+
 func NewSteemAPI(url string, options ...func(s *SteemAPI)) *SteemAPI {
 
 	rpc := NewJSONRPC(url, func(x *JSONRPC) { x.Debug = true })
@@ -29,6 +32,21 @@ func NewSteemAPI(url string, options ...func(s *SteemAPI)) *SteemAPI {
 	return self
 }
 
+func (self *SteemAPI) GetDynamicGlobalProperties() (GetDynamicGlobalPropertiesResponse, error) {
+
+	var resp DynamicGlobalProperties
+
+	raw, err := self.rpc.Call("get_dynamic_global_properties", EmptyParamsRaw)
+
+	if err != nil {
+		return nil, err
+	}
+
+	json.Unmarshal(raw, &resp)
+
+	return &resp, nil
+}
+
 func (self *SteemAPI) GetOpsInBlock(blockNum int) (GetOpsInBlockResponse, error) {
 
 	// first fetch virtual ops
@@ -56,5 +74,5 @@ func (self *SteemAPI) GetOpsInBlock(blockNum int) (GetOpsInBlockResponse, error)
 		return nil, err
 	}
 	result = append(result, secondResult...)
-	return result, nil
+	return &result, nil
 }

steemtypes.go

@@ -1,80 +0,0 @@
-package main
-
-import "encoding/json"
-import "github.com/joeshaw/iso8601"
-
-// TODO(sneak)
-//func (r *SteemOpInBlock) UnmarshalJSON() ([]byte, error) {
-//}
-
-type OperationObject struct {
-	TransactionID      string            `json:"trx_id"`
-	BlockNumber        uint64            `json:"block"`
-	TransactionInBlock uint64            `json:"trx_in_block"`
-	OpInTx             int               `json:"op_in_trx"`
-	VirtualOperation   int               `json:"virtual_op"`
-	Timestamp          iso8601.Time      `json:"timestamp"`
-	Operation          []json.RawMessage `json:"op"`
-}
-
-/*
-type operationTuple struct {
-	Type OpType
-	Data Operation
-}
-
-func (op *operationTuple) MarshalJSON() ([]byte, error) {
-	return json.Marshal([]interface{}{
-		op.Type,
-		op.Data,
-	})
-}
-
-func (op *operationTuple) UnmarshalJSON(data []byte) error {
-	// The operation object is [opType, opBody].
-	raw := make([]*json.RawMessage, 2)
-	if err := json.Unmarshal(data, &raw); err != nil {
-		return errors.Wrapf(err, "failed to unmarshal operation object: %v", string(data))
-	}
-	if len(raw) != 2 {
-		return errors.Errorf("invalid operation object: %v", string(data))
-	}
-
-	// Unmarshal the type.
-	var opType OpType
-	if err := json.Unmarshal(*raw[0], &opType); err != nil {
-		return errors.Wrapf(err, "failed to unmarshal Operation.Type: %v", string(*raw[0]))
-	}
-
-	// Unmarshal the data.
-	var opData Operation
-	template, ok := dataObjects[opType]
-	if ok {
-		opData = reflect.New(
-			reflect.Indirect(reflect.ValueOf(template)).Type(),
-		).Interface().(Operation)
-
-		if err := json.Unmarshal(*raw[1], opData); err != nil {
-			return errors.Wrapf(err, "failed to unmarshal Operation.Data: %v", string(*raw[1]))
-		}
-	} else {
-		opData = &UnknownOperation{opType, raw[1]}
-	}
-
-	// Update fields.
-	op.Type = opType
-	op.Data = opData
-	return nil
-}
-*/
-type GetOpsInBlockRequestParams struct {
-	BlockNum   int
-	VirtualOps bool
-}
-
-type GetOpsInBlockResponse []OperationObject
-
-func (r *GetOpsInBlockRequestParams) MarshalJSON() ([]byte, error) {
-	arr := []interface{}{r.BlockNum, r.VirtualOps}
-	return json.Marshal(arr)
-}

types.go

@@ -0,0 +1,59 @@
+package main
+
+import "encoding/json"
+import "github.com/joeshaw/iso8601"
+
+type OperationObject struct {
+	BlockNumber        uint64            `json:"block"`
+	OpInTx             int               `json:"op_in_trx"`
+	Operation          []json.RawMessage `json:"op"`
+	Timestamp          iso8601.Time      `json:"timestamp"`
+	TransactionID      string            `json:"trx_id"`
+	TransactionInBlock uint64            `json:"trx_in_block"`
+	VirtualOperation   int               `json:"virtual_op"`
+}
+
+type GetOpsInBlockRequestParams struct {
+	BlockNum   int
+	VirtualOps bool
+}
+
+type DynamicGlobalProperties struct {
+	ConfidentialSbdSupply        string `json:"confidential_sbd_supply"`
+	ConfidentialSupply           string `json:"confidential_supply"`
+	CurrentAslot                 int    `json:"current_aslot"`
+	CurrentSbdSupply             string `json:"current_sbd_supply"`
+	CurrentSupply                string `json:"current_supply"`
+	CurrentWitness               string `json:"current_witness"`
+	DelegationReturnPeriod       int    `json:"delegation_return_period"`
+	HeadBlockID                  string `json:"head_block_id"`
+	HeadBlockNumber              int    `json:"head_block_number"`
+	LastIrreversibleBlockNum     int    `json:"last_irreversible_block_num"`
+	MaximumBlockSize             int    `json:"maximum_block_size"`
+	NumPowWitnesses              int    `json:"num_pow_witnesses"`
+	ParticipationCount           int    `json:"participation_count"`
+	PendingRewardedVestingShares string `json:"pending_rewarded_vesting_shares"`
+	PendingRewardedVestingSteem  string `json:"pending_rewarded_vesting_steem"`
+	RecentSlotsFilled            string `json:"recent_slots_filled"`
+	ReverseAuctionSeconds        int    `json:"reverse_auction_seconds"`
+	SbdInterestRate              int    `json:"sbd_interest_rate"`
+	SbdPrintRate                 int    `json:"sbd_print_rate"`
+	SbdStartPercent              int    `json:"sbd_start_percent"`
+	SbdStopPercent               int    `json:"sbd_stop_percent"`
+	Time                         string `json:"time"`
+	TotalPow                     int    `json:"total_pow"`
+	TotalRewardFundSteem         string `json:"total_reward_fund_steem"`
+	TotalRewardShares2           string `json:"total_reward_shares2"`
+	TotalVestingFundSteem        string `json:"total_vesting_fund_steem"`
+	TotalVestingShares           string `json:"total_vesting_shares"`
+	VirtualSupply                string `json:"virtual_supply"`
+	VotePowerReserveRate         int    `json:"vote_power_reserve_rate"`
+}
+
+type GetOpsInBlockResponse *[]OperationObject
+type GetDynamicGlobalPropertiesResponse *DynamicGlobalProperties
+
+func (r *GetOpsInBlockRequestParams) MarshalJSON() ([]byte, error) {
+	arr := []interface{}{r.BlockNum, r.VirtualOps}
+	return json.Marshal(arr)
+}