diff --git a/badgerdb.go b/badgerdb.go deleted file mode 100644 index ae7f011..0000000 --- a/badgerdb.go +++ /dev/null @@ -1,62 +0,0 @@ -package main - -import "log" -import "io/ioutil" -import "github.com/dgraph-io/badger" - -type KeyValueStore struct { - db *badger.DB -} - -func NewKeyValueStore() *KeyValueStore { - kv := new(KeyValueStore) - kv.init() - return kv -} - -func (kv *KeyValueStore) init() { - dir, err := ioutil.TempDir("", "badger") - if err != nil { - log.Fatal(err) - } - - opts := badger.DefaultOptions - opts.Dir = dir - opts.ValueDir = dir - kv.db, err = badger.Open(opts) - if err != nil { - log.Fatal(err) - } -} - -func (kv *KeyValueStore) Close() { - kv.db.Close() -} - -func (kv *KeyValueStore) Put(key *string, value *string) error { - txn := kv.db.NewTransaction(true) // Read-write txn - err := txn.Set([]byte(*key), []byte(*value)) - if err != nil { - log.Fatal(err) - } - err = txn.Commit(nil) - if err != nil { - log.Fatal(err) - } - return nil -} - -func (kv *KeyValueStore) Get(key *string) (*string, error) { - txn := kv.db.NewTransaction(true) // Read-write txn - item, err := txn.Get([]byte(*key)) - if err != nil { - return nil, err - } - - val, err := item.ValueCopy(nil) - if err != nil { - return nil, err - } - s := string(val) - return &s, nil -} diff --git a/db.go b/db.go new file mode 100644 index 0000000..16880aa --- /dev/null +++ b/db.go @@ -0,0 +1,91 @@ +package main + +import "log" +import "io/ioutil" +import "github.com/dgraph-io/badger" + +// SteemDataStore is the object with which the rest of this tool interacts +type SteemDataStore struct { + kv KeyValueStorer +} + +func NewSteemDataStore(dir string) *SteemDataStore { + self := new(SteemDataStore) + self.kv = NewBadgerKeyValueStore(dir) + return self +} + +func (self *SteemDataStore) StoreBlockOps(blockNum int, blockOps []byte) { + panic("not implemented") +} + +// KeyValueStorer is an interface for the backend kv store used by +// SteemDataStore +// it could be fs, badgerdb, whatever + +type KeyValueStorer interface { + Open(string) + Get(*string) (*string, error) + Put(*string, *string) error + Close() +} + +// BadgerKeyValueStore is an object that conforms to KeyValueStorer for use +// by SteemDataStore to persist Steem data + +type BadgerKeyValueStore struct { + db *badger.DB +} + +func NewBadgerKeyValueStore(dir string) *BadgerKeyValueStore { + kv := new(BadgerKeyValueStore) + kv.Open(dir) + return kv +} + +func (kv *BadgerKeyValueStore) Open(dir string) { + dir, err := ioutil.TempDir("", "badger") + if err != nil { + log.Fatal(err) + } + + opts := badger.DefaultOptions + opts.Dir = dir + opts.ValueDir = dir + kv.db, err = badger.Open(opts) + if err != nil { + log.Fatal(err) + } +} + +func (kv *BadgerKeyValueStore) Close() { + kv.db.Close() +} + +func (kv *BadgerKeyValueStore) Put(key *string, value *string) error { + txn := kv.db.NewTransaction(true) // Read-write txn + err := txn.Set([]byte(*key), []byte(*value)) + if err != nil { + log.Fatal(err) + } + err = txn.Commit(nil) + if err != nil { + log.Fatal(err) + } + return nil +} + +func (kv *BadgerKeyValueStore) Get(key *string) (*string, error) { + txn := kv.db.NewTransaction(true) // Read-write txn + item, err := txn.Get([]byte(*key)) + if err != nil { + return nil, err + } + + val, err := item.ValueCopy(nil) + if err != nil { + return nil, err + } + s := string(val) + return &s, nil +} diff --git a/main.go b/main.go index 52b99bb..cca809b 100644 --- a/main.go +++ b/main.go @@ -1,21 +1,90 @@ package main import ( - "github.com/davecgh/go-spew/spew" + "encoding/json" + "fmt" log "github.com/sirupsen/logrus" + "github.com/spf13/afero" ) const steemAPIURL = "https://api.steemit.com" func main() { + log.SetLevel(log.DebugLevel) - s := NewSteemAPI(steemAPIURL) + var fs = afero.NewBasePathFs(afero.NewOsFs(), "./d") + var s = NewSteemAPI(steemAPIURL) + var state = NewSteemDataStore("./d") - r, err := s.GetOpsInBlock(20000000) + var endBlock = *currentBlockHeight(s) + + var startBlock = 1 + + fetchBlockRange(s, state, startBlock, endBlock) +} + +func fetchBlockRange(s *SteemAPI, fs afero.Fs, startBlock int, endBlock int) *error { + log.Debugf("fetching block range %d to %d inclusive", startBlock, endBlock) + for i := startBlock; i <= endBlock; i++ { + fetchSingleBlock(s, fs, i) + } + return nil +} + +func fetchSingleBlock(s *SteemAPI, fs afero.Fs, blockNum int) { + tmpName := fmt.Sprintf("/blockOps/%d.json.tmp", blockNum) + realName := fmt.Sprintf("/blockOps/%d.json", blockNum) + + done, err := afero.Exists(fs, realName) if err != nil { - log.Fatal(err) + panic(err) } - spew.Dump(r) + if done { + return + } + + r, err := s.GetOpsInBlock(blockNum) + if err != nil { + panic(err) + } + bytes, err := json.Marshal(r) + if err != nil { + panic(err) + } + err = afero.WriteFile(fs, tmpName, bytes, 0) + if err != nil { + panic(err) + } + fs.Rename(tmpName, realName) +} + +func nope() { + + //r2, err := s.GetOpsInBlock(20000000) + + //if err != nil { + // log.Fatal(err) + //} + + //spew.Dump(r) + + //bytes, err := json.Marshal(r2) + + //if err != nil { + // fmt.Println(err) + // } + // fmt.Println(string(bytes)) +} + +func currentBlockHeight(s *SteemAPI) *int { + r, err := s.GetDynamicGlobalProperties() + if err != nil { + return nil + } + if r.LastIrreversibleBlockNum > 0 { + return &r.LastIrreversibleBlockNum + } + return nil } diff --git a/steemapi.go b/steemapi.go index 571651d..82968e1 100644 --- a/steemapi.go +++ b/steemapi.go @@ -12,6 +12,9 @@ type SteemAPI struct { log *log.Logger } +var EmptyParams = []string{} +var EmptyParamsRaw, _ = json.Marshal(EmptyParams) + func NewSteemAPI(url string, options ...func(s *SteemAPI)) *SteemAPI { rpc := NewJSONRPC(url, func(x *JSONRPC) { x.Debug = true }) @@ -29,6 +32,21 @@ func NewSteemAPI(url string, options ...func(s *SteemAPI)) *SteemAPI { return self } +func (self *SteemAPI) GetDynamicGlobalProperties() (GetDynamicGlobalPropertiesResponse, error) { + + var resp DynamicGlobalProperties + + raw, err := self.rpc.Call("get_dynamic_global_properties", EmptyParamsRaw) + + if err != nil { + return nil, err + } + + json.Unmarshal(raw, &resp) + + return &resp, nil +} + func (self *SteemAPI) GetOpsInBlock(blockNum int) (GetOpsInBlockResponse, error) { // first fetch virtual ops @@ -56,5 +74,5 @@ func (self *SteemAPI) GetOpsInBlock(blockNum int) (GetOpsInBlockResponse, error) return nil, err } result = append(result, secondResult...) - return result, nil + return &result, nil } diff --git a/steemtypes.go b/steemtypes.go deleted file mode 100644 index 78d5dea..0000000 --- a/steemtypes.go +++ /dev/null @@ -1,80 +0,0 @@ -package main - -import "encoding/json" -import "github.com/joeshaw/iso8601" - -// TODO(sneak) -//func (r *SteemOpInBlock) UnmarshalJSON() ([]byte, error) { -//} - -type OperationObject struct { - TransactionID string `json:"trx_id"` - BlockNumber uint64 `json:"block"` - TransactionInBlock uint64 `json:"trx_in_block"` - OpInTx int `json:"op_in_trx"` - VirtualOperation int `json:"virtual_op"` - Timestamp iso8601.Time `json:"timestamp"` - Operation []json.RawMessage `json:"op"` -} - -/* -type operationTuple struct { - Type OpType - Data Operation -} - -func (op *operationTuple) MarshalJSON() ([]byte, error) { - return json.Marshal([]interface{}{ - op.Type, - op.Data, - }) -} - -func (op *operationTuple) UnmarshalJSON(data []byte) error { - // The operation object is [opType, opBody]. - raw := make([]*json.RawMessage, 2) - if err := json.Unmarshal(data, &raw); err != nil { - return errors.Wrapf(err, "failed to unmarshal operation object: %v", string(data)) - } - if len(raw) != 2 { - return errors.Errorf("invalid operation object: %v", string(data)) - } - - // Unmarshal the type. - var opType OpType - if err := json.Unmarshal(*raw[0], &opType); err != nil { - return errors.Wrapf(err, "failed to unmarshal Operation.Type: %v", string(*raw[0])) - } - - // Unmarshal the data. - var opData Operation - template, ok := dataObjects[opType] - if ok { - opData = reflect.New( - reflect.Indirect(reflect.ValueOf(template)).Type(), - ).Interface().(Operation) - - if err := json.Unmarshal(*raw[1], opData); err != nil { - return errors.Wrapf(err, "failed to unmarshal Operation.Data: %v", string(*raw[1])) - } - } else { - opData = &UnknownOperation{opType, raw[1]} - } - - // Update fields. - op.Type = opType - op.Data = opData - return nil -} -*/ -type GetOpsInBlockRequestParams struct { - BlockNum int - VirtualOps bool -} - -type GetOpsInBlockResponse []OperationObject - -func (r *GetOpsInBlockRequestParams) MarshalJSON() ([]byte, error) { - arr := []interface{}{r.BlockNum, r.VirtualOps} - return json.Marshal(arr) -} diff --git a/types.go b/types.go new file mode 100644 index 0000000..b9a13b6 --- /dev/null +++ b/types.go @@ -0,0 +1,59 @@ +package main + +import "encoding/json" +import "github.com/joeshaw/iso8601" + +type OperationObject struct { + BlockNumber uint64 `json:"block"` + OpInTx int `json:"op_in_trx"` + Operation []json.RawMessage `json:"op"` + Timestamp iso8601.Time `json:"timestamp"` + TransactionID string `json:"trx_id"` + TransactionInBlock uint64 `json:"trx_in_block"` + VirtualOperation int `json:"virtual_op"` +} + +type GetOpsInBlockRequestParams struct { + BlockNum int + VirtualOps bool +} + +type DynamicGlobalProperties struct { + ConfidentialSbdSupply string `json:"confidential_sbd_supply"` + ConfidentialSupply string `json:"confidential_supply"` + CurrentAslot int `json:"current_aslot"` + CurrentSbdSupply string `json:"current_sbd_supply"` + CurrentSupply string `json:"current_supply"` + CurrentWitness string `json:"current_witness"` + DelegationReturnPeriod int `json:"delegation_return_period"` + HeadBlockID string `json:"head_block_id"` + HeadBlockNumber int `json:"head_block_number"` + LastIrreversibleBlockNum int `json:"last_irreversible_block_num"` + MaximumBlockSize int `json:"maximum_block_size"` + NumPowWitnesses int `json:"num_pow_witnesses"` + ParticipationCount int `json:"participation_count"` + PendingRewardedVestingShares string `json:"pending_rewarded_vesting_shares"` + PendingRewardedVestingSteem string `json:"pending_rewarded_vesting_steem"` + RecentSlotsFilled string `json:"recent_slots_filled"` + ReverseAuctionSeconds int `json:"reverse_auction_seconds"` + SbdInterestRate int `json:"sbd_interest_rate"` + SbdPrintRate int `json:"sbd_print_rate"` + SbdStartPercent int `json:"sbd_start_percent"` + SbdStopPercent int `json:"sbd_stop_percent"` + Time string `json:"time"` + TotalPow int `json:"total_pow"` + TotalRewardFundSteem string `json:"total_reward_fund_steem"` + TotalRewardShares2 string `json:"total_reward_shares2"` + TotalVestingFundSteem string `json:"total_vesting_fund_steem"` + TotalVestingShares string `json:"total_vesting_shares"` + VirtualSupply string `json:"virtual_supply"` + VotePowerReserveRate int `json:"vote_power_reserve_rate"` +} + +type GetOpsInBlockResponse *[]OperationObject +type GetDynamicGlobalPropertiesResponse *DynamicGlobalProperties + +func (r *GetOpsInBlockRequestParams) MarshalJSON() ([]byte, error) { + arr := []interface{}{r.BlockNum, r.VirtualOps} + return json.Marshal(arr) +}