moving to db
This commit is contained in:
parent
aba19d1e89
commit
f91f8e1927
GPG Key ID:
052443F4DF2A55C2
62
badgerdb.go
62
badgerdb.go
|
|
@ -1,62 +0,0 @@
|
|||
package main
|
||||
|
||||
import "log"
|
||||
import "io/ioutil"
|
||||
import "github.com/dgraph-io/badger"
|
||||
|
||||
type KeyValueStore struct {
|
||||
db *badger.DB
|
||||
}
|
||||
|
||||
func NewKeyValueStore() *KeyValueStore {
|
||||
kv := new(KeyValueStore)
|
||||
kv.init()
|
||||
return kv
|
||||
}
|
||||
|
||||
func (kv *KeyValueStore) init() {
|
||||
dir, err := ioutil.TempDir("", "badger")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
opts := badger.DefaultOptions
|
||||
opts.Dir = dir
|
||||
opts.ValueDir = dir
|
||||
kv.db, err = badger.Open(opts)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (kv *KeyValueStore) Close() {
|
||||
kv.db.Close()
|
||||
}
|
||||
|
||||
func (kv *KeyValueStore) Put(key *string, value *string) error {
|
||||
txn := kv.db.NewTransaction(true) // Read-write txn
|
||||
err := txn.Set([]byte(*key), []byte(*value))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
err = txn.Commit(nil)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (kv *KeyValueStore) Get(key *string) (*string, error) {
|
||||
txn := kv.db.NewTransaction(true) // Read-write txn
|
||||
item, err := txn.Get([]byte(*key))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
val, err := item.ValueCopy(nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s := string(val)
|
||||
return &s, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
package main
|
||||
|
||||
import "log"
|
||||
import "io/ioutil"
|
||||
import "github.com/dgraph-io/badger"
|
||||
|
||||
// SteemDataStore is the object with which the rest of this tool interacts
|
||||
type SteemDataStore struct {
|
||||
kv KeyValueStorer
|
||||
}
|
||||
|
||||
func NewSteemDataStore(dir string) *SteemDataStore {
|
||||
self := new(SteemDataStore)
|
||||
self.kv = NewBadgerKeyValueStore(dir)
|
||||
return self
|
||||
}
|
||||
|
||||
func (self *SteemDataStore) StoreBlockOps(blockNum int, blockOps []byte) {
|
||||
panic("not implemented")
|
||||
}
|
||||
|
||||
// KeyValueStorer is an interface for the backend kv store used by
|
||||
// SteemDataStore
|
||||
// it could be fs, badgerdb, whatever
|
||||
|
||||
type KeyValueStorer interface {
|
||||
Open(string)
|
||||
Get(*string) (*string, error)
|
||||
Put(*string, *string) error
|
||||
Close()
|
||||
}
|
||||
|
||||
// BadgerKeyValueStore is an object that conforms to KeyValueStorer for use
|
||||
// by SteemDataStore to persist Steem data
|
||||
|
||||
type BadgerKeyValueStore struct {
|
||||
db *badger.DB
|
||||
}
|
||||
|
||||
func NewBadgerKeyValueStore(dir string) *BadgerKeyValueStore {
|
||||
kv := new(BadgerKeyValueStore)
|
||||
kv.Open(dir)
|
||||
return kv
|
||||
}
|
||||
|
||||
func (kv *BadgerKeyValueStore) Open(dir string) {
|
||||
dir, err := ioutil.TempDir("", "badger")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
opts := badger.DefaultOptions
|
||||
opts.Dir = dir
|
||||
opts.ValueDir = dir
|
||||
kv.db, err = badger.Open(opts)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (kv *BadgerKeyValueStore) Close() {
|
||||
kv.db.Close()
|
||||
}
|
||||
|
||||
func (kv *BadgerKeyValueStore) Put(key *string, value *string) error {
|
||||
txn := kv.db.NewTransaction(true) // Read-write txn
|
||||
err := txn.Set([]byte(*key), []byte(*value))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
err = txn.Commit(nil)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (kv *BadgerKeyValueStore) Get(key *string) (*string, error) {
|
||||
txn := kv.db.NewTransaction(true) // Read-write txn
|
||||
item, err := txn.Get([]byte(*key))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
val, err := item.ValueCopy(nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s := string(val)
|
||||
return &s, nil
|
||||
}
|
||||
79
main.go
79
main.go
|
|
@ -1,21 +1,90 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
const steemAPIURL = "https://api.steemit.com"
|
||||
|
||||
func main() {
|
||||
log.SetLevel(log.DebugLevel)
|
||||
|
||||
s := NewSteemAPI(steemAPIURL)
|
||||
var fs = afero.NewBasePathFs(afero.NewOsFs(), "./d")
|
||||
var s = NewSteemAPI(steemAPIURL)
|
||||
var state = NewSteemDataStore("./d")
|
||||
|
||||
r, err := s.GetOpsInBlock(20000000)
|
||||
var endBlock = *currentBlockHeight(s)
|
||||
|
||||
var startBlock = 1
|
||||
|
||||
fetchBlockRange(s, state, startBlock, endBlock)
|
||||
}
|
||||
|
||||
func fetchBlockRange(s *SteemAPI, fs afero.Fs, startBlock int, endBlock int) *error {
|
||||
log.Debugf("fetching block range %d to %d inclusive", startBlock, endBlock)
|
||||
for i := startBlock; i <= endBlock; i++ {
|
||||
fetchSingleBlock(s, fs, i)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fetchSingleBlock(s *SteemAPI, fs afero.Fs, blockNum int) {
|
||||
tmpName := fmt.Sprintf("/blockOps/%d.json.tmp", blockNum)
|
||||
realName := fmt.Sprintf("/blockOps/%d.json", blockNum)
|
||||
|
||||
done, err := afero.Exists(fs, realName)
|
||||
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
panic(err)
|
||||
}
|
||||
|
||||
spew.Dump(r)
|
||||
if done {
|
||||
return
|
||||
}
|
||||
|
||||
r, err := s.GetOpsInBlock(blockNum)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
bytes, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = afero.WriteFile(fs, tmpName, bytes, 0)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fs.Rename(tmpName, realName)
|
||||
}
|
||||
|
||||
func nope() {
|
||||
|
||||
//r2, err := s.GetOpsInBlock(20000000)
|
||||
|
||||
//if err != nil {
|
||||
// log.Fatal(err)
|
||||
//}
|
||||
|
||||
//spew.Dump(r)
|
||||
|
||||
//bytes, err := json.Marshal(r2)
|
||||
|
||||
//if err != nil {
|
||||
// fmt.Println(err)
|
||||
// }
|
||||
// fmt.Println(string(bytes))
|
||||
}
|
||||
|
||||
func currentBlockHeight(s *SteemAPI) *int {
|
||||
r, err := s.GetDynamicGlobalProperties()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if r.LastIrreversibleBlockNum > 0 {
|
||||
return &r.LastIrreversibleBlockNum
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
20
steemapi.go
20
steemapi.go
|
|
@ -12,6 +12,9 @@ type SteemAPI struct {
|
|||
log *log.Logger
|
||||
}
|
||||
|
||||
var EmptyParams = []string{}
|
||||
var EmptyParamsRaw, _ = json.Marshal(EmptyParams)
|
||||
|
||||
func NewSteemAPI(url string, options ...func(s *SteemAPI)) *SteemAPI {
|
||||
|
||||
rpc := NewJSONRPC(url, func(x *JSONRPC) { x.Debug = true })
|
||||
|
|
@ -29,6 +32,21 @@ func NewSteemAPI(url string, options ...func(s *SteemAPI)) *SteemAPI {
|
|||
return self
|
||||
}
|
||||
|
||||
func (self *SteemAPI) GetDynamicGlobalProperties() (GetDynamicGlobalPropertiesResponse, error) {
|
||||
|
||||
var resp DynamicGlobalProperties
|
||||
|
||||
raw, err := self.rpc.Call("get_dynamic_global_properties", EmptyParamsRaw)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
json.Unmarshal(raw, &resp)
|
||||
|
||||
return &resp, nil
|
||||
}
|
||||
|
||||
func (self *SteemAPI) GetOpsInBlock(blockNum int) (GetOpsInBlockResponse, error) {
|
||||
|
||||
// first fetch virtual ops
|
||||
|
|
@ -56,5 +74,5 @@ func (self *SteemAPI) GetOpsInBlock(blockNum int) (GetOpsInBlockResponse, error)
|
|||
return nil, err
|
||||
}
|
||||
result = append(result, secondResult...)
|
||||
return result, nil
|
||||
return &result, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,80 +0,0 @@
|
|||
package main
|
||||
|
||||
import "encoding/json"
|
||||
import "github.com/joeshaw/iso8601"
|
||||
|
||||
// TODO(sneak)
|
||||
//func (r *SteemOpInBlock) UnmarshalJSON() ([]byte, error) {
|
||||
//}
|
||||
|
||||
type OperationObject struct {
|
||||
TransactionID string `json:"trx_id"`
|
||||
BlockNumber uint64 `json:"block"`
|
||||
TransactionInBlock uint64 `json:"trx_in_block"`
|
||||
OpInTx int `json:"op_in_trx"`
|
||||
VirtualOperation int `json:"virtual_op"`
|
||||
Timestamp iso8601.Time `json:"timestamp"`
|
||||
Operation []json.RawMessage `json:"op"`
|
||||
}
|
||||
|
||||
/*
|
||||
type operationTuple struct {
|
||||
Type OpType
|
||||
Data Operation
|
||||
}
|
||||
|
||||
func (op *operationTuple) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal([]interface{}{
|
||||
op.Type,
|
||||
op.Data,
|
||||
})
|
||||
}
|
||||
|
||||
func (op *operationTuple) UnmarshalJSON(data []byte) error {
|
||||
// The operation object is [opType, opBody].
|
||||
raw := make([]*json.RawMessage, 2)
|
||||
if err := json.Unmarshal(data, &raw); err != nil {
|
||||
return errors.Wrapf(err, "failed to unmarshal operation object: %v", string(data))
|
||||
}
|
||||
if len(raw) != 2 {
|
||||
return errors.Errorf("invalid operation object: %v", string(data))
|
||||
}
|
||||
|
||||
// Unmarshal the type.
|
||||
var opType OpType
|
||||
if err := json.Unmarshal(*raw[0], &opType); err != nil {
|
||||
return errors.Wrapf(err, "failed to unmarshal Operation.Type: %v", string(*raw[0]))
|
||||
}
|
||||
|
||||
// Unmarshal the data.
|
||||
var opData Operation
|
||||
template, ok := dataObjects[opType]
|
||||
if ok {
|
||||
opData = reflect.New(
|
||||
reflect.Indirect(reflect.ValueOf(template)).Type(),
|
||||
).Interface().(Operation)
|
||||
|
||||
if err := json.Unmarshal(*raw[1], opData); err != nil {
|
||||
return errors.Wrapf(err, "failed to unmarshal Operation.Data: %v", string(*raw[1]))
|
||||
}
|
||||
} else {
|
||||
opData = &UnknownOperation{opType, raw[1]}
|
||||
}
|
||||
|
||||
// Update fields.
|
||||
op.Type = opType
|
||||
op.Data = opData
|
||||
return nil
|
||||
}
|
||||
*/
|
||||
type GetOpsInBlockRequestParams struct {
|
||||
BlockNum int
|
||||
VirtualOps bool
|
||||
}
|
||||
|
||||
type GetOpsInBlockResponse []OperationObject
|
||||
|
||||
func (r *GetOpsInBlockRequestParams) MarshalJSON() ([]byte, error) {
|
||||
arr := []interface{}{r.BlockNum, r.VirtualOps}
|
||||
return json.Marshal(arr)
|
||||
}
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
package main
|
||||
|
||||
import "encoding/json"
|
||||
import "github.com/joeshaw/iso8601"
|
||||
|
||||
type OperationObject struct {
|
||||
BlockNumber uint64 `json:"block"`
|
||||
OpInTx int `json:"op_in_trx"`
|
||||
Operation []json.RawMessage `json:"op"`
|
||||
Timestamp iso8601.Time `json:"timestamp"`
|
||||
TransactionID string `json:"trx_id"`
|
||||
TransactionInBlock uint64 `json:"trx_in_block"`
|
||||
VirtualOperation int `json:"virtual_op"`
|
||||
}
|
||||
|
||||
type GetOpsInBlockRequestParams struct {
|
||||
BlockNum int
|
||||
VirtualOps bool
|
||||
}
|
||||
|
||||
type DynamicGlobalProperties struct {
|
||||
ConfidentialSbdSupply string `json:"confidential_sbd_supply"`
|
||||
ConfidentialSupply string `json:"confidential_supply"`
|
||||
CurrentAslot int `json:"current_aslot"`
|
||||
CurrentSbdSupply string `json:"current_sbd_supply"`
|
||||
CurrentSupply string `json:"current_supply"`
|
||||
CurrentWitness string `json:"current_witness"`
|
||||
DelegationReturnPeriod int `json:"delegation_return_period"`
|
||||
HeadBlockID string `json:"head_block_id"`
|
||||
HeadBlockNumber int `json:"head_block_number"`
|
||||
LastIrreversibleBlockNum int `json:"last_irreversible_block_num"`
|
||||
MaximumBlockSize int `json:"maximum_block_size"`
|
||||
NumPowWitnesses int `json:"num_pow_witnesses"`
|
||||
ParticipationCount int `json:"participation_count"`
|
||||
PendingRewardedVestingShares string `json:"pending_rewarded_vesting_shares"`
|
||||
PendingRewardedVestingSteem string `json:"pending_rewarded_vesting_steem"`
|
||||
RecentSlotsFilled string `json:"recent_slots_filled"`
|
||||
ReverseAuctionSeconds int `json:"reverse_auction_seconds"`
|
||||
SbdInterestRate int `json:"sbd_interest_rate"`
|
||||
SbdPrintRate int `json:"sbd_print_rate"`
|
||||
SbdStartPercent int `json:"sbd_start_percent"`
|
||||
SbdStopPercent int `json:"sbd_stop_percent"`
|
||||
Time string `json:"time"`
|
||||
TotalPow int `json:"total_pow"`
|
||||
TotalRewardFundSteem string `json:"total_reward_fund_steem"`
|
||||
TotalRewardShares2 string `json:"total_reward_shares2"`
|
||||
TotalVestingFundSteem string `json:"total_vesting_fund_steem"`
|
||||
TotalVestingShares string `json:"total_vesting_shares"`
|
||||
VirtualSupply string `json:"virtual_supply"`
|
||||
VotePowerReserveRate int `json:"vote_power_reserve_rate"`
|
||||
}
|
||||
|
||||
type GetOpsInBlockResponse *[]OperationObject
|
||||
type GetDynamicGlobalPropertiesResponse *DynamicGlobalProperties
|
||||
|
||||
func (r *GetOpsInBlockRequestParams) MarshalJSON() ([]byte, error) {
|
||||
arr := []interface{}{r.BlockNum, r.VirtualOps}
|
||||
return json.Marshal(arr)
|
||||
}
|
||||
Loading…
Reference in New Issue