Skip to content

Commit 08bc042

Browse files
iuwqyiriuwqyir
committed
Use VersionedCollapsingMergeTree and insert based deletes
1 parent 2733e5c commit 08bc042

17 files changed

+512
-430
lines changed

‎api/api.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ type QueryParams struct {
4444

4545
// @Description Flag to enable abi decoding of tx data
4646
Decode bool `schema:"decode"`
47+
// @Description Flag to force consistent data at the expense of query speed
48+
ForceConsistentData bool `schema:"force_consistent_data"`
4749
}
4850

4951
// Meta represents metadata for a query response

‎internal/common/BlockTimestamp.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package common
2+
3+
import (
4+
"fmt"
5+
"time"
6+
)
7+
8+
// temporary type to handle backwards compatibility with old block timestamps
9+
type BlockTimestamp struct {
10+
time.Time
11+
}
12+
13+
// Implement sql.Scanner interface
14+
func (fs *BlockTimestamp) Scan(value interface{}) error {
15+
if value == nil {
16+
*fs = BlockTimestamp{time.Time{}}
17+
return nil
18+
}
19+
20+
switch v := value.(type) {
21+
case uint64:
22+
*fs = BlockTimestamp{time.Unix(int64(v), 0)}
23+
return nil
24+
case time.Time:
25+
*fs = BlockTimestamp{v}
26+
return nil
27+
default:
28+
return fmt.Errorf("expected uint64 or time.Time for BlockTimestamp, got %T", value)
29+
}
30+
}

‎internal/common/block.go‎

Lines changed: 25 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -2,31 +2,34 @@ package common
22

33
import (
44
"math/big"
5+
"time"
56
)
67

78
type Block struct {
8-
ChainId *big.Int `json:"chain_id" ch:"chain_id"`
9-
Number *big.Int `json:"number" ch:"number"`
10-
Hash string `json:"hash" ch:"hash"`
11-
ParentHash string `json:"parent_hash" ch:"parent_hash"`
12-
Timestamp uint64 `json:"timestamp" ch:"timestamp"`
13-
Nonce string `json:"nonce" ch:"nonce"`
14-
Sha3Uncles string `json:"sha3_uncles" ch:"sha3_uncles"`
15-
MixHash string `json:"mix_hash" ch:"mix_hash"`
16-
Miner string `json:"miner" ch:"miner"`
17-
StateRoot string `json:"state_root" ch:"state_root"`
18-
TransactionsRoot string `json:"transactions_root" ch:"transactions_root"`
19-
ReceiptsRoot string `json:"receipts_root" ch:"receipts_root"`
20-
LogsBloom string `json:"logs_bloom" ch:"logs_bloom"`
21-
Size uint64 `json:"size" ch:"size"`
22-
ExtraData string `json:"extra_data" ch:"extra_data"`
23-
Difficulty *big.Int `json:"difficulty" ch:"difficulty"`
24-
TotalDifficulty *big.Int `json:"total_difficulty" ch:"total_difficulty"`
25-
TransactionCount uint64 `json:"transaction_count" ch:"transaction_count"`
26-
GasLimit *big.Int `json:"gas_limit" ch:"gas_limit"`
27-
GasUsed *big.Int `json:"gas_used" ch:"gas_used"`
28-
WithdrawalsRoot string `json:"withdrawals_root" ch:"withdrawals_root"`
29-
BaseFeePerGas uint64 `json:"base_fee_per_gas" ch:"base_fee_per_gas"`
9+
ChainId *big.Int `json:"chain_id" ch:"chain_id"`
10+
Number *big.Int `json:"block_number" ch:"block_number"`
11+
Hash string `json:"hash" ch:"hash"`
12+
ParentHash string `json:"parent_hash" ch:"parent_hash"`
13+
Timestamp BlockTimestamp `json:"block_timestamp" ch:"block_timestamp"`
14+
Nonce string `json:"nonce" ch:"nonce"`
15+
Sha3Uncles string `json:"sha3_uncles" ch:"sha3_uncles"`
16+
MixHash string `json:"mix_hash" ch:"mix_hash"`
17+
Miner string `json:"miner" ch:"miner"`
18+
StateRoot string `json:"state_root" ch:"state_root"`
19+
TransactionsRoot string `json:"transactions_root" ch:"transactions_root"`
20+
ReceiptsRoot string `json:"receipts_root" ch:"receipts_root"`
21+
LogsBloom string `json:"logs_bloom" ch:"logs_bloom"`
22+
Size uint64 `json:"size" ch:"size"`
23+
ExtraData string `json:"extra_data" ch:"extra_data"`
24+
Difficulty *big.Int `json:"difficulty" ch:"difficulty"`
25+
TotalDifficulty *big.Int `json:"total_difficulty" ch:"total_difficulty"`
26+
TransactionCount uint64 `json:"transaction_count" ch:"transaction_count"`
27+
GasLimit *big.Int `json:"gas_limit" ch:"gas_limit"`
28+
GasUsed *big.Int `json:"gas_used" ch:"gas_used"`
29+
WithdrawalsRoot string `json:"withdrawals_root" ch:"withdrawals_root"`
30+
BaseFeePerGas uint64 `json:"base_fee_per_gas" ch:"base_fee_per_gas"`
31+
Sign int8 `json:"sign" ch:"sign"`
32+
InsertTimestamp time.Time `json:"insert_timestamp" ch:"insert_timestamp"`
3033
}
3134

3235
type BlockData struct {

‎internal/common/log.go‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,26 +5,29 @@ import (
55
"fmt"
66
"math/big"
77
"sync"
8+
"time"
89

910
"github.com/ethereum/go-ethereum/accounts/abi"
1011
gethCommon "github.com/ethereum/go-ethereum/common"
1112
"github.com/rs/zerolog/log"
1213
)
1314

1415
type Log struct {
15-
ChainId *big.Int `json:"chain_id" ch:"chain_id" swaggertype:"string"`
16-
BlockNumber *big.Int `json:"block_number" ch:"block_number" swaggertype:"string"`
17-
BlockHash string `json:"block_hash" ch:"block_hash"`
18-
BlockTimestamp uint64 `json:"block_timestamp" ch:"block_timestamp"`
19-
TransactionHash string `json:"transaction_hash" ch:"transaction_hash"`
20-
TransactionIndex uint64 `json:"transaction_index" ch:"transaction_index"`
21-
LogIndex uint64 `json:"log_index" ch:"log_index"`
22-
Address string `json:"address" ch:"address"`
23-
Data string `json:"data" ch:"data"`
24-
Topic0 string `json:"topic_0" ch:"topic_0"`
25-
Topic1 string `json:"topic_1" ch:"topic_1"`
26-
Topic2 string `json:"topic_2" ch:"topic_2"`
27-
Topic3 string `json:"topic_3" ch:"topic_3"`
16+
ChainId *big.Int `json:"chain_id" ch:"chain_id" swaggertype:"string"`
17+
BlockNumber *big.Int `json:"block_number" ch:"block_number" swaggertype:"string"`
18+
BlockHash string `json:"block_hash" ch:"block_hash"`
19+
BlockTimestamp BlockTimestamp `json:"block_timestamp" ch:"block_timestamp"`
20+
TransactionHash string `json:"transaction_hash" ch:"transaction_hash"`
21+
TransactionIndex uint64 `json:"transaction_index" ch:"transaction_index"`
22+
LogIndex uint64 `json:"log_index" ch:"log_index"`
23+
Address string `json:"address" ch:"address"`
24+
Data string `json:"data" ch:"data"`
25+
Topic0 string `json:"topic_0" ch:"topic_0"`
26+
Topic1 string `json:"topic_1" ch:"topic_1"`
27+
Topic2 string `json:"topic_2" ch:"topic_2"`
28+
Topic3 string `json:"topic_3" ch:"topic_3"`
29+
Sign int8 `json:"sign" ch:"sign"`
30+
InsertTimestamp time.Time `json:"insert_timestamp" ch:"insert_timestamp"`
2831
}
2932

3033
func (l *Log) GetTopic(index int) (string, error) {

‎internal/common/trace.go‎

Lines changed: 24 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2,30 +2,33 @@ package common
22

33
import (
44
"math/big"
5+
"time"
56
)
67

78
type Trace struct {
8-
ChainID *big.Int `json:"chain_id" ch:"chain_id"`
9-
BlockNumber *big.Int `json:"block_number" ch:"block_number"`
10-
BlockHash string `json:"block_hash" ch:"block_hash"`
11-
BlockTimestamp uint64 `json:"block_timestamp" ch:"block_timestamp"`
12-
TransactionHash string `json:"transaction_hash" ch:"transaction_hash"`
13-
TransactionIndex uint64 `json:"transaction_index" ch:"transaction_index"`
14-
Subtraces int64 `json:"subtraces" ch:"subtraces"`
15-
TraceAddress []uint64 `json:"trace_address" ch:"trace_address"`
16-
TraceType string `json:"trace_type" ch:"type"`
17-
CallType string `json:"call_type" ch:"call_type"`
18-
Error string `json:"error" ch:"error"`
19-
FromAddress string `json:"from_address" ch:"from_address"`
20-
ToAddress string `json:"to_address" ch:"to_address"`
21-
Gas *big.Int `json:"gas" ch:"gas"`
22-
GasUsed *big.Int `json:"gas_used" ch:"gas_used"`
23-
Input string `json:"input" ch:"input"`
24-
Output string `json:"output" ch:"output"`
25-
Value *big.Int `json:"value" ch:"value"`
26-
Author string `json:"author" ch:"author"`
27-
RewardType string `json:"reward_type" ch:"reward_type"`
28-
RefundAddress string `json:"refund_address" ch:"refund_address"`
9+
ChainID *big.Int `json:"chain_id" ch:"chain_id"`
10+
BlockNumber *big.Int `json:"block_number" ch:"block_number"`
11+
BlockHash string `json:"block_hash" ch:"block_hash"`
12+
BlockTimestamp time.Time `json:"block_timestamp" ch:"block_timestamp"`
13+
TransactionHash string `json:"transaction_hash" ch:"transaction_hash"`
14+
TransactionIndex uint64 `json:"transaction_index" ch:"transaction_index"`
15+
Subtraces int64 `json:"subtraces" ch:"subtraces"`
16+
TraceAddress []uint64 `json:"trace_address" ch:"trace_address"`
17+
TraceType string `json:"trace_type" ch:"type"`
18+
CallType string `json:"call_type" ch:"call_type"`
19+
Error string `json:"error" ch:"error"`
20+
FromAddress string `json:"from_address" ch:"from_address"`
21+
ToAddress string `json:"to_address" ch:"to_address"`
22+
Gas *big.Int `json:"gas" ch:"gas"`
23+
GasUsed *big.Int `json:"gas_used" ch:"gas_used"`
24+
Input string `json:"input" ch:"input"`
25+
Output string `json:"output" ch:"output"`
26+
Value *big.Int `json:"value" ch:"value"`
27+
Author string `json:"author" ch:"author"`
28+
RewardType string `json:"reward_type" ch:"reward_type"`
29+
RefundAddress string `json:"refund_address" ch:"refund_address"`
30+
Sign int8 `json:"sign" ch:"sign"`
31+
InsertTimestamp time.Time `json:"insert_timestamp" ch:"insert_timestamp"`
2932
}
3033

3134
type RawTraces = []map[string]interface{}

‎internal/common/transaction.go‎

Lines changed: 32 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -5,41 +5,44 @@ import (
55
"math/big"
66
"strings"
77
"sync"
8+
"time"
89

910
"github.com/ethereum/go-ethereum/accounts/abi"
1011
"github.com/rs/zerolog/log"
1112
)
1213

1314
type Transaction struct {
14-
ChainId *big.Int `json:"chain_id" ch:"chain_id" swaggertype:"string"`
15-
Hash string `json:"hash" ch:"hash"`
16-
Nonce uint64 `json:"nonce" ch:"nonce"`
17-
BlockHash string `json:"block_hash" ch:"block_hash"`
18-
BlockNumber *big.Int `json:"block_number" ch:"block_number" swaggertype:"string"`
19-
BlockTimestamp uint64 `json:"block_timestamp" ch:"block_timestamp"`
20-
TransactionIndex uint64 `json:"transaction_index" ch:"transaction_index"`
21-
FromAddress string `json:"from_address" ch:"from_address"`
22-
ToAddress string `json:"to_address" ch:"to_address"`
23-
Value *big.Int `json:"value" ch:"value" swaggertype:"string"`
24-
Gas uint64 `json:"gas" ch:"gas"`
25-
GasPrice *big.Int `json:"gas_price" ch:"gas_price" swaggertype:"string"`
26-
Data string `json:"data" ch:"data"`
27-
FunctionSelector string `json:"function_selector" ch:"function_selector"`
28-
MaxFeePerGas *big.Int `json:"max_fee_per_gas" ch:"max_fee_per_gas" swaggertype:"string"`
29-
MaxPriorityFeePerGas *big.Int `json:"max_priority_fee_per_gas" ch:"max_priority_fee_per_gas" swaggertype:"string"`
30-
TransactionType uint8 `json:"transaction_type" ch:"transaction_type"`
31-
R *big.Int `json:"r" ch:"r" swaggertype:"string"`
32-
S *big.Int `json:"s" ch:"s" swaggertype:"string"`
33-
V *big.Int `json:"v" ch:"v" swaggertype:"string"`
34-
AccessListJson *string `json:"access_list_json" ch:"access_list"`
35-
ContractAddress *string `json:"contract_address" ch:"contract_address"`
36-
GasUsed *uint64 `json:"gas_used" ch:"gas_used"`
37-
CumulativeGasUsed *uint64 `json:"cumulative_gas_used" ch:"cumulative_gas_used"`
38-
EffectiveGasPrice *big.Int `json:"effective_gas_price" ch:"effective_gas_price" swaggertype:"string"`
39-
BlobGasUsed *uint64 `json:"blob_gas_used" ch:"blob_gas_used"`
40-
BlobGasPrice *big.Int `json:"blob_gas_price" ch:"blob_gas_price" swaggertype:"string"`
41-
LogsBloom *string `json:"logs_bloom" ch:"logs_bloom"`
42-
Status *uint64 `json:"status" ch:"status"`
15+
ChainId *big.Int `json:"chain_id" ch:"chain_id" swaggertype:"string"`
16+
Hash string `json:"hash" ch:"hash"`
17+
Nonce uint64 `json:"nonce" ch:"nonce"`
18+
BlockHash string `json:"block_hash" ch:"block_hash"`
19+
BlockNumber *big.Int `json:"block_number" ch:"block_number" swaggertype:"string"`
20+
BlockTimestamp BlockTimestamp `json:"block_timestamp" ch:"block_timestamp"`
21+
TransactionIndex uint64 `json:"transaction_index" ch:"transaction_index"`
22+
FromAddress string `json:"from_address" ch:"from_address"`
23+
ToAddress string `json:"to_address" ch:"to_address"`
24+
Value *big.Int `json:"value" ch:"value" swaggertype:"string"`
25+
Gas uint64 `json:"gas" ch:"gas"`
26+
GasPrice *big.Int `json:"gas_price" ch:"gas_price" swaggertype:"string"`
27+
Data string `json:"data" ch:"data"`
28+
FunctionSelector string `json:"function_selector" ch:"function_selector"`
29+
MaxFeePerGas *big.Int `json:"max_fee_per_gas" ch:"max_fee_per_gas" swaggertype:"string"`
30+
MaxPriorityFeePerGas *big.Int `json:"max_priority_fee_per_gas" ch:"max_priority_fee_per_gas" swaggertype:"string"`
31+
TransactionType uint8 `json:"transaction_type" ch:"transaction_type"`
32+
R *big.Int `json:"r" ch:"r" swaggertype:"string"`
33+
S *big.Int `json:"s" ch:"s" swaggertype:"string"`
34+
V *big.Int `json:"v" ch:"v" swaggertype:"string"`
35+
AccessListJson *string `json:"access_list_json" ch:"access_list"`
36+
ContractAddress *string `json:"contract_address" ch:"contract_address"`
37+
GasUsed *uint64 `json:"gas_used" ch:"gas_used"`
38+
CumulativeGasUsed *uint64 `json:"cumulative_gas_used" ch:"cumulative_gas_used"`
39+
EffectiveGasPrice *big.Int `json:"effective_gas_price" ch:"effective_gas_price" swaggertype:"string"`
40+
BlobGasUsed *uint64 `json:"blob_gas_used" ch:"blob_gas_used"`
41+
BlobGasPrice *big.Int `json:"blob_gas_price" ch:"blob_gas_price" swaggertype:"string"`
42+
LogsBloom *string `json:"logs_bloom" ch:"logs_bloom"`
43+
Status *uint64 `json:"status" ch:"status"`
44+
Sign int8 `json:"sign" ch:"sign"`
45+
InsertTimestamp time.Time `json:"insert_timestamp" ch:"insert_timestamp"`
4346
}
4447

4548
type DecodedTransactionData struct {

‎internal/common/utils.go‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -163,20 +163,21 @@ func isType(word string) bool {
163163
var allowedFunctions = map[string]struct{}{
164164
"sum": {},
165165
"count": {},
166-
"reinterpretAsUInt256": {},
166+
"reinterpretasuint256": {},
167167
"reverse": {},
168168
"unhex": {},
169169
"substring": {},
170170
"length": {},
171-
"toUInt256": {},
171+
"touint256": {},
172172
"if": {},
173-
"toStartOfMonth": {},
174-
"toStartOfDay": {},
175-
"toStartOfHour": {},
176-
"toStartOfMinute": {},
177-
"toDate": {},
178-
"toDateTime": {},
173+
"tostartofmonth": {},
174+
"tostartofday": {},
175+
"tostartofhour": {},
176+
"tostartofminute": {},
177+
"todate": {},
178+
"todatetime": {},
179179
"concat": {},
180+
"in": {},
180181
}
181182

182183
var disallowedPatterns = []string{
@@ -207,7 +208,7 @@ func ValidateQuery(query string) error {
207208
matches := functionPattern.FindAllStringSubmatch(query, -1)
208209
for _, match := range matches {
209210
funcName := match[1]
210-
if _, ok := allowedFunctions[funcName]; !ok {
211+
if _, ok := allowedFunctions[strings.ToLower(funcName)]; !ok {
211212
return fmt.Errorf("function '%s' is not allowed", funcName)
212213
}
213214
}

‎internal/handlers/blocks_handlers.go‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ type BlockModel struct {
4848
// @Param page query int false "Page number for pagination"
4949
// @Param limit query int false "Number of items per page" default(5)
5050
// @Param aggregate query []string false "List of aggregate functions to apply"
51+
// @Param force_consistent_data query bool false "Force consistent data at the expense of query speed"
5152
// @Success 200 {object} api.QueryResponse{data=[]BlockModel}
5253
// @Failure 400 {object} api.Error
5354
// @Failure 401 {object} api.Error
@@ -79,12 +80,13 @@ func handleBlocksRequest(c *gin.Context) {
7980

8081
// Prepare the QueryFilter
8182
qf := storage.QueryFilter{
82-
FilterParams: queryParams.FilterParams,
83-
ChainId: chainId,
84-
SortBy: queryParams.SortBy,
85-
SortOrder: queryParams.SortOrder,
86-
Page: queryParams.Page,
87-
Limit: queryParams.Limit,
83+
FilterParams: queryParams.FilterParams,
84+
ChainId: chainId,
85+
SortBy: queryParams.SortBy,
86+
SortOrder: queryParams.SortOrder,
87+
Page: queryParams.Page,
88+
Limit: queryParams.Limit,
89+
ForceConsistentData: queryParams.ForceConsistentData,
8890
}
8991

9092
// Initialize the QueryResult
@@ -139,7 +141,7 @@ func serializeBlocks(blocks []common.Block) []BlockModel {
139141
Number: block.Number.Uint64(),
140142
Hash: block.Hash,
141143
ParentHash: block.ParentHash,
142-
Timestamp: block.Timestamp,
144+
Timestamp: uint64(block.Timestamp.Unix()),
143145
Nonce: block.Nonce,
144146
Sha3Uncles: block.Sha3Uncles,
145147
MixHash: block.MixHash,

0 commit comments

Comments
 (0)