Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 40 additions & 33 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,43 +16,46 @@ Events are created with a simple schema.

```json
{
"type": "event type",
"timestamp": "wall clock timestamp of event",
"context": "metadata about the event",
"payload": "the full event specific payload"
"type": "event type",
"timestamp": "wall clock timestamp of event",
"context": "metadata about the event",
"payload": "the full event specific payload"
}
```

The chainsync input produces three event types: `block`, `rollback`, and
`transaction`. Each type has a unique payload.

block:

```json
{
"context": {
"blockNumber": 123,
"slotNumber": 1234567,
},
"payload": {
"blockBodySize": 123,
"issuerVkey": "a712f81ab2eac...",
"blockHash": "abcd123...",
"blockCbor": "85828a1a000995c21..."
}
"context": {
"blockNumber": 123,
"slotNumber": 1234567
},
"payload": {
"blockBodySize": 123,
"issuerVkey": "a712f81ab2eac...",
"blockHash": "abcd123...",
"blockCbor": "85828a1a000995c21..."
}
}
```

rollback:

```json
{
"payload": {
"blockHash": "abcd123...",
"slotNumber": 1234567
}
"payload": {
"blockHash": "abcd123...",
"slotNumber": 1234567
}
}
```

transaction:

```json
{
"context": {
Expand Down Expand Up @@ -105,22 +108,23 @@ Adder supports multiple configuration methods for versatility: commandline
arguments, YAML config file, and environment variables (in that order).

You can get a list of all available commandline arguments by using the
`-h`/`-help` flag.
`--help` flag.

```bash
$ ./adder -h
Usage of adder:
-config string
path to config file to load
-input string
input plugin to use, 'list' to show available (default "chainsync")
-input-chainsync-address string
specifies the TCP address of the node to connect to
$ ./adder --help

Usage:
adder [flags]

Flags:
--config string path to config file to load
--input string input plugin to use, 'list' to show available (default "chainsync")
--input-chainsync-address string
specifies the TCP address of the node to connect to
...
-output string
output plugin to use, 'list' to show available (default "log")
-output-log-level string
specifies the log level to use (default "info")
--output string output plugin to use, 'list' to show available (default "log")
--output-log-level string specifies the log level to use (default "info")
-h, --help help for adder
```

Each commandline argument (other than `-config`) has a corresponding environment
Expand All @@ -133,7 +137,7 @@ environment variable, and `-output` has `OUTPUT`.
Core configuration options can be set using environment variables:

- `INPUT` - Input plugin to use (default: "chainsync")
- `OUTPUT` - Output plugin to use (default: "log")
- `OUTPUT` - Output plugin to use (default: "log")
- `KUPO_URL` - URL for Kupo service integration
- `LOGGING_LEVEL` - Log level (default: "info")
- `API_ADDRESS` - API server listen address (default: "0.0.0.0")
Expand All @@ -144,14 +148,17 @@ Core configuration options can be set using environment variables:
Genesis configuration can also be controlled via environment variables:

**Network Transition:**

- `SHELLEY_TRANS_EPOCH` - Epoch number when Shelley era begins (default: 208 for mainnet)

**Byron Genesis:**

- `BYRON_GENESIS_END_SLOT` - End slot for Byron era
- `BYRON_GENESIS_EPOCH_LENGTH` - Slot length of Byron epochs (default: 21600)
- `BYRON_GENESIS_BYRON_SLOTS_PER_EPOCH` - Byron slots per epoch

**Shelley Genesis:**

- `SHELLEY_GENESIS_EPOCH_LENGTH` - Slot length of Shelley epochs (default: 432000)

You can also specify each option in the config file.
Expand Down Expand Up @@ -211,7 +218,7 @@ filters will be output.

```bash
export INPUT_CHAINSYNC_NETWORK=preview
./adder
./adder
```

Alternatively using equivalent commandline options:
Expand Down
65 changes: 37 additions & 28 deletions cmd/adder/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,22 @@ import (
"github.com/blinklabs-io/adder/pipeline"
"github.com/blinklabs-io/adder/plugin"
"github.com/inconshreveable/mousetrap"
"github.com/spf13/cobra"
"go.uber.org/automaxprocs/maxprocs"
)

var programName string = "adder"
var (
programName string = "adder"
cfg = config.GetConfig()
rootCmd = &cobra.Command{
Use: programName,
SilenceUsage: true,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
return run()
},
}
)

func slogPrintf(format string, v ...any) {
slog.Info(fmt.Sprintf(format, v...))
Expand All @@ -46,6 +58,7 @@ func slogPrintf(format string, v ...any) {
func init() {
if os.Args != nil && os.Args[0] != programName {
programName = os.Args[0]
rootCmd.Use = programName
}

// Bail if we were run via double click on Windows, borrowed from ngrok
Expand All @@ -64,58 +77,47 @@ func init() {
os.Exit(1)
}
}
}

func main() {
cfg := config.GetConfig()

if os.Args == nil {
fmt.Println("Failed to detect arguments, aborting")
os.Exit(1)
}

if err := cfg.ParseCmdlineArgs(programName, os.Args[1:]); err != nil {
fmt.Printf("Failed to parse commandline: %s\n", err)
os.Exit(1)
if err := cfg.BindFlags(rootCmd.Flags()); err != nil {
panic(err)
}
}

func run() error {
if cfg.Version {
fmt.Printf("%s %s\n", programName, version.GetVersionString())
os.Exit(0)
return nil
}

if cfg.Input == "list" {
fmt.Printf("Available input plugins:\n\n")
for _, plugin := range plugin.GetPlugins(plugin.PluginTypeInput) {
fmt.Printf("%- 14s %s\n", plugin.Name, plugin.Description)
}
return
return nil
}

if cfg.Output == "list" {
fmt.Printf("Available output plugins:\n\n")
for _, plugin := range plugin.GetPlugins(plugin.PluginTypeOutput) {
fmt.Printf("%- 14s %s\n", plugin.Name, plugin.Description)
}
return
return nil
}

// Load config
if err := cfg.Load(cfg.ConfigFile); err != nil {
fmt.Printf("Failed to load config: %s\n", err)
os.Exit(1)
return fmt.Errorf("failed to load config: %w", err)
}

// Process config for plugins
if err := plugin.ProcessConfig(cfg.Plugin); err != nil {
fmt.Printf("Failed to process plugin config: %s\n", err)
os.Exit(1)
return fmt.Errorf("failed to process plugin config: %w", err)
}

// Process env vars for plugins
if err := plugin.ProcessEnvVars(); err != nil {
fmt.Printf("Failed to process env vars: %s\n", err)
os.Exit(1)
return fmt.Errorf("failed to process env vars: %w", err)
}

// Configure logging
Expand All @@ -128,7 +130,7 @@ func main() {
if err != nil {
// If we hit this, something really wrong happened
logger.Error(err.Error())
os.Exit(1)
return err
}

// Start debug listener
Expand Down Expand Up @@ -170,7 +172,7 @@ func main() {
input := plugin.GetPlugin(plugin.PluginTypeInput, cfg.Input)
if input == nil {
logger.Error("unknown input: " + cfg.Input)
os.Exit(1)
return fmt.Errorf("unknown input: %s", cfg.Input)
}
pipe.AddInput(input)

Expand All @@ -184,7 +186,7 @@ func main() {
output := plugin.GetPlugin(plugin.PluginTypeOutput, cfg.Output)
if output == nil {
logger.Error("unknown output: " + cfg.Output)
os.Exit(1)
return fmt.Errorf("unknown output: %s", cfg.Output)
}
// Check if output plugin implements APIRouteRegistrar
if registrar, ok := any(output).(api.APIRouteRegistrar); ok {
Expand All @@ -195,13 +197,13 @@ func main() {
// Start API after plugins are configured
if err := apiInstance.Start(); err != nil {
logger.Error(fmt.Sprintf("failed to start API: %s", err))
os.Exit(1)
return fmt.Errorf("failed to start API: %w", err)
}

// Start pipeline and wait for error
if err := pipe.Start(); err != nil {
logger.Error(fmt.Sprintf("failed to start pipeline: %s", err))
os.Exit(1)
return fmt.Errorf("failed to start pipeline: %w", err)
}

// Setup graceful shutdown
Expand All @@ -225,8 +227,15 @@ func main() {
// Graceful shutdown using Stop() method
if err := pipe.Stop(); err != nil {
logger.Error(fmt.Sprintf("failed to stop pipeline: %s", err))
os.Exit(1)
return fmt.Errorf("failed to stop pipeline: %w", err)
}

logger.Info("Adder stopped gracefully")
return nil
}

func main() {
if err := rootCmd.Execute(); err != nil {
os.Exit(1)
}
}
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ require (
github.com/gin-gonic/gin v1.11.0
github.com/inconshreveable/mousetrap v1.1.0
github.com/kelseyhightower/envconfig v1.4.0
github.com/spf13/cobra v1.8.1
github.com/spf13/pflag v1.0.6
github.com/stretchr/testify v1.11.1
github.com/swaggo/files v1.0.1
github.com/swaggo/gin-swagger v1.6.1
Expand Down
7 changes: 7 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI
github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU=
github.com/consensys/gnark-crypto v0.19.2 h1:qrEAIXq3T4egxqiliFFoNrepkIWVEeIYwt3UL0fvS80=
github.com/consensys/gnark-crypto v0.19.2/go.mod h1:rT23F0XSZqE0mUA0+pRtnL56IbPxs6gp4CeRsBk4XS0=
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand Down Expand Up @@ -218,10 +219,16 @@ github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2e
github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sergeymakinen/go-bmp v1.0.0 h1:SdGTzp9WvCV0A1V0mBeaS7kQAwNLdVJbmHlqNWq0R+M=
github.com/sergeymakinen/go-bmp v1.0.0/go.mod h1:/mxlAQZRLxSvJFNIEGGLBE/m40f3ZnUifpgVDlcUIEY=
github.com/sergeymakinen/go-ico v1.0.0-beta.0 h1:m5qKH7uPKLdrygMWxbamVn+tl2HfiA3K6MFJw4GfZvQ=
github.com/sergeymakinen/go-ico v1.0.0-beta.0/go.mod h1:wQ47mTczswBO5F0NoDt7O0IXgnV4Xy3ojrroMQzyhUk=
github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
Expand Down
13 changes: 3 additions & 10 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,12 @@
package config

import (
"flag"
"fmt"
"os"

"github.com/blinklabs-io/adder/plugin"
"github.com/kelseyhightower/envconfig"
"github.com/spf13/pflag"
"gopkg.in/yaml.v2"
)

Expand Down Expand Up @@ -151,8 +151,7 @@ func (c *Config) Load(configFile string) error {
return nil
}

func (c *Config) ParseCmdlineArgs(programName string, args []string) error {
fs := flag.NewFlagSet(programName, flag.ExitOnError)
func (c *Config) BindFlags(fs *pflag.FlagSet) error {
fs.StringVar(&c.ConfigFile, "config", "", "path to config file to load")
fs.BoolVar(&c.Version, "version", false, "show version and exit")
fs.StringVar(
Expand All @@ -167,13 +166,7 @@ func (c *Config) ParseCmdlineArgs(programName string, args []string) error {
DefaultOutputPlugin,
"output plugin to use, 'list' to show available",
)
if err := plugin.PopulateCmdlineOptions(fs); err != nil {
return err
}
if err := fs.Parse(args); err != nil {
return err
}
return nil
return plugin.PopulateCmdlineOptions(fs)
}

// GetConfig returns the global config instance
Expand Down
5 changes: 3 additions & 2 deletions plugin/option.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,12 @@
package plugin

import (
"flag"
"fmt"
"os"
"strconv"
"strings"

"github.com/spf13/pflag"
)

type PluginOptionType int
Expand All @@ -42,7 +43,7 @@ type PluginOption struct {
}

func (p *PluginOption) AddToFlagSet(
fs *flag.FlagSet,
fs *pflag.FlagSet,
pluginType string,
pluginName string,
) error {
Expand Down
Loading