Skip to content

Commit 5b72f62

Browse files
author
Ali Akca
committed
Add first running version
Signed-off-by: Ali AKCA <ali@akca.io>
1 parent d286cee commit 5b72f62

File tree

18 files changed

+1009
-5
lines changed

18 files changed

+1009
-5
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# Project Directories
22
vendor/
33
build/
4+
tmp/
45

56
# IDE
67
.idea/

‎README.md‎

Lines changed: 84 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,95 @@ database with a small dataset for testing or development.
1515

1616
```
1717
Usage of pdd:
18-
--log-format string log format to use. ('fmt', 'json') (default "fmt")
19-
--log-level string log filtering level. ('error', 'warn', 'info', 'debug') (default "error")
20-
-v, --verbose verbose output
21-
--version Prints version info
18+
--log-format string log format to use. ('fmt', 'json') (default "fmt")
19+
--log-level string log filtering level. ('error', 'warn', 'info', 'debug') (default "error")
20+
-v, --verbose verbose output
21+
--addr string TCP host:port or Unix socket depending on Network (default "localhost:5432")
22+
--database string Database name (default "postgres")
23+
--user string Database user (default "postgres")
24+
--pass string Database password (default "postgres")
25+
--dial-timeout duration Dial timeout for establishing new connections (default 5s)
26+
--read-timeout duration Timeout for socket reads. If reached, commands will fail (default 30s)
27+
--max-retry int Maximum number of retries before giving up.
28+
--manifest-file string Path to manifest file (default ".pdd.yaml")
29+
--backend string storage backend to use (filesystem) (default "filesystem")
30+
--filesystem-root string local filesystem root directory (default "/tmp/pdd")
31+
--version Prints version info
2232
```
2333

34+
35+
### Environment Variables
36+
37+
Environment variables sets property value if it's not set from arguments. CLI arguments has higher priority.
38+
39+
| Name | Property |
40+
|:---|:---|
41+
| PDD_LOG_LEVEL | `--log-format` |
42+
| PDD_LOG_FORMAT | `--log-level` |
43+
| PDD_ADDR | `--addr` |
44+
| PDD_DATABASE | `--database` |
45+
| PDD_USER | `--user` |
46+
| PDD_PASS | `--pass` |
47+
| PDD_DIAL_TIMEOUT | `--dial-timeout` |
48+
| PDD_READ_TIMEOUT | `--read-timeout` |
49+
| PDD_MAX_RETRY | `--max-retry` |
50+
| PDD_MANIFEST_FILE | `--manifest-file` |
51+
| PDD_BACKEND | `--backend` |
52+
| PDD_FILESYSTEM_ROOT | `--filesystem-root` |
53+
54+
### Manifest file
55+
56+
The main difference between `pg_dump_sample` and `pg_dump(1)` is that
57+
`pg_dump_sample` requires a manifest file describing how to dump the database.
58+
The manifest file is a YAML file describing what tables to dump and how to dump
59+
them.
60+
61+
A quick example:
62+
63+
---
64+
vars:
65+
# Condition to dump only certain users
66+
matching_user_id: "(users.id BETWEEN 1000 AND 2000)"
67+
68+
tables:
69+
# Dump everything from table "consts"
70+
- table: consts
71+
72+
# Dump only matching users
73+
- table: users
74+
query: "SELECT * FROM users WHERE {{matching_user_id}}"
75+
post_actions:
76+
- "SELECT pg_catalog.setval('users_id_seq', MAX(id) + 1, true) FROM users"
77+
78+
# Dump only tickets that were bought by matching users
79+
- table: tickets
80+
query: >
81+
SELECT purchases.* FROM purchases, users
82+
WHERE
83+
purchases.buyer_id = users.id
84+
AND {{matching_user_id}}
85+
86+
87+
Currently, these top-level keys are available:
88+
89+
#### `vars`
90+
91+
Definitions of variables which will be used to replace placeholders in queries.
92+
93+
#### `tables`
94+
95+
List of tables to dump. Tables are dumped in the order they are specified in the
96+
manifest file, with one exception: if the table contains foreign keys
97+
referencing another table, the referenced table will be dumped first. This is to
98+
ensure that the dump can be loaded later without errors.
99+
100+
By default, all rows of the table will be dumped. If you don't want to dump all
101+
the rows use the `query` to specify a SELECT SQL statement which returns the
102+
rows you want to dump.
103+
24104
## Development
25105

26106
```
27-
28107
Usage:
29108
make <target>
30109

‎cmd/main.go‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,19 @@
11
package main
22

33
import (
4+
"fmt"
5+
"io"
46
syslog "log"
57
"os"
8+
"time"
69

10+
"github.com/aweris/postgres-data-dump/database"
11+
"github.com/aweris/postgres-data-dump/dump"
12+
"github.com/aweris/postgres-data-dump/internal/helpers"
713
"github.com/aweris/postgres-data-dump/internal/log"
14+
"github.com/aweris/postgres-data-dump/storage"
15+
"github.com/aweris/postgres-data-dump/storage/backend"
16+
"github.com/aweris/postgres-data-dump/storage/backend/fs"
817
"github.com/spf13/pflag"
918
)
1019

@@ -23,6 +32,15 @@ func main() {
2332
logFormat string
2433
verbose bool
2534

35+
// database
36+
dbc = database.Config{}
37+
38+
// dump
39+
dc = dump.Config{}
40+
41+
// backend
42+
bc = backend.Config{}
43+
2644
// other
2745
showVersion bool
2846
)
@@ -37,13 +55,49 @@ func main() {
3755
flag.StringVar(&logFormat, "log-format", log.FormatFmt, "log format to use. ('fmt', 'json')")
3856
flag.BoolVarP(&verbose, "verbose", "v", false, "verbose output")
3957

58+
// database flags
59+
flag.StringVar(&dbc.Addr, "addr", database.DefaultAddr, "TCP host:port or Unix socket depending on Network")
60+
flag.StringVar(&dbc.Database, "database", database.DefaultDatabase, "Database name")
61+
flag.StringVar(&dbc.User, "user", database.DefaultUser, "Database user")
62+
flag.StringVar(&dbc.Password, "pass", database.DefaultPassword, "Database password")
63+
flag.DurationVar(&dbc.DialTimeout, "dial-timeout", database.DefaultDialTimeout, "Dial timeout for establishing new connections")
64+
flag.DurationVar(&dbc.ReadTimeout, "read-timeout", database.DefaultReadTimeout, "Timeout for socket reads. If reached, commands will fail")
65+
flag.IntVar(&dbc.MaxRetries, "max-retry", database.DefaultMaxRetries, "Maximum number of retries before giving up.")
66+
67+
// dump flags
68+
flag.StringVar(&dc.ManifestFile, "manifest-file", dump.DefaultManifestFile, "Path to manifest file")
69+
70+
// backend
71+
flag.StringVar(&bc.Type, "backend", backend.FileSystem, "storage backend to use (filesystem)")
72+
73+
// backend filesystem
74+
flag.StringVar(&bc.FileSystem.Root, "filesystem-root", fs.DefaultRoot, "local filesystem root directory")
75+
4076
// other flags
4177
flag.BoolVar(&showVersion, "version", false, "Prints version info")
4278

4379
// bind environment variables
4480
bindEnv(flag.Lookup("log-level"), "PDD_LOG_LEVEL")
4581
bindEnv(flag.Lookup("log-format"), "PDD_LOG_FORMAT")
4682

83+
// database variables
84+
bindEnv(flag.Lookup("addr"), "PDD_ADDR")
85+
bindEnv(flag.Lookup("database"), "PDD_DATABASE")
86+
bindEnv(flag.Lookup("user"), "PDD_USER")
87+
bindEnv(flag.Lookup("pass"), "PDD_PASS")
88+
bindEnv(flag.Lookup("dial-timeout"), "PDD_DIAL_TIMEOUT")
89+
bindEnv(flag.Lookup("read-timeout"), "PDD_READ_TIMEOUT")
90+
bindEnv(flag.Lookup("max-retry"), "PDD_MAX_RETRY")
91+
92+
// dump variables
93+
bindEnv(flag.Lookup("manifest-file"), "PDD_MANIFEST_FILE")
94+
95+
// backend variables
96+
bindEnv(flag.Lookup("backend"), "PDD_BACKEND")
97+
98+
// backend - fs
99+
bindEnv(flag.Lookup("filesystem-root"), "PDD_FILESYSTEM_ROOT")
100+
47101
// Parse Options
48102
if err := flag.Parse(os.Args); err != nil {
49103
syslog.Fatalf("%#v", err)
@@ -68,6 +122,60 @@ func main() {
68122
}
69123

70124
logger.Debug("version", version, "git commit", commit, "build date", date)
125+
126+
// initialize db
127+
db, err := database.ConnectDB(logger, &dbc)
128+
if err != nil {
129+
logger.Error("msg", "failed to create database", "error", err)
130+
os.Exit(1)
131+
}
132+
133+
// initialize dumper
134+
dumper, err := dump.NewDumper(logger, db, dc)
135+
if err != nil {
136+
logger.Error("msg", "failed to create dumper", "error", err)
137+
os.Exit(1)
138+
}
139+
140+
// initialize backend
141+
b, err := backend.FromConfig(logger, bc)
142+
if err != nil {
143+
logger.Error("msg", "failed to create backend", "error", err)
144+
os.Exit(1)
145+
}
146+
147+
// initialize storage
148+
s := storage.New(logger, b, storage.DefaultOperationTimeout)
149+
150+
if err := run(logger, dumper, s); err != nil {
151+
logger.Error("msg", "failed to dump database", "error", err)
152+
os.Exit(1)
153+
}
154+
155+
logger.Debug("msg", "export finished")
156+
}
157+
158+
func run(logger log.Logger, dumper dump.Dumper, s storage.Storage) error {
159+
// create a synchronous in-memory pipe.
160+
pr, pw := io.Pipe()
161+
162+
defer helpers.CloseWithErrLogf(logger, pr, "dump error")
163+
164+
go func() {
165+
defer helpers.CloseWithErrLogf(logger, pw, "dump error")
166+
167+
if err := dumper.Dump(pw); err != nil {
168+
logger.Error("error", err)
169+
}
170+
}()
171+
172+
return s.Put(generateFileName(), pr)
173+
}
174+
175+
// generateFileName generates new file name for dump based on timestamp.
176+
func generateFileName() string {
177+
t := time.Now()
178+
return fmt.Sprintf("dump-%s.sql", t.Format("20060102-150405"))
71179
}
72180

73181
func bindEnv(fn *pflag.Flag, env string) {

‎database/config.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package database
2+
3+
import "time"
4+
5+
// default values.
6+
const (
7+
DefaultAddr = "localhost:5432"
8+
DefaultDatabase = "postgres"
9+
DefaultUser = "postgres"
10+
DefaultPassword = "postgres"
11+
DefaultMaxRetries = 0
12+
DefaultDialTimeout = 5 * time.Second
13+
DefaultReadTimeout = 30 * time.Second
14+
)
15+
16+
// Config contains database connection options.
17+
type Config struct {
18+
Addr string
19+
Database string
20+
User string
21+
Password string
22+
MaxRetries int
23+
DialTimeout time.Duration
24+
ReadTimeout time.Duration
25+
}

0 commit comments

Comments
 (0)