Skip to content

Commit e6818cf

Browse files
crazy-maxcrazy-max
committed
export(local): split opt
Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
1 parent b85b5ab commit e6818cf

File tree

4 files changed

+248
-11
lines changed

4 files changed

+248
-11
lines changed

‎README.md‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -293,6 +293,50 @@ COPY --from=builder /usr/src/app/testresult.xml .
293293
buildctl build ... --opt target=testresult --output type=local,dest=path/to/output-dir
294294
```
295295

296+
With a [multi-platform build](docs/multi-platform.md), a subfolder matching
297+
each target platform will be created in the destination directory:
298+
299+
```dockerfile
300+
FROM busybox AS build
301+
ARG TARGETOS
302+
ARG TARGETARCH
303+
RUN mkdir /out && echo foo > /out/hello-$TARGETOS-$TARGETARCH
304+
305+
FROM scratch
306+
COPY --from=build /out /
307+
```
308+
309+
```bash
310+
$ buildctl build \
311+
--frontend dockerfile.v0 \
312+
--opt platform=linux/amd64,linux/arm64 \
313+
--output type=local,dest=./bin/release
314+
315+
$ tree ./bin
316+
./bin/
317+
└── release
318+
├── linux_amd64
319+
│ └── hello-linux-amd64
320+
└── linux_arm64
321+
└── hello-linux-arm64
322+
```
323+
324+
You can set `platform-split=false` to merge files from all platforms together
325+
into same directory:
326+
327+
```bash
328+
$ buildctl build \
329+
--frontend dockerfile.v0 \
330+
--opt platform=linux/amd64,linux/arm64 \
331+
--output type=local,dest=./bin/release,platform-split=false
332+
333+
$ tree ./bin
334+
./bin/
335+
└── release
336+
├── hello-linux-amd64
337+
└── hello-linux-arm64
338+
```
339+
296340
Tar exporter is similar to local exporter but transfers the files through a tarball.
297341

298342
```bash

‎client/client_test.go‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,8 @@ func TestIntegration(t *testing.T) {
200200
testLLBMountPerformance,
201201
testClientCustomGRPCOpts,
202202
testMultipleRecordsWithSameLayersCacheImportExport,
203+
testExportLocalNoPlatformSplit,
204+
testExportLocalNoPlatformSplitOverwrite,
203205
)
204206
}
205207

@@ -5249,6 +5251,152 @@ func testMultipleRecordsWithSameLayersCacheImportExport(t *testing.T, sb integra
52495251
ensurePruneAll(t, c, sb)
52505252
}
52515253

5254+
func testExportLocalNoPlatformSplit(t *testing.T, sb integration.Sandbox) {
5255+
integration.CheckFeatureCompat(t, sb, integration.FeatureOCIExporter, integration.FeatureMultiPlatform)
5256+
c, err := New(sb.Context(), sb.Address())
5257+
require.NoError(t, err)
5258+
defer c.Close()
5259+
5260+
platformsToTest := []string{"linux/amd64", "linux/arm64"}
5261+
frontend := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
5262+
res := gateway.NewResult()
5263+
expPlatforms := &exptypes.Platforms{
5264+
Platforms: make([]exptypes.Platform, len(platformsToTest)),
5265+
}
5266+
for i, platform := range platformsToTest {
5267+
st := llb.Scratch().File(
5268+
llb.Mkfile("hello-"+strings.ReplaceAll(platform, "/", "-"), 0600, []byte(platform)),
5269+
)
5270+
5271+
def, err := st.Marshal(ctx)
5272+
if err != nil {
5273+
return nil, err
5274+
}
5275+
5276+
r, err := c.Solve(ctx, gateway.SolveRequest{
5277+
Definition: def.ToPB(),
5278+
})
5279+
if err != nil {
5280+
return nil, err
5281+
}
5282+
5283+
ref, err := r.SingleRef()
5284+
if err != nil {
5285+
return nil, err
5286+
}
5287+
5288+
_, err = ref.ToState()
5289+
if err != nil {
5290+
return nil, err
5291+
}
5292+
res.AddRef(platform, ref)
5293+
5294+
expPlatforms.Platforms[i] = exptypes.Platform{
5295+
ID: platform,
5296+
Platform: platforms.MustParse(platform),
5297+
}
5298+
}
5299+
dt, err := json.Marshal(expPlatforms)
5300+
if err != nil {
5301+
return nil, err
5302+
}
5303+
res.AddMeta(exptypes.ExporterPlatformsKey, dt)
5304+
5305+
return res, nil
5306+
}
5307+
5308+
destDir := t.TempDir()
5309+
_, err = c.Build(sb.Context(), SolveOpt{
5310+
Exports: []ExportEntry{
5311+
{
5312+
Type: ExporterLocal,
5313+
OutputDir: destDir,
5314+
Attrs: map[string]string{
5315+
"platform-split": "false",
5316+
},
5317+
},
5318+
},
5319+
}, "", frontend, nil)
5320+
require.NoError(t, err)
5321+
5322+
dt, err := os.ReadFile(filepath.Join(destDir, "hello-linux-amd64"))
5323+
require.NoError(t, err)
5324+
require.Equal(t, "linux/amd64", string(dt))
5325+
5326+
dt, err = os.ReadFile(filepath.Join(destDir, "hello-linux-arm64"))
5327+
require.NoError(t, err)
5328+
require.Equal(t, "linux/arm64", string(dt))
5329+
}
5330+
5331+
func testExportLocalNoPlatformSplitOverwrite(t *testing.T, sb integration.Sandbox) {
5332+
integration.CheckFeatureCompat(t, sb, integration.FeatureOCIExporter, integration.FeatureMultiPlatform)
5333+
c, err := New(sb.Context(), sb.Address())
5334+
require.NoError(t, err)
5335+
defer c.Close()
5336+
5337+
platformsToTest := []string{"linux/amd64", "linux/arm64"}
5338+
frontend := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
5339+
res := gateway.NewResult()
5340+
expPlatforms := &exptypes.Platforms{
5341+
Platforms: make([]exptypes.Platform, len(platformsToTest)),
5342+
}
5343+
for i, platform := range platformsToTest {
5344+
st := llb.Scratch().File(
5345+
llb.Mkfile("hello-linux", 0600, []byte(platform)),
5346+
)
5347+
5348+
def, err := st.Marshal(ctx)
5349+
if err != nil {
5350+
return nil, err
5351+
}
5352+
5353+
r, err := c.Solve(ctx, gateway.SolveRequest{
5354+
Definition: def.ToPB(),
5355+
})
5356+
if err != nil {
5357+
return nil, err
5358+
}
5359+
5360+
ref, err := r.SingleRef()
5361+
if err != nil {
5362+
return nil, err
5363+
}
5364+
5365+
_, err = ref.ToState()
5366+
if err != nil {
5367+
return nil, err
5368+
}
5369+
res.AddRef(platform, ref)
5370+
5371+
expPlatforms.Platforms[i] = exptypes.Platform{
5372+
ID: platform,
5373+
Platform: platforms.MustParse(platform),
5374+
}
5375+
}
5376+
dt, err := json.Marshal(expPlatforms)
5377+
if err != nil {
5378+
return nil, err
5379+
}
5380+
res.AddMeta(exptypes.ExporterPlatformsKey, dt)
5381+
5382+
return res, nil
5383+
}
5384+
5385+
destDir := t.TempDir()
5386+
_, err = c.Build(sb.Context(), SolveOpt{
5387+
Exports: []ExportEntry{
5388+
{
5389+
Type: ExporterLocal,
5390+
OutputDir: destDir,
5391+
Attrs: map[string]string{
5392+
"platform-split": "false",
5393+
},
5394+
},
5395+
},
5396+
}, "", frontend, nil)
5397+
require.Error(t, err)
5398+
}
5399+
52525400
func readFileInImage(ctx context.Context, t *testing.T, c *Client, ref, path string) ([]byte, error) {
52535401
def, err := llb.Image(ref).Marshal(ctx)
52545402
if err != nil {

‎exporter/local/export.go‎

Lines changed: 38 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"os"
66
"strings"
7+
"sync"
78
"time"
89

910
"github.com/moby/buildkit/cache"
@@ -92,6 +93,9 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source
9293

9394
now := time.Now().Truncate(time.Second)
9495

96+
visitedPath := map[string]string{}
97+
var visitedMu sync.Mutex
98+
9599
export := func(ctx context.Context, k string, ref cache.ImmutableRef, attestations []exporter.Attestation) func() error {
96100
return func() error {
97101
outputFS, cleanup, err := CreateFS(ctx, sessionID, k, ref, attestations, now, e.opts)
@@ -102,20 +106,43 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source
102106
defer cleanup()
103107
}
104108

109+
if !e.opts.PlatformSplit {
110+
// check for duplicate paths
111+
err = outputFS.Walk(ctx, func(p string, fi os.FileInfo, err error) error {
112+
if fi.IsDir() {
113+
return nil
114+
}
115+
if err != nil && !errors.Is(err, os.ErrNotExist) {
116+
return err
117+
}
118+
visitedMu.Lock()
119+
defer visitedMu.Unlock()
120+
if vp, ok := visitedPath[p]; ok {
121+
return errors.Errorf("cannot overwrite %s from %s with %s when split option is disabled", p, vp, k)
122+
}
123+
visitedPath[p] = k
124+
return nil
125+
})
126+
if err != nil {
127+
return err
128+
}
129+
}
130+
105131
lbl := "copying files"
106132
if isMap {
107133
lbl += " " + k
108-
st := fstypes.Stat{
109-
Mode: uint32(os.ModeDir | 0755),
110-
Path: strings.Replace(k, "/", "_", -1),
111-
}
112-
if e.opts.Epoch != nil {
113-
st.ModTime = e.opts.Epoch.UnixNano()
114-
}
115-
116-
outputFS, err = fsutil.SubDirFS([]fsutil.Dir{{FS: outputFS, Stat: st}})
117-
if err != nil {
118-
return err
134+
if e.opts.PlatformSplit {
135+
st := fstypes.Stat{
136+
Mode: uint32(os.ModeDir | 0755),
137+
Path: strings.Replace(k, "/", "_", -1),
138+
}
139+
if e.opts.Epoch != nil {
140+
st.ModTime = e.opts.Epoch.UnixNano()
141+
}
142+
outputFS, err = fsutil.SubDirFS([]fsutil.Dir{{FS: outputFS, Stat: st}})
143+
if err != nil {
144+
return err
145+
}
119146
}
120147
}
121148

‎exporter/local/fs.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,13 @@ package local
33
import (
44
"context"
55
"encoding/json"
6+
"fmt"
67
"io"
78
"io/fs"
89
"os"
910
"path"
1011
"strconv"
12+
"strings"
1113
"time"
1214

1315
"github.com/docker/docker/pkg/idtools"
@@ -28,15 +30,20 @@ import (
2830

2931
const (
3032
keyAttestationPrefix = "attestation-prefix"
33+
// keyPlatformSplit is an exporter option which can be used to split result
34+
// in subfolders when multiple platform references are exported.
35+
keyPlatformSplit = "platform-split"
3136
)
3237

3338
type CreateFSOpts struct {
3439
Epoch *time.Time
3540
AttestationPrefix string
41+
PlatformSplit bool
3642
}
3743

3844
func (c *CreateFSOpts) Load(opt map[string]string) (map[string]string, error) {
3945
rest := make(map[string]string)
46+
c.PlatformSplit = true
4047

4148
var err error
4249
c.Epoch, opt, err = epoch.ParseExporterAttrs(opt)
@@ -48,6 +55,12 @@ func (c *CreateFSOpts) Load(opt map[string]string) (map[string]string, error) {
4855
switch k {
4956
case keyAttestationPrefix:
5057
c.AttestationPrefix = v
58+
case keyPlatformSplit:
59+
b, err := strconv.ParseBool(v)
60+
if err != nil {
61+
return nil, errors.Wrapf(err, "non-bool value for %s: %s", keyPlatformSplit, v)
62+
}
63+
c.PlatformSplit = b
5164
default:
5265
rest[k] = v
5366
}
@@ -164,6 +177,11 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab
164177
}
165178

166179
name := opt.AttestationPrefix + path.Base(attestations[i].Path)
180+
if !opt.PlatformSplit {
181+
nameExt := path.Ext(name)
182+
namBase := strings.TrimSuffix(name, nameExt)
183+
name = fmt.Sprintf("%s.%s%s", namBase, strings.Replace(k, "/", "_", -1), nameExt)
184+
}
167185
if _, ok := names[name]; ok {
168186
return nil, nil, errors.Errorf("duplicate attestation path name %s", name)
169187
}

0 commit comments

Comments
 (0)