Skip to content

Commit 652bcac

Browse files
cmaglieumbynos
cmaglie
and
umbynos
authored
Allow downloading index and signature in a single tar.bz2 archive (#1734)
* Allow downloading index and signature in a tar.bz2 archive * Make some configurations resilient to empty settings * Added tests * fix comment * Apply suggestions from code review Co-authored-by: Umberto Baldi <34278123+umbynos@users.noreply.github.com> * Allow .tar.bz2 package index URL * Fix errors in timeout checks * Fixed check for stale package_index.json files Co-authored-by: Umberto Baldi <34278123+umbynos@users.noreply.github.com>
1 parent b7721c8 commit 652bcac

File tree

12 files changed

+114
-20
lines changed

12 files changed

+114
-20
lines changed

‎arduino/cores/packagemanager/package_manager.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,11 @@ func (pm *PackageManager) ResolveFQBN(fqbn *cores.FQBN) (
237237

238238
// LoadPackageIndex loads a package index by looking up the local cached file from the specified URL
239239
func (pm *PackageManager) LoadPackageIndex(URL *url.URL) error {
240-
indexPath := pm.IndexDir.Join(path.Base(URL.Path))
240+
indexFileName := path.Base(URL.Path)
241+
if strings.HasSuffix(indexFileName, ".tar.bz2") {
242+
indexFileName = strings.TrimSuffix(indexFileName, ".tar.bz2") + ".json"
243+
}
244+
indexPath := pm.IndexDir.Join(indexFileName)
241245
index, err := packageindex.LoadIndex(indexPath)
242246
if err != nil {
243247
return fmt.Errorf(tr("loading json index file %[1]s: %[2]s"), indexPath, err)

‎arduino/resources/index.go‎

Lines changed: 46 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
package resources
1717

1818
import (
19+
"context"
1920
"net/url"
2021
"path"
2122
"strings"
@@ -25,6 +26,8 @@ import (
2526
"github.com/arduino/arduino-cli/arduino/security"
2627
rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
2728
"github.com/arduino/go-paths-helper"
29+
"github.com/codeclysm/extract/v3"
30+
"github.com/sirupsen/logrus"
2831
"go.bug.st/downloader/v2"
2932
)
3033

@@ -56,8 +59,43 @@ func (res *IndexResource) Download(destDir *paths.Path, downloadCB rpc.DownloadP
5659
return &arduino.FailedDownloadError{Message: tr("Error downloading index '%s'", res.URL), Cause: err}
5760
}
5861

62+
var signaturePath, tmpSignaturePath *paths.Path
63+
hasSignature := false
64+
5965
// Expand the index if it is compressed
60-
if strings.HasSuffix(indexFileName, ".gz") {
66+
if strings.HasSuffix(indexFileName, ".tar.bz2") {
67+
indexFileName = strings.TrimSuffix(indexFileName, ".tar.bz2") + ".json" // == package_index.json
68+
signatureFileName := indexFileName + ".sig"
69+
signaturePath = destDir.Join(signatureFileName)
70+
71+
// .tar.bz2 archive may contain both index and signature
72+
73+
// Extract archive in a tmp/archive subdirectory
74+
f, err := tmpIndexPath.Open()
75+
if err != nil {
76+
return &arduino.PermissionDeniedError{Message: tr("Error opening %s", tmpIndexPath), Cause: err}
77+
}
78+
defer f.Close()
79+
tmpArchivePath := tmp.Join("archive")
80+
_ = tmpArchivePath.MkdirAll()
81+
if err := extract.Bz2(context.Background(), f, tmpArchivePath.String(), nil); err != nil {
82+
return &arduino.PermissionDeniedError{Message: tr("Error extracting %s", tmpIndexPath), Cause: err}
83+
}
84+
85+
// Look for index.json
86+
tmpIndexPath = tmpArchivePath.Join(indexFileName)
87+
if !tmpIndexPath.Exist() {
88+
return &arduino.NotFoundError{Message: tr("Invalid archive: file %{1}s not found in archive %{2}s", indexFileName, tmpArchivePath.Base())}
89+
}
90+
91+
// Look for signature
92+
if t := tmpArchivePath.Join(signatureFileName); t.Exist() {
93+
tmpSignaturePath = t
94+
hasSignature = true
95+
} else {
96+
logrus.Infof("No signature %s found in package index archive %s", signatureFileName, tmpArchivePath.Base())
97+
}
98+
} else if strings.HasSuffix(indexFileName, ".gz") {
6199
indexFileName = strings.TrimSuffix(indexFileName, ".gz") // == package_index.json
62100
tmpUnzippedIndexPath := tmp.Join(indexFileName)
63101
if err := paths.GUnzip(tmpIndexPath, tmpUnzippedIndexPath); err != nil {
@@ -67,7 +105,6 @@ func (res *IndexResource) Download(destDir *paths.Path, downloadCB rpc.DownloadP
67105
}
68106

69107
// Check the signature if needed
70-
var signaturePath, tmpSignaturePath *paths.Path
71108
if res.SignatureURL != nil {
72109
// Compose signature URL
73110
signatureFileName := path.Base(res.SignatureURL.Path)
@@ -79,6 +116,10 @@ func (res *IndexResource) Download(destDir *paths.Path, downloadCB rpc.DownloadP
79116
return &arduino.FailedDownloadError{Message: tr("Error downloading index signature '%s'", res.SignatureURL), Cause: err}
80117
}
81118

119+
hasSignature = true
120+
}
121+
122+
if hasSignature {
82123
// Check signature...
83124
if valid, _, err := security.VerifyArduinoDetachedSignature(tmpIndexPath, tmpSignaturePath); err != nil {
84125
return &arduino.PermissionDeniedError{Message: tr("Error verifying signature"), Cause: err}
@@ -109,12 +150,12 @@ func (res *IndexResource) Download(destDir *paths.Path, downloadCB rpc.DownloadP
109150
if err := tmpIndexPath.CopyTo(indexPath); err != nil {
110151
return &arduino.PermissionDeniedError{Message: tr("Error saving downloaded index"), Cause: err}
111152
}
112-
if res.SignatureURL != nil {
153+
if hasSignature {
113154
if err := tmpSignaturePath.CopyTo(signaturePath); err != nil {
114155
return &arduino.PermissionDeniedError{Message: tr("Error saving downloaded index signature"), Cause: err}
115156
}
116157
}
117-
oldIndex.Remove()
118-
oldSignature.Remove()
158+
_ = oldIndex.Remove()
159+
_ = oldSignature.Remove()
119160
return nil
120161
}

‎arduino/resources/resources_test.go‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@ package resources
1818
import (
1919
"crypto"
2020
"encoding/hex"
21+
"net"
22+
"net/http"
23+
"net/url"
2124
"testing"
2225

2326
rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
@@ -110,3 +113,38 @@ func TestDownloadAndChecksums(t *testing.T) {
110113
_, err = r.TestLocalArchiveChecksum(tmp)
111114
require.Error(t, err)
112115
}
116+
117+
func TestIndexDownloadAndSignatureWithinArchive(t *testing.T) {
118+
// Spawn test webserver
119+
mux := http.NewServeMux()
120+
fs := http.FileServer(http.Dir("testdata"))
121+
mux.Handle("/", fs)
122+
server := &http.Server{Handler: mux}
123+
ln, err := net.Listen("tcp", "127.0.0.1:")
124+
require.NoError(t, err)
125+
defer ln.Close()
126+
go server.Serve(ln)
127+
128+
validIdxURL, err := url.Parse("http://" + ln.Addr().String() + "/valid/package_index.tar.bz2")
129+
require.NoError(t, err)
130+
idxResource := &IndexResource{URL: validIdxURL}
131+
destDir, err := paths.MkTempDir("", "")
132+
require.NoError(t, err)
133+
defer destDir.RemoveAll()
134+
err = idxResource.Download(destDir, func(curr *rpc.DownloadProgress) {})
135+
require.NoError(t, err)
136+
require.True(t, destDir.Join("package_index.json").Exist())
137+
require.True(t, destDir.Join("package_index.json.sig").Exist())
138+
139+
invalidIdxURL, err := url.Parse("http://" + ln.Addr().String() + "/invalid/package_index.tar.bz2")
140+
require.NoError(t, err)
141+
invIdxResource := &IndexResource{URL: invalidIdxURL}
142+
invDestDir, err := paths.MkTempDir("", "")
143+
require.NoError(t, err)
144+
defer invDestDir.RemoveAll()
145+
err = invIdxResource.Download(invDestDir, func(curr *rpc.DownloadProgress) {})
146+
require.Error(t, err)
147+
require.Contains(t, err.Error(), "invalid signature")
148+
require.False(t, invDestDir.Join("package_index.json").Exist())
149+
require.False(t, invDestDir.Join("package_index.json.sig").Exist())
150+
}

‎arduino/resources/testdata/invalid/package_index.tar.bz2‎

39.5 KB
Binary file not shown.

‎arduino/resources/testdata/valid/package_index.tar.bz2‎

39.5 KB
Binary file not shown.

‎cli/core/search.go‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -134,10 +134,9 @@ func indexesNeedUpdating(duration string) bool {
134134

135135
now := time.Now()
136136
modTimeThreshold, err := time.ParseDuration(duration)
137-
// Not the most elegant way of handling this error
138-
// but it does its job
139137
if err != nil {
140-
modTimeThreshold, _ = time.ParseDuration("24h")
138+
feedback.Error(tr("Invalid timeout: %s", err))
139+
os.Exit(errorcodes.ErrBadArgument)
141140
}
142141

143142
urls := []string{globals.DefaultIndexURL}
@@ -153,7 +152,18 @@ func indexesNeedUpdating(duration string) bool {
153152
continue
154153
}
155154

156-
coreIndexPath := indexpath.Join(path.Base(URL.Path))
155+
// should handle:
156+
// - package_index.json
157+
// - package_index.json.sig
158+
// - package_index.json.gz
159+
// - package_index.tar.bz2
160+
indexFileName := path.Base(URL.Path)
161+
indexFileName = strings.TrimSuffix(indexFileName, ".tar.bz2")
162+
indexFileName = strings.TrimSuffix(indexFileName, ".gz")
163+
indexFileName = strings.TrimSuffix(indexFileName, ".sig")
164+
indexFileName = strings.TrimSuffix(indexFileName, ".json")
165+
// and obtain package_index.json as result
166+
coreIndexPath := indexpath.Join(indexFileName + ".json")
157167
if coreIndexPath.NotExist() {
158168
return true
159169
}

‎cli/globals/globals.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,5 @@ var (
2626
// VersionInfo contains all info injected during build
2727
VersionInfo = version.NewInfo(filepath.Base(os.Args[0]))
2828
// DefaultIndexURL is the default index url
29-
DefaultIndexURL = "https://downloads.arduino.cc/packages/package_index.json"
29+
DefaultIndexURL = "https://downloads.arduino.cc/packages/package_index.tar.bz2"
3030
)

‎commands/instances.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -434,7 +434,7 @@ func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexRequest, downloadCB rp
434434
indexResource := resources.IndexResource{
435435
URL: URL,
436436
}
437-
if strings.HasSuffix(URL.Host, "arduino.cc") {
437+
if strings.HasSuffix(URL.Host, "arduino.cc") && strings.HasSuffix(URL.Path, ".json") {
438438
indexResource.SignatureURL, _ = url.Parse(u) // should not fail because we already parsed it
439439
indexResource.SignatureURL.Path += ".sig"
440440
}

‎configuration/network.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,10 @@ import (
2626

2727
// UserAgent returns the user agent (mainly used by HTTP clients)
2828
func UserAgent(settings *viper.Viper) string {
29-
subComponent := settings.GetString("network.user_agent_ext")
29+
subComponent := ""
30+
if settings != nil {
31+
subComponent = settings.GetString("network.user_agent_ext")
32+
}
3033
if subComponent != "" {
3134
subComponent = " " + subComponent
3235
}
@@ -41,7 +44,7 @@ func UserAgent(settings *viper.Viper) string {
4144

4245
// NetworkProxy returns the proxy configuration (mainly used by HTTP clients)
4346
func NetworkProxy(settings *viper.Viper) (*url.URL, error) {
44-
if !settings.IsSet("network.proxy") {
47+
if settings == nil || !settings.IsSet("network.proxy") {
4548
return nil, nil
4649
}
4750
if proxyConfig := settings.GetString("network.proxy"); proxyConfig == "" {

‎test/test_board.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
"official": true,
3131
"package": {
3232
"maintainer": "Arduino",
33-
"url": "https://downloads.arduino.cc/packages/package_index.json",
33+
"url": "https://downloads.arduino.cc/packages/package_index.tar.bz2",
3434
"website_url": "http://www.arduino.cc/",
3535
"email": "packages@arduino.cc",
3636
"name": "arduino",

0 commit comments

Comments
 (0)