Releasing version 65.50.0

Releasing version 65.50.0

Author: oci-dex-release-bot

CHANGELOG.md

@@ -4,6 +4,38 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 
+## 65.50.0 - 2023-10-17
+### Added
+- Support for the Caching Service
+- Support for the Marketplace Publisher service
+- Support for higher limits for network firewalls in the Network Firewall service
+- Support for exporting access request reports in the Lockbox service
+- Support for storage mounts for jobs and notebooks in the Data Science service
+- Support for unified agent operational metrics for the service configurations in the Logging Management service
+- Support for Dynamic refresh for custom certs
+
+### Breaking Changes
+- The properties `DisplayName` and `RqsType` were removed in the `Parameter` model in the Logging Management service
+- The enum members `EnumString` and `RqsFilter` were remoeved from the `mappingParameterTypeEnum` in the `Parameter` model in the Logging Management service
+- The property `ServiceStage` was removed in the `ListServicesRequest` model in the Logging Management service
+- The models `TcpApplication` and `UdpApplication` were removed in the Network Firewall service
+- The type `DecryptionProfileTypeEnum` was removed in the model `DecryptionProfile` in the Network Firewall service
+- The properties `MappedSecrets`, `ApplicationLists`, `UrlLists`, `IpAddressLists`, `SecurityRules`, `DecryptionRules` and `DecryptionProfiles` were removed in the model `CreateNetworkFirewallPolicyDetails` in the Network Firewall service
+- The type `DecryptionRuleActionEnum` was removed in the model `DecryptionRule` in the Network Firewall service
+- The type of property `Action` was changed to `DecryptionActionTypeEnum` in the model `DecryptionRule` in the Network Firewall service
+- The property `Sources` has been replaced by `SourceAddress` in the models `SecurityRuleMatchCriteria` and `DecryptionRuleMatchCriteria` in the Network Firewall service
+- The property `Destinations` has been replaced by `DestinationAddress` in the models `SecurityRuleMatchCriteria` and `DecryptionRuleMatchCriteria` in the Network Firewall service
+- The type `MappedSecretTypeEnum` was removed in the model `MappedSecret` in the Network Firewall service
+- The type of property `Type` was changed to `InspectionType` in the model `MappedSecret` in the Network Firewall service
+- The properties `ApplicationLists`, `UrlLists`, `IpAddressLists`, `SecurityRules`, `DecryptionRules`, `DecryptionProfiles`, `MappedSecrets` and `IsFirewallAttached` were removed in the model `NetworkFirewallPolicy` in the Network Firewall service
+- The types `SecurityRuleActionEnum` and `SecurityRuleInspectionEnum` were removed in the model `SecurityRule` in the Network Firewall service
+- The type of property `Action` was changed to `TrafficActionTypeEnum` in the model `SecurityRule` in the Network Firewall service
+- The type of property `Inspection` was changed to `TrafficInspectionTypeEnum` in the model `SecurityRule` in the Network Firewall service
+- The property `Applications` has been replaced by `Application` in the model `SecurityRuleMatchCriteria` in the Network Firewall service
+- The property `Urls` has been replaced by `Url` in the model `SecurityRuleMatchCriteria` in the Network Firewall service
+- The properties `MappedSecrets`, `ApplicationLists`, `UrlLists`, `IpAddressLists`, `SecurityRules`, `DecryptionRules` and `DecryptionProfiles` were removed in the model `UpdateNetworkFirewallPolicyDetails` in the Network Firewall service
+
+
 ## 65.49.4 - 2023-10-10
 n### Added
 - Support for creating flow log type capture filters in the Virtual Cloud Network service

Makefile

@@ -1,6 +1,6 @@
 DOC_SERVER_URL=https:\/\/docs.cloud.oracle.com
 
-GEN_TARGETS = identity core objectstorage loadbalancer database audit dns filestorage email containerengine resourcesearch keymanagement announcementsservice healthchecks waas autoscaling streaming ons monitoring resourcemanager budget workrequests functions limits events dts oce oda analytics integration osmanagement marketplace apigateway applicationmigration datacatalog dataflow datascience nosql secrets vault bds cims datasafe mysql dataintegration ocvp usageapi blockchain loggingingestion logging loganalytics managementdashboard sch loggingsearch managementagent cloudguard opsi computeinstanceagent optimizer tenantmanagercontrolplane rover databasemanagement artifacts apmsynthetics goldengate apmcontrolplane apmtraces networkloadbalancer vulnerabilityscanning databasemigration servicecatalog ailanguage operatoraccesscontrol bastion genericartifactscontent jms devops aianomalydetection datalabelingservice datalabelingservicedataplane apmconfig waf certificates certificatesmanagement usage databasetools servicemanagerproxy appmgmtcontrol ospgateway identitydataplane visualbuilder osubusage osubsubscription osuborganizationsubscription osubbillingschedule dashboardservice threatintelligence aivision aispeech stackmonitoring servicemesh adm licensemanager onesubscription governancerulescontrolplane waa networkfirewall vnmonitoring emwarehouse lockbox fusionapps mediaservices opa opensearch cloudmigrations cloudbridge disasterrecovery containerinstances aidocument queue recovery vbsinst identitydomains accessgovernancecp ocicontrolcenter osmanagementhub fleetsoftwareupdate computecloudatcustomer ##SPECNAME##
+GEN_TARGETS = identity core objectstorage loadbalancer database audit dns filestorage email containerengine resourcesearch keymanagement announcementsservice healthchecks waas autoscaling streaming ons monitoring resourcemanager budget workrequests functions limits events dts oce oda analytics integration osmanagement marketplace apigateway applicationmigration datacatalog dataflow datascience nosql secrets vault bds cims datasafe mysql dataintegration ocvp usageapi blockchain loggingingestion logging loganalytics managementdashboard sch loggingsearch managementagent cloudguard opsi computeinstanceagent optimizer tenantmanagercontrolplane rover databasemanagement artifacts apmsynthetics goldengate apmcontrolplane apmtraces networkloadbalancer vulnerabilityscanning databasemigration servicecatalog ailanguage operatoraccesscontrol bastion genericartifactscontent jms devops aianomalydetection datalabelingservice datalabelingservicedataplane apmconfig waf certificates certificatesmanagement usage databasetools servicemanagerproxy appmgmtcontrol ospgateway identitydataplane visualbuilder osubusage osubsubscription osuborganizationsubscription osubbillingschedule dashboardservice threatintelligence aivision aispeech stackmonitoring servicemesh adm licensemanager onesubscription governancerulescontrolplane waa networkfirewall vnmonitoring emwarehouse lockbox fusionapps mediaservices opa opensearch cloudmigrations cloudbridge disasterrecovery containerinstances aidocument queue recovery vbsinst identitydomains accessgovernancecp ocicontrolcenter osmanagementhub fleetsoftwareupdate computecloudatcustomer marketplacepublisher redis ##SPECNAME##
 NON_GEN_TARGETS = common common/auth objectstorage/transfer example
 TARGETS = $(NON_GEN_TARGETS) $(GEN_TARGETS)
 

common/client.go

@@ -7,13 +7,10 @@ package common
 import (
 	"bytes"
 	"context"
-	"crypto/tls"
-	"crypto/x509"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"math/rand"
-	"net"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
@@ -23,6 +20,7 @@ import (
 	"path/filepath"
 	"reflect"
 	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -103,13 +101,36 @@ const (
 	//circuitBreakerNumberOfHistoryResponseEnv is the number of recorded history responses
 	circuitBreakerNumberOfHistoryResponseEnv = "OCI_SDK_CIRCUITBREAKER_NUM_HISTORY_RESPONSE"
 
+	// ociDefaultRefreshIntervalForCustomCerts is the env var for overriding the defaultRefreshIntervalForCustomCerts.
+	// The value represents the refresh interval in minutes and has a higher precedence than defaultRefreshIntervalForCustomCerts
+	// but has a lower precedence then the refresh interval configured via OciGlobalRefreshIntervalForCustomCerts
+	// If the value is negative, then it is assumed that this property is not configured
+	// if the value is Zero, then the refresh of custom certs will be disabled
+	ociDefaultRefreshIntervalForCustomCerts = "OCI_DEFAULT_REFRESH_INTERVAL_FOR_CUSTOM_CERTS"
+
 	// ociDefaultCertsPath is the env var for the path to the SSL cert file
 	ociDefaultCertsPath = "OCI_DEFAULT_CERTS_PATH"
 
+	// ociDefaultClientCertsPath is the env var for the path to the custom client cert
+	ociDefaultClientCertsPath = "OCI_DEFAULT_CLIENT_CERTS_PATH"
+
+	// ociDefaultClientCertsPrivateKeyPath is the env var for the path to the custom client cert private key
+	ociDefaultClientCertsPrivateKeyPath = "OCI_DEFAULT_CLIENT_CERTS_PRIVATE_KEY_PATH"
+
 	//maxAttemptsForRefreshableRetry is the number of retry when 401 happened on a refreshable auth type
 	maxAttemptsForRefreshableRetry = 3
+
+	//defaultRefreshIntervalForCustomCerts is the default refresh interval in minutes
+	defaultRefreshIntervalForCustomCerts = 30
 )
 
+// OciGlobalRefreshIntervalForCustomCerts is the global policy for overriding the refresh interval in minutes.
+// This variable has a higher precedence than the env variable OCI_DEFAULT_REFRESH_INTERVAL_FOR_CUSTOM_CERTS
+// and the defaultRefreshIntervalForCustomCerts values.
+// If the value is negative, then it is assumed that this property is not configured
+// if the value is Zero, then the refresh of custom certs will be disabled
+var OciGlobalRefreshIntervalForCustomCerts int = -1
+
 // RequestInterceptor function used to customize the request before calling the underlying service
 type RequestInterceptor func(*http.Request) error
 
@@ -213,32 +234,13 @@ func newBaseClient(signer HTTPRequestSigner, dispatcher HTTPRequestDispatcher) B
 
 func defaultHTTPDispatcher() http.Client {
 	var httpClient http.Client
-	var tp = http.DefaultTransport.(*http.Transport)
-	if isExpectHeaderDisabled := IsEnvVarFalse(UsingExpectHeaderEnvVar); !isExpectHeaderDisabled {
-		tp.Proxy = http.ProxyFromEnvironment
-		tp.DialContext = (&net.Dialer{
-			Timeout:   30 * time.Second,
-			KeepAlive: 30 * time.Second,
-			DualStack: true,
-		}).DialContext
-		tp.ForceAttemptHTTP2 = true
-		tp.MaxIdleConns = 100
-		tp.IdleConnTimeout = 90 * time.Second
-		tp.TLSHandshakeTimeout = 10 * time.Second
-		tp.ExpectContinueTimeout = 3 * time.Second
-	}
-	if certFile, ok := os.LookupEnv(ociDefaultCertsPath); ok {
-		pool := x509.NewCertPool()
-		pemCert := readCertPem(certFile)
-		cert, err := x509.ParseCertificate(pemCert)
-		if err != nil {
-			Logf("unable to parse content to cert fallback to pem format from env var value: %s", certFile)
-			pool.AppendCertsFromPEM(pemCert)
-		} else {
-			Logf("using custom cert parsed from env var value: %s", certFile)
-			pool.AddCert(cert)
-		}
-		tp.TLSClientConfig = &tls.Config{RootCAs: pool}
+	refreshInterval := getCustomCertRefreshInterval()
+	if refreshInterval <= 0 {
+		Debug("Custom cert refresh has been disabled")
+	}
+	var tp = &OciHTTPTransportWrapper{
+		RefreshRate:       time.Duration(refreshInterval) * time.Minute,
+		TLSConfigProvider: GetTLSConfigTemplateForTransport(),
 	}
 	httpClient = http.Client{
 		Timeout:   defaultTimeout,
@@ -731,3 +733,21 @@ func (client BaseClient) IsOciRealmSpecificServiceEndpointTemplateEnabled() bool
 	}
 	return IsEnvVarTrue(OciRealmSpecificServiceEndpointTemplateEnabledEnvVar)
 }
+
+func getCustomCertRefreshInterval() int {
+	if OciGlobalRefreshIntervalForCustomCerts >= 0 {
+		Debugf("Setting refresh interval as %d for custom certs via OciGlobalRefreshIntervalForCustomCerts", OciGlobalRefreshIntervalForCustomCerts)
+		return OciGlobalRefreshIntervalForCustomCerts
+	}
+	if refreshIntervalValue, ok := os.LookupEnv(ociDefaultRefreshIntervalForCustomCerts); ok {
+		refreshInterval, err := strconv.Atoi(refreshIntervalValue)
+		if err != nil || refreshInterval < 0 {
+			Debugf("The environment variable %s is not a valid int or is a negative value, skipping this configuration", ociDefaultRefreshIntervalForCustomCerts)
+		} else {
+			Debugf("Setting refresh interval as %d for custom certs via the env variable %s", refreshInterval, ociDefaultRefreshIntervalForCustomCerts)
+			return refreshInterval
+		}
+	}
+	Debugf("Setting the default refresh interval %d for custom certs", defaultRefreshIntervalForCustomCerts)
+	return defaultRefreshIntervalForCustomCerts
+}

common/configuration_test.go

@@ -1051,11 +1051,11 @@ func TestExpandPath(t *testing.T) {
 			inPath:       "~/somepath",
 			expectedPath: filepath.Join(home, "somepath"),
 		},
-		{
-			name:         "should not do anything",
-			inPath:       "/somepath/some/dir/~/file",
-			expectedPath: "/somepath/some/dir/~/file",
-		},
+		// {  // This test case fails onm Windows image tests, as all instances of '/' are replaced with the windows path seperator '\'
+		// 	name:         "should not do anything",
+		// 	inPath:       "/somepath/some/dir/~/file",
+		// 	expectedPath: "/somepath/some/dir/~/file",
+		// },
 		{
 			name:         "should replace one tilde only",
 			inPath:       "~/~/some/path",

common/errors_test.go

@@ -7,15 +7,12 @@ import (
 	"bytes"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"net"
 	"net/http"
 	"net/url"
-	"os"
 	"strings"
 	"syscall"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -126,50 +123,3 @@ func TestNetworkErrors(t *testing.T) {
 	assert.Equal(t, valid, true)
 
 }
-
-func TestConnectionReset(t *testing.T) {
-	go server()
-
-	time.Sleep(3 * time.Second) // wait for server to run
-
-	conn, err := net.Dial("tcp", "localhost:8080")
-	if err != nil {
-		log.Fatal("client", err)
-	}
-
-	if _, err := conn.Write([]byte("ab")); err != nil {
-		log.Printf("client: %v", err)
-	}
-
-	time.Sleep(1 * time.Second) // wait for close on the server side
-
-	data := make([]byte, 1)
-
-	_, resetErr := conn.Read(data)
-
-	success := IsNetworkError(resetErr)
-
-	assert.Equal(t, success, true)
-
-}
-
-func server() {
-	listener, err := net.Listen("tcp", ":8080")
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	defer listener.Close()
-
-	conn, err := listener.Accept()
-	if err != nil {
-		log.Fatal("server", err)
-		os.Exit(1)
-	}
-	data := make([]byte, 1)
-	if _, err := conn.Read(data); err != nil {
-		log.Fatal("server", err)
-	}
-
-	conn.Close()
-}

common/helpers.go

@@ -9,7 +9,6 @@ import (
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
-	"io/ioutil"
 	"net/textproto"
 	"os"
 	"reflect"
@@ -296,12 +295,3 @@ func IsEnvVarTrue(envVarKey string) bool {
 	val, existed := os.LookupEnv(envVarKey)
 	return existed && strings.ToLower(val) == "true"
 }
-
-// Reads the certs from pem file pointed by the R1_CERT_PEM env variable
-func readCertPem(path string) []byte {
-	pem, err := ioutil.ReadFile(path)
-	if err != nil {
-		panic("can not read cert " + err.Error())
-	}
-	return pem
-}

common/oci_http_transport_wrapper.go

@@ -0,0 +1,120 @@
+// Copyright (c) 2016, 2018, 2023, Oracle and/or its affiliates.  All rights reserved.
+// This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license.
+
+package common
+
+import (
+	"fmt"
+	"net/http"
+	"sync"
+	"time"
+)
+
+// OciHTTPTransportWrapper is a http.RoundTripper that periodically refreshes
+// the underlying http.Transport according to its templates.
+// Upon the first use (or once the RefreshRate duration is elapsed),
+// a new transport will be created from the TransportTemplate (if set).
+type OciHTTPTransportWrapper struct {
+	// RefreshRate specifies the duration at which http.Transport
+	// (with its tls.Config) must be refreshed.
+	// Defaults to 5 minutes.
+	RefreshRate time.Duration
+
+	// TLSConfigProvider creates a new tls.Config.
+	// If not set, nil tls.Config is returned.
+	TLSConfigProvider TLSConfigProvider
+
+	// ClientTemplate is responsible for creating a new http.Client with
+	// a given tls.Config.
+	//
+	// If not set, a new http.Client with a cloned http.DefaultTransport is returned.
+	TransportTemplate TransportTemplateProvider
+
+	// mutable properties
+	mux             sync.RWMutex
+	lastRefreshedAt time.Time
+	delegate        http.RoundTripper
+}
+
+// RoundTrip implements http.RoundTripper.
+func (t *OciHTTPTransportWrapper) RoundTrip(req *http.Request) (*http.Response, error) {
+	delegate, err := t.refreshDelegate(false /* force */)
+	if err != nil {
+		return nil, err
+	}
+
+	return delegate.RoundTrip(req)
+}
+
+// Refresh forces refresh of the underlying delegate.
+func (t *OciHTTPTransportWrapper) Refresh(force bool) error {
+	_, err := t.refreshDelegate(force)
+	return err
+}
+
+// Delegate returns the currently active http.RoundTripper.
+// Might be nil.
+func (t *OciHTTPTransportWrapper) Delegate() http.RoundTripper {
+	t.mux.RLock()
+	defer t.mux.RUnlock()
+
+	return t.delegate
+}
+
+// refreshDelegate refreshes the delegate (and its TLS config) if:
+//   - force is true
+//   - it's been more than RefreshRate since the last time the client was refreshed.
+func (t *OciHTTPTransportWrapper) refreshDelegate(force bool) (http.RoundTripper, error) {
+	// read-lock first, since it's cheaper than write lock
+	t.mux.RLock()
+	if !t.shouldRefreshLocked(force) {
+		delegate := t.delegate
+		t.mux.RUnlock()
+
+		return delegate, nil
+	}
+
+	// upgrade to write-lock, and we'll need to check again for the same condition as above
+	// to avoid multiple initializations by multiple "refresher" goroutines
+	t.mux.RUnlock()
+	t.mux.Lock()
+	defer t.mux.Unlock()
+	if !t.shouldRefreshLocked(force) {
+		return t.delegate, nil
+	}
+
+	// For this check we need the delegate to be set once before we check for change in cert files
+	if t.delegate != nil && !t.TLSConfigProvider.WatchedFilesModified() {
+		Debug("No modification in custom certs or ca bundle skipping refresh")
+		// Updating the last refresh time to make sure the next check is only done after the refresh interval has passed
+		t.lastRefreshedAt = time.Now()
+		return t.delegate, nil
+	}
+
+	Logf("Loading tls config from TLSConfigProvider")
+	tlsConfig, err := t.TLSConfigProvider.NewOrDefault()
+	if err != nil {
+		return nil, fmt.Errorf("refreshing tls.Config from template: %w", err)
+	}
+
+	t.delegate, err = t.TransportTemplate.NewOrDefault(tlsConfig)
+	if err != nil {
+		return nil, fmt.Errorf("refreshing http.RoundTripper from template: %w", err)
+	}
+
+	t.lastRefreshedAt = time.Now()
+	return t.delegate, nil
+}
+
+// shouldRefreshLocked returns whether the client (and its TLS config)
+// needs to be refreshed.
+func (t *OciHTTPTransportWrapper) shouldRefreshLocked(force bool) bool {
+	if force || t.delegate == nil {
+		return true
+	}
+	return t.refreshRate() > 0 && time.Since(t.lastRefreshedAt) > t.refreshRate()
+}
+
+func (t *OciHTTPTransportWrapper) refreshRate() time.Duration {
+	return t.RefreshRate
+}