diff --git a/docs/reference/config.md b/docs/reference/config.md index f8e678fa45..aea2d0a09c 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -473,6 +473,11 @@ The following settings are available: `azure.storage.accountName` : The blob storage account name. Defaults to environment variable `AZURE_STORAGE_ACCOUNT_NAME`. +`azure.storage.endpoint` +: :::{versionadded} 25.11.0-edge + ::: +: The blob storage service endpoint e.g. `https://nfbatch1.blob.core.windows.net`. + `azure.storage.fileShares..mountOptions` : The file share mount options. diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzHelper.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzHelper.groovy index ba4ba6cd71..ebc62ebfc6 100644 --- a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzHelper.groovy +++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzHelper.groovy @@ -169,17 +169,11 @@ class AzHelper { return client.generateAccountSas(signature) } - static String generateAccountSas(String accountName, String accountKey, Duration duration) { - final client = getOrCreateBlobServiceWithKey(accountName, accountKey) - return generateAccountSas(client, duration) - } - @Memoized - static synchronized BlobServiceClient getOrCreateBlobServiceWithKey(String accountName, String accountKey) { + static synchronized BlobServiceClient getOrCreateBlobServiceWithKey(String accountName, String accountKey, String endpoint) { log.debug "Creating Azure blob storage client -- accountName=$accountName; accountKey=${accountKey?.substring(0,5)}.." final credential = new StorageSharedKeyCredential(accountName, accountKey) - final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName) return new BlobServiceClientBuilder() .endpoint(endpoint) @@ -189,7 +183,7 @@ class AzHelper { } @Memoized - static synchronized BlobServiceClient getOrCreateBlobServiceWithToken(String accountName, String sasToken) { + static synchronized BlobServiceClient getOrCreateBlobServiceWithToken(String accountName, String sasToken, String endpoint) { if( !sasToken ) throw new IllegalArgumentException("Missing Azure blob SAS token") if( sasToken.length()<100 ) @@ -197,8 +191,6 @@ class AzHelper { log.debug "Creating Azure blob storage client -- accountName: $accountName; sasToken: ${sasToken?.substring(0,10)}.." - final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName) - return new BlobServiceClientBuilder() .endpoint(endpoint) .sasToken(sasToken) @@ -207,11 +199,9 @@ class AzHelper { } @Memoized - static synchronized BlobServiceClient getOrCreateBlobServiceWithServicePrincipal(String accountName, String clientId, String clientSecret, String tenantId) { + static synchronized BlobServiceClient getOrCreateBlobServiceWithServicePrincipal(String accountName, String clientId, String clientSecret, String tenantId, String endpoint) { log.debug "Creating Azure Blob storage client using Service Principal credentials" - final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName) - final credential = new ClientSecretCredentialBuilder() .clientId(clientId) .clientSecret(clientSecret) @@ -226,11 +216,9 @@ class AzHelper { } @Memoized - static synchronized BlobServiceClient getOrCreateBlobServiceWithManagedIdentity(String accountName, String clientId) { + static synchronized BlobServiceClient getOrCreateBlobServiceWithManagedIdentity(String accountName, String clientId, String endpoint) { log.debug "Creating Azure blob storage client using Managed Identity ${clientId ?: ''}" - final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName) - final credentialBuilder = new ManagedIdentityCredentialBuilder() if( clientId ) credentialBuilder.clientId(clientId) diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzStorageOpts.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzStorageOpts.groovy index e85e0b07ca..3409107662 100644 --- a/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzStorageOpts.groovy +++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzStorageOpts.groovy @@ -45,6 +45,12 @@ class AzStorageOpts implements ConfigScope { """) final String accountName + @ConfigOption + @Description(""" + The blob storage service endpoint e.g. `https://nfbatch1.blob.core.windows.net`. + """) + final String endpoint + @ConfigOption @Description(""" The blob storage shared access signature (SAS) token, which can be provided instead of an account key. Defaults to environment variable `AZURE_STORAGE_SAS_TOKEN`. @@ -65,6 +71,7 @@ class AzStorageOpts implements ConfigScope { assert config!=null this.accountKey = config.accountKey ?: env.get('AZURE_STORAGE_ACCOUNT_KEY') this.accountName = config.accountName ?: env.get('AZURE_STORAGE_ACCOUNT_NAME') + this.endpoint = parseEndpoint(config.endpoint as String, accountName) this.sasToken = config.sasToken ?: env.get('AZURE_STORAGE_SAS_TOKEN') this.tokenDuration = (config.tokenDuration as Duration) ?: Duration.of('48h') this.fileShares = parseFileShares(config.fileShares instanceof Map ? config.fileShares as Map @@ -76,10 +83,19 @@ class AzStorageOpts implements ConfigScope { Map props = new HashMap<>(); props.put(AzFileSystemProvider.AZURE_STORAGE_ACCOUNT_KEY, accountKey) props.put(AzFileSystemProvider.AZURE_STORAGE_ACCOUNT_NAME, accountName) + props.put(AzFileSystemProvider.AZURE_STORAGE_ENDPOINT, endpoint) props.put(AzFileSystemProvider.AZURE_STORAGE_SAS_TOKEN, sasToken) return props } + static String parseEndpoint(String endpoint, String accountName) { + if( endpoint ) + return endpoint + if( accountName ) + return String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName) + return null + } + static Map parseFileShares(Map shares) { final result = new LinkedHashMap() shares.each { Map.Entry entry -> diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileSystemProvider.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileSystemProvider.groovy index 69b6fefe8f..14d1314403 100644 --- a/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileSystemProvider.groovy +++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileSystemProvider.groovy @@ -54,6 +54,7 @@ class AzFileSystemProvider extends FileSystemProvider { public static final String AZURE_STORAGE_ACCOUNT_NAME = 'AZURE_STORAGE_ACCOUNT_NAME' public static final String AZURE_STORAGE_ACCOUNT_KEY = 'AZURE_STORAGE_ACCOUNT_KEY' + public static final String AZURE_STORAGE_ENDPOINT = 'AZURE_STORAGE_ENDPOINT' public static final String AZURE_STORAGE_SAS_TOKEN = 'AZURE_STORAGE_SAS_TOKEN' public static final String AZURE_CLIENT_ID = 'AZURE_CLIENT_ID' @@ -110,20 +111,20 @@ class AzFileSystemProvider extends FileSystemProvider { return uri.authority.toLowerCase() } - protected BlobServiceClient createBlobServiceWithKey(String accountName, String accountKey) { - AzHelper.getOrCreateBlobServiceWithKey(accountName, accountKey) + protected BlobServiceClient createBlobServiceWithKey(String accountName, String accountKey, String endpoint) { + AzHelper.getOrCreateBlobServiceWithKey(accountName, accountKey, endpoint) } - protected BlobServiceClient createBlobServiceWithToken(String accountName, String sasToken) { - AzHelper.getOrCreateBlobServiceWithToken(accountName, sasToken) + protected BlobServiceClient createBlobServiceWithToken(String accountName, String sasToken, String endpoint) { + AzHelper.getOrCreateBlobServiceWithToken(accountName, sasToken, endpoint) } - protected BlobServiceClient createBlobServiceWithServicePrincipal(String accountName, String clientId, String clientSecret, String tenantId) { - AzHelper.getOrCreateBlobServiceWithServicePrincipal(accountName, clientId, clientSecret, tenantId) + protected BlobServiceClient createBlobServiceWithServicePrincipal(String accountName, String clientId, String clientSecret, String tenantId, String endpoint) { + AzHelper.getOrCreateBlobServiceWithServicePrincipal(accountName, clientId, clientSecret, tenantId, endpoint) } - protected BlobServiceClient createBlobServiceWithManagedIdentity(String accountName, String clientId) { - AzHelper.getOrCreateBlobServiceWithManagedIdentity(accountName, clientId) + protected BlobServiceClient createBlobServiceWithManagedIdentity(String accountName, String clientId, String endpoint) { + AzHelper.getOrCreateBlobServiceWithManagedIdentity(accountName, clientId, endpoint) } /** @@ -190,6 +191,7 @@ class AzFileSystemProvider extends FileSystemProvider { final accountName = config.get(AZURE_STORAGE_ACCOUNT_NAME) as String final accountKey = config.get(AZURE_STORAGE_ACCOUNT_KEY) as String + final endpoint = config.get(AZURE_STORAGE_ENDPOINT) as String final sasToken = config.get(AZURE_STORAGE_SAS_TOKEN) as String final servicePrincipalId = config.get(AZURE_CLIENT_ID) as String @@ -205,17 +207,17 @@ class AzFileSystemProvider extends FileSystemProvider { BlobServiceClient client if( managedIdentityUser || managedIdentitySystem ) { - client = createBlobServiceWithManagedIdentity(accountName, managedIdentityUser) + client = createBlobServiceWithManagedIdentity(accountName, managedIdentityUser, endpoint) } else if( servicePrincipalSecret && servicePrincipalId && tenantId ) { - client = createBlobServiceWithServicePrincipal(accountName, servicePrincipalId, servicePrincipalSecret, tenantId) + client = createBlobServiceWithServicePrincipal(accountName, servicePrincipalId, servicePrincipalSecret, tenantId, endpoint) } else if( sasToken ) { - client = createBlobServiceWithToken(accountName, sasToken) + client = createBlobServiceWithToken(accountName, sasToken, endpoint) this.sasToken = sasToken } else if( accountKey ) { - client = createBlobServiceWithKey(accountName, accountKey) + client = createBlobServiceWithKey(accountName, accountKey, endpoint) this.accountKey = accountKey } else {