Skip to content

Add azure.storage.endpoint config option #6596

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+39 −28
Open

Add azure.storage.endpoint config option #6596

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/reference/config.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -473,6 +473,11 @@ The following settings are available:
`azure.storage.accountName`
: The blob storage account name. Defaults to environment variable `AZURE_STORAGE_ACCOUNT_NAME`.

`azure.storage.endpoint`
: :::{versionadded} 25.11.0-edge
:::
: The blob storage service endpoint e.g. `https://nfbatch1.blob.core.windows.net`.

`azure.storage.fileShares.<name>.mountOptions`
: The file share mount options.

Expand Down
20 changes: 4 additions & 16 deletions plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzHelper.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,17 +169,11 @@ class AzHelper {
return client.generateAccountSas(signature)
}

static String generateAccountSas(String accountName, String accountKey, Duration duration) {
final client = getOrCreateBlobServiceWithKey(accountName, accountKey)
return generateAccountSas(client, duration)
}

@Memoized
static synchronized BlobServiceClient getOrCreateBlobServiceWithKey(String accountName, String accountKey) {
static synchronized BlobServiceClient getOrCreateBlobServiceWithKey(String accountName, String accountKey, String endpoint) {
log.debug "Creating Azure blob storage client -- accountName=$accountName; accountKey=${accountKey?.substring(0,5)}.."

final credential = new StorageSharedKeyCredential(accountName, accountKey)
final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName)

return new BlobServiceClientBuilder()
.endpoint(endpoint)
Expand All @@ -189,16 +183,14 @@ class AzHelper {
}

@Memoized
static synchronized BlobServiceClient getOrCreateBlobServiceWithToken(String accountName, String sasToken) {
static synchronized BlobServiceClient getOrCreateBlobServiceWithToken(String accountName, String sasToken, String endpoint) {
if( !sasToken )
throw new IllegalArgumentException("Missing Azure blob SAS token")
if( sasToken.length()<100 )
throw new IllegalArgumentException("Invalid Azure blob SAS token -- offending value: $sasToken")

log.debug "Creating Azure blob storage client -- accountName: $accountName; sasToken: ${sasToken?.substring(0,10)}.."

final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName)

return new BlobServiceClientBuilder()
.endpoint(endpoint)
.sasToken(sasToken)
Expand All @@ -207,11 +199,9 @@ class AzHelper {
}

@Memoized
static synchronized BlobServiceClient getOrCreateBlobServiceWithServicePrincipal(String accountName, String clientId, String clientSecret, String tenantId) {
static synchronized BlobServiceClient getOrCreateBlobServiceWithServicePrincipal(String accountName, String clientId, String clientSecret, String tenantId, String endpoint) {
log.debug "Creating Azure Blob storage client using Service Principal credentials"

final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName)

final credential = new ClientSecretCredentialBuilder()
.clientId(clientId)
.clientSecret(clientSecret)
Expand All @@ -226,11 +216,9 @@ class AzHelper {
}

@Memoized
static synchronized BlobServiceClient getOrCreateBlobServiceWithManagedIdentity(String accountName, String clientId) {
static synchronized BlobServiceClient getOrCreateBlobServiceWithManagedIdentity(String accountName, String clientId, String endpoint) {
log.debug "Creating Azure blob storage client using Managed Identity ${clientId ?: '<system-assigned identity>'}"

final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName)

final credentialBuilder = new ManagedIdentityCredentialBuilder()
if( clientId )
credentialBuilder.clientId(clientId)
Expand Down
16 changes: 16 additions & 0 deletions plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzStorageOpts.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@ class AzStorageOpts implements ConfigScope {
""")
final String accountName

@ConfigOption
@Description("""
The blob storage service endpoint e.g. `https://nfbatch1.blob.core.windows.net`.
""")
final String endpoint

@ConfigOption
@Description("""
The blob storage shared access signature (SAS) token, which can be provided instead of an account key. Defaults to environment variable `AZURE_STORAGE_SAS_TOKEN`.
Expand All @@ -65,6 +71,7 @@ class AzStorageOpts implements ConfigScope {
assert config!=null
this.accountKey = config.accountKey ?: env.get('AZURE_STORAGE_ACCOUNT_KEY')
this.accountName = config.accountName ?: env.get('AZURE_STORAGE_ACCOUNT_NAME')
this.endpoint = parseEndpoint(config.endpoint as String, accountName)
this.sasToken = config.sasToken ?: env.get('AZURE_STORAGE_SAS_TOKEN')
this.tokenDuration = (config.tokenDuration as Duration) ?: Duration.of('48h')
this.fileShares = parseFileShares(config.fileShares instanceof Map ? config.fileShares as Map<String, Map>
Expand All @@ -76,10 +83,19 @@ class AzStorageOpts implements ConfigScope {
Map<String, Object> props = new HashMap<>();
props.put(AzFileSystemProvider.AZURE_STORAGE_ACCOUNT_KEY, accountKey)
props.put(AzFileSystemProvider.AZURE_STORAGE_ACCOUNT_NAME, accountName)
props.put(AzFileSystemProvider.AZURE_STORAGE_ENDPOINT, endpoint)
props.put(AzFileSystemProvider.AZURE_STORAGE_SAS_TOKEN, sasToken)
return props
}

static String parseEndpoint(String endpoint, String accountName) {
if( endpoint )
return endpoint
if( accountName )
return String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName)
return null
}

static Map<String,AzFileShareOpts> parseFileShares(Map<String,Map> shares) {
final result = new LinkedHashMap<String,AzFileShareOpts>()
shares.each { Map.Entry<String, Map> entry ->
Expand Down
26 changes: 14 additions & 12 deletions plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileSystemProvider.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ class AzFileSystemProvider extends FileSystemProvider {

public static final String AZURE_STORAGE_ACCOUNT_NAME = 'AZURE_STORAGE_ACCOUNT_NAME'
public static final String AZURE_STORAGE_ACCOUNT_KEY = 'AZURE_STORAGE_ACCOUNT_KEY'
public static final String AZURE_STORAGE_ENDPOINT = 'AZURE_STORAGE_ENDPOINT'
public static final String AZURE_STORAGE_SAS_TOKEN = 'AZURE_STORAGE_SAS_TOKEN'

public static final String AZURE_CLIENT_ID = 'AZURE_CLIENT_ID'
Expand Down Expand Up @@ -110,20 +111,20 @@ class AzFileSystemProvider extends FileSystemProvider {
return uri.authority.toLowerCase()
}

protected BlobServiceClient createBlobServiceWithKey(String accountName, String accountKey) {
AzHelper.getOrCreateBlobServiceWithKey(accountName, accountKey)
protected BlobServiceClient createBlobServiceWithKey(String accountName, String accountKey, String endpoint) {
AzHelper.getOrCreateBlobServiceWithKey(accountName, accountKey, endpoint)
}

protected BlobServiceClient createBlobServiceWithToken(String accountName, String sasToken) {
AzHelper.getOrCreateBlobServiceWithToken(accountName, sasToken)
protected BlobServiceClient createBlobServiceWithToken(String accountName, String sasToken, String endpoint) {
AzHelper.getOrCreateBlobServiceWithToken(accountName, sasToken, endpoint)
}

protected BlobServiceClient createBlobServiceWithServicePrincipal(String accountName, String clientId, String clientSecret, String tenantId) {
AzHelper.getOrCreateBlobServiceWithServicePrincipal(accountName, clientId, clientSecret, tenantId)
protected BlobServiceClient createBlobServiceWithServicePrincipal(String accountName, String clientId, String clientSecret, String tenantId, String endpoint) {
AzHelper.getOrCreateBlobServiceWithServicePrincipal(accountName, clientId, clientSecret, tenantId, endpoint)
}

protected BlobServiceClient createBlobServiceWithManagedIdentity(String accountName, String clientId) {
AzHelper.getOrCreateBlobServiceWithManagedIdentity(accountName, clientId)
protected BlobServiceClient createBlobServiceWithManagedIdentity(String accountName, String clientId, String endpoint) {
AzHelper.getOrCreateBlobServiceWithManagedIdentity(accountName, clientId, endpoint)
}

/**
Expand Down Expand Up @@ -190,6 +191,7 @@ class AzFileSystemProvider extends FileSystemProvider {

final accountName = config.get(AZURE_STORAGE_ACCOUNT_NAME) as String
final accountKey = config.get(AZURE_STORAGE_ACCOUNT_KEY) as String
final endpoint = config.get(AZURE_STORAGE_ENDPOINT) as String
final sasToken = config.get(AZURE_STORAGE_SAS_TOKEN) as String

final servicePrincipalId = config.get(AZURE_CLIENT_ID) as String
Expand All @@ -205,17 +207,17 @@ class AzFileSystemProvider extends FileSystemProvider {
BlobServiceClient client

if( managedIdentityUser || managedIdentitySystem ) {
client = createBlobServiceWithManagedIdentity(accountName, managedIdentityUser)
client = createBlobServiceWithManagedIdentity(accountName, managedIdentityUser, endpoint)
}
else if( servicePrincipalSecret && servicePrincipalId && tenantId ) {
client = createBlobServiceWithServicePrincipal(accountName, servicePrincipalId, servicePrincipalSecret, tenantId)
client = createBlobServiceWithServicePrincipal(accountName, servicePrincipalId, servicePrincipalSecret, tenantId, endpoint)
}
else if( sasToken ) {
client = createBlobServiceWithToken(accountName, sasToken)
client = createBlobServiceWithToken(accountName, sasToken, endpoint)
this.sasToken = sasToken
}
else if( accountKey ) {
client = createBlobServiceWithKey(accountName, accountKey)
client = createBlobServiceWithKey(accountName, accountKey, endpoint)
this.accountKey = accountKey
}
else {
Expand Down
Loading