Skip to content

Commit 09f0066

Browse files
weirongw23-msftweirongw23-msft
authored
[Storage] Added Support for UseDevelopmentStorage=true; for Connection String Client Constructors (#43565)
1 parent b7dc156 commit 09f0066

File tree

18 files changed

+220
-41
lines changed

18 files changed

+220
-41
lines changed

‎sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
from .authentication import SharedKeyCredentialPolicy
3838
from .constants import CONNECTION_TIMEOUT, DEFAULT_OAUTH_SCOPE, READ_TIMEOUT, SERVICE_HOST_BASE, STORAGE_OAUTH_SCOPE
3939
from .models import LocationMode, StorageConfiguration
40+
from .parser import DEVSTORE_ACCOUNT_KEY, _get_development_storage_endpoint
4041
from .policies import (
4142
ExponentialRetry,
4243
QueueMessagePolicy,
@@ -407,6 +408,8 @@ def parse_connection_str(
407408
if any(len(tup) != 2 for tup in conn_settings_list):
408409
raise ValueError("Connection string is either blank or malformed.")
409410
conn_settings = dict((key.upper(), val) for key, val in conn_settings_list)
411+
if conn_settings.get('USEDEVELOPMENTSTORAGE') == 'true':
412+
return _get_development_storage_endpoint(service), None, DEVSTORE_ACCOUNT_KEY
410413
endpoints = _SERVICE_PARAMS[service]
411414
primary = None
412415
secondary = None

‎sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client_async.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
from .base_client import create_configuration
2727
from .constants import CONNECTION_TIMEOUT, DEFAULT_OAUTH_SCOPE, READ_TIMEOUT, SERVICE_HOST_BASE, STORAGE_OAUTH_SCOPE
2828
from .models import StorageConfiguration
29+
from .parser import DEVSTORE_ACCOUNT_KEY, _get_development_storage_endpoint
2930
from .policies import (
3031
QueueMessagePolicy,
3132
StorageContentValidation,
@@ -201,6 +202,8 @@ def parse_connection_str(
201202
if any(len(tup) != 2 for tup in conn_settings_list):
202203
raise ValueError("Connection string is either blank or malformed.")
203204
conn_settings = dict((key.upper(), val) for key, val in conn_settings_list)
205+
if conn_settings.get('USEDEVELOPMENTSTORAGE') == 'true':
206+
return _get_development_storage_endpoint(service), None, DEVSTORE_ACCOUNT_KEY
204207
endpoints = _SERVICE_PARAMS[service]
205208
primary = None
206209
secondary = None

‎sdk/storage/azure-storage-blob/azure/storage/blob/_shared/parser.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,14 @@
1010
EPOCH_AS_FILETIME = 116444736000000000 # January 1, 1970 as MS filetime
1111
HUNDREDS_OF_NANOSECONDS = 10000000
1212

13+
DEVSTORE_PORTS = {
14+
"blob": 10000,
15+
"dfs": 10000,
16+
"queue": 10001,
17+
}
18+
DEVSTORE_ACCOUNT_NAME = "devstoreaccount1"
19+
DEVSTORE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
20+
1321

1422
def _to_utc_datetime(value: datetime) -> str:
1523
return value.strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -51,3 +59,15 @@ def _filetime_to_datetime(filetime: str) -> Optional[datetime]:
5159

5260
# Try RFC 1123 as backup
5361
return _rfc_1123_to_datetime(filetime)
62+
63+
64+
def _get_development_storage_endpoint(service: str) -> str:
65+
"""Creates a development storage endpoint for Azurite Storage Emulator.
66+
67+
:param str service: The service name.
68+
:return: The development storage endpoint.
69+
:rtype: str
70+
"""
71+
if service.lower() not in DEVSTORE_PORTS:
72+
raise ValueError(f"Unsupported service name: {service}")
73+
return f"http://127.0.0.1:{DEVSTORE_PORTS[service]}/{DEVSTORE_ACCOUNT_NAME}"

‎sdk/storage/azure-storage-blob/tests/test_blob_client.py‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
VERSION,
2020
)
2121
from azure.storage.blob._shared.base_client import create_configuration
22+
from azure.storage.blob._shared.parser import DEVSTORE_ACCOUNT_KEY, DEVSTORE_ACCOUNT_NAME
2223

2324
from devtools_testutils import recorded_by_proxy
2425
from devtools_testutils.storage import StorageRecordedTestCase
@@ -101,6 +102,24 @@ def test_create_service_with_connection_string(self, **kwargs):
101102
self.validate_standard_account_endpoints(service, service_type[1], storage_account_name, storage_account_key)
102103
assert service.scheme == 'https'
103104

105+
@BlobPreparer()
106+
def test_create_service_use_development_storage(self):
107+
for service_type in SERVICES.items():
108+
# Act
109+
service = service_type[0].from_connection_string(
110+
"UseDevelopmentStorage=true;",
111+
container_name="test",
112+
blob_name="test"
113+
)
114+
115+
# Assert
116+
assert service is not None
117+
assert service.scheme == "http"
118+
assert service.account_name == DEVSTORE_ACCOUNT_NAME
119+
assert service.credential.account_name == DEVSTORE_ACCOUNT_NAME
120+
assert service.credential.account_key == DEVSTORE_ACCOUNT_KEY
121+
assert f"127.0.0.1:10000/{DEVSTORE_ACCOUNT_NAME}" in service.url
122+
104123
@BlobPreparer()
105124
def test_create_service_with_sas(self, **kwargs):
106125
storage_account_name = kwargs.pop("storage_account_name")
@@ -343,19 +362,6 @@ def test_create_service_with_connection_string_endpoint_protocol(self, **kwargs)
343362
'http://{}-secondary.{}.core.chinacloudapi.cn'.format(storage_account_name, service_type[1]))
344363
assert service.scheme == 'http'
345364

346-
@BlobPreparer()
347-
def test_create_service_with_connection_string_emulated(self, **kwargs):
348-
storage_account_name = kwargs.pop("storage_account_name")
349-
storage_account_key = kwargs.pop("storage_account_key")
350-
351-
# Arrange
352-
for service_type in SERVICES.items():
353-
conn_string = 'UseDevelopmentStorage=true;'.format(storage_account_name, storage_account_key)
354-
355-
# Act
356-
with pytest.raises(ValueError):
357-
service = service_type[0].from_connection_string(conn_string, container_name="foo", blob_name="bar")
358-
359365
@BlobPreparer()
360366
def test_create_service_with_cstr_anonymous(self):
361367
# Arrange

‎sdk/storage/azure-storage-blob/tests/test_blob_client_async.py‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
ResourceTypes,
1616
VERSION,
1717
)
18+
from azure.storage.blob._shared.parser import DEVSTORE_ACCOUNT_KEY, DEVSTORE_ACCOUNT_NAME
1819
from azure.storage.blob.aio import (
1920
BlobClient,
2021
ContainerClient,
@@ -89,6 +90,24 @@ def test_create_service_with_connection_string(self, **kwargs):
8990
self.validate_standard_account_endpoints(service, service_type[1], storage_account_name, storage_account_key)
9091
assert service.scheme == 'https'
9192

93+
@BlobPreparer()
94+
def test_create_service_use_development_storage(self):
95+
for service_type in SERVICES.items():
96+
# Act
97+
service = service_type[0].from_connection_string(
98+
"UseDevelopmentStorage=true;",
99+
container_name="test",
100+
blob_name="test"
101+
)
102+
103+
# Assert
104+
assert service is not None
105+
assert service.scheme == "http"
106+
assert service.account_name == DEVSTORE_ACCOUNT_NAME
107+
assert service.credential.account_name == DEVSTORE_ACCOUNT_NAME
108+
assert service.credential.account_key == DEVSTORE_ACCOUNT_KEY
109+
assert f"127.0.0.1:10000/{DEVSTORE_ACCOUNT_NAME}" in service.url
110+
92111
@BlobPreparer()
93112
def test_create_service_with_sas(self, **kwargs):
94113
storage_account_name = kwargs.pop("storage_account_name")
@@ -345,19 +364,6 @@ def test_creat_serv_w_connstr_endpoint_protocol(self, **kwargs):
345364
'http://{}-secondary.{}.core.chinacloudapi.cn'.format(storage_account_name, service_type[1]))
346365
assert service.scheme == 'http'
347366

348-
@BlobPreparer()
349-
def test_create_service_with_connection_string_emulated(self, **kwargs):
350-
storage_account_name = kwargs.pop("storage_account_name")
351-
storage_account_key = kwargs.pop("storage_account_key")
352-
353-
# Arrange
354-
for service_type in SERVICES.items():
355-
conn_string = 'UseDevelopmentStorage=true;'.format(storage_account_name, storage_account_key)
356-
357-
# Act
358-
with pytest.raises(ValueError):
359-
service = service_type[0].from_connection_string(conn_string, container_name="foo", blob_name="bar")
360-
361367
@BlobPreparer()
362368
def test_create_service_with_connection_string_anonymous(self):
363369
# Arrange

‎sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/base_client.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
from .authentication import SharedKeyCredentialPolicy
3838
from .constants import CONNECTION_TIMEOUT, DEFAULT_OAUTH_SCOPE, READ_TIMEOUT, SERVICE_HOST_BASE, STORAGE_OAUTH_SCOPE
3939
from .models import LocationMode, StorageConfiguration
40+
from .parser import DEVSTORE_ACCOUNT_KEY, _get_development_storage_endpoint
4041
from .policies import (
4142
ExponentialRetry,
4243
QueueMessagePolicy,
@@ -407,6 +408,8 @@ def parse_connection_str(
407408
if any(len(tup) != 2 for tup in conn_settings_list):
408409
raise ValueError("Connection string is either blank or malformed.")
409410
conn_settings = dict((key.upper(), val) for key, val in conn_settings_list)
411+
if conn_settings.get('USEDEVELOPMENTSTORAGE') == 'true':
412+
return _get_development_storage_endpoint(service), None, DEVSTORE_ACCOUNT_KEY
410413
endpoints = _SERVICE_PARAMS[service]
411414
primary = None
412415
secondary = None

‎sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/base_client_async.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
from .base_client import create_configuration
2727
from .constants import CONNECTION_TIMEOUT, DEFAULT_OAUTH_SCOPE, READ_TIMEOUT, SERVICE_HOST_BASE, STORAGE_OAUTH_SCOPE
2828
from .models import StorageConfiguration
29+
from .parser import DEVSTORE_ACCOUNT_KEY, _get_development_storage_endpoint
2930
from .policies import (
3031
QueueMessagePolicy,
3132
StorageContentValidation,
@@ -201,6 +202,8 @@ def parse_connection_str(
201202
if any(len(tup) != 2 for tup in conn_settings_list):
202203
raise ValueError("Connection string is either blank or malformed.")
203204
conn_settings = dict((key.upper(), val) for key, val in conn_settings_list)
205+
if conn_settings.get('USEDEVELOPMENTSTORAGE') == 'true':
206+
return _get_development_storage_endpoint(service), None, DEVSTORE_ACCOUNT_KEY
204207
endpoints = _SERVICE_PARAMS[service]
205208
primary = None
206209
secondary = None

‎sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/parser.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,14 @@
1010
EPOCH_AS_FILETIME = 116444736000000000 # January 1, 1970 as MS filetime
1111
HUNDREDS_OF_NANOSECONDS = 10000000
1212

13+
DEVSTORE_PORTS = {
14+
"blob": 10000,
15+
"dfs": 10000,
16+
"queue": 10001,
17+
}
18+
DEVSTORE_ACCOUNT_NAME = "devstoreaccount1"
19+
DEVSTORE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
20+
1321

1422
def _to_utc_datetime(value: datetime) -> str:
1523
return value.strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -51,3 +59,15 @@ def _filetime_to_datetime(filetime: str) -> Optional[datetime]:
5159

5260
# Try RFC 1123 as backup
5361
return _rfc_1123_to_datetime(filetime)
62+
63+
64+
def _get_development_storage_endpoint(service: str) -> str:
65+
"""Creates a development storage endpoint for Azurite Storage Emulator.
66+
67+
:param str service: The service name.
68+
:return: The development storage endpoint.
69+
:rtype: str
70+
"""
71+
if service.lower() not in DEVSTORE_PORTS:
72+
raise ValueError(f"Unsupported service name: {service}")
73+
return f"http://127.0.0.1:{DEVSTORE_PORTS[service]}/{DEVSTORE_ACCOUNT_NAME}"

‎sdk/storage/azure-storage-file-datalake/tests/test_datalake_service_client.py‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,9 @@
1818
FileSystemClient,
1919
Metrics,
2020
RetentionPolicy,
21-
StaticWebsite)
21+
StaticWebsite
22+
)
23+
from azure.storage.filedatalake._shared.parser import DEVSTORE_ACCOUNT_KEY, DEVSTORE_ACCOUNT_NAME
2224

2325
from devtools_testutils import recorded_by_proxy
2426
from devtools_testutils.storage import StorageRecordedTestCase
@@ -109,6 +111,14 @@ def _assert_retention_equal(self, ret1, ret2):
109111
assert ret1.enabled == ret2.enabled
110112
assert ret1.days == ret2.days
111113

114+
def _assert_devstore_conn_str(self, service):
115+
assert service is not None
116+
assert service.scheme == "http"
117+
assert service.account_name == DEVSTORE_ACCOUNT_NAME
118+
assert service.credential.account_name == DEVSTORE_ACCOUNT_NAME
119+
assert service.credential.account_key == DEVSTORE_ACCOUNT_KEY
120+
assert f"127.0.0.1:10000/{DEVSTORE_ACCOUNT_NAME}" in service.url
121+
112122
# --Test cases per service ---------------------------------------
113123
@DataLakePreparer()
114124
@recorded_by_proxy
@@ -398,6 +408,22 @@ def test_connectionstring_without_secondary(self):
398408
assert client.primary_hostname == 'foo.dfs.core.windows.net'
399409
assert not client.secondary_hostname
400410

411+
@DataLakePreparer()
412+
def test_connectionstring_use_development_storage(self):
413+
test_conn_str = "UseDevelopmentStorage=true;"
414+
415+
dsc = DataLakeServiceClient.from_connection_string(test_conn_str)
416+
self._assert_devstore_conn_str(dsc)
417+
418+
fsc = FileSystemClient.from_connection_string(test_conn_str, "fsname")
419+
self._assert_devstore_conn_str(fsc)
420+
421+
dfc = DataLakeFileClient.from_connection_string(test_conn_str, "fsname", "fpath")
422+
self._assert_devstore_conn_str(dfc)
423+
424+
ddc = DataLakeDirectoryClient.from_connection_string(test_conn_str, "fsname", "dname")
425+
self._assert_devstore_conn_str(ddc)
426+
401427
@DataLakePreparer()
402428
@recorded_by_proxy
403429
def test_azure_named_key_credential_access(self, **kwargs):

‎sdk/storage/azure-storage-file-datalake/tests/test_datalake_service_client_async.py‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,15 @@
1515
CorsRule,
1616
Metrics,
1717
RetentionPolicy,
18-
StaticWebsite)
18+
StaticWebsite
19+
)
20+
from azure.storage.filedatalake._shared.parser import DEVSTORE_ACCOUNT_KEY, DEVSTORE_ACCOUNT_NAME
1921
from azure.storage.filedatalake.aio import (
2022
DataLakeDirectoryClient,
2123
DataLakeFileClient,
2224
DataLakeServiceClient,
23-
FileSystemClient)
25+
FileSystemClient
26+
)
2427

2528
from devtools_testutils.aio import recorded_by_proxy_async
2629
from devtools_testutils.storage.aio import AsyncStorageRecordedTestCase
@@ -114,6 +117,14 @@ def _assert_retention_equal(self, ret1, ret2):
114117
assert ret1.enabled == ret2.enabled
115118
assert ret1.days == ret2.days
116119

120+
def _assert_devstore_conn_str(self, service):
121+
assert service is not None
122+
assert service.scheme == "http"
123+
assert service.account_name == DEVSTORE_ACCOUNT_NAME
124+
assert service.credential.account_name == DEVSTORE_ACCOUNT_NAME
125+
assert service.credential.account_key == DEVSTORE_ACCOUNT_KEY
126+
assert f"127.0.0.1:10000/{DEVSTORE_ACCOUNT_NAME}" in service.url
127+
117128
# --Test cases per service ---------------------------------------
118129
@DataLakePreparer()
119130
@recorded_by_proxy_async
@@ -399,6 +410,22 @@ async def test_connectionstring_without_secondary(self):
399410
assert client.primary_hostname == 'foo.dfs.core.windows.net'
400411
assert not client.secondary_hostname
401412

413+
@DataLakePreparer()
414+
def test_connectionstring_use_development_storage(self):
415+
test_conn_str = "UseDevelopmentStorage=true;"
416+
417+
dsc = DataLakeServiceClient.from_connection_string(test_conn_str)
418+
self._assert_devstore_conn_str(dsc)
419+
420+
fsc = FileSystemClient.from_connection_string(test_conn_str, "fsname")
421+
self._assert_devstore_conn_str(fsc)
422+
423+
dfc = DataLakeFileClient.from_connection_string(test_conn_str, "fsname", "fpath")
424+
self._assert_devstore_conn_str(dfc)
425+
426+
ddc = DataLakeDirectoryClient.from_connection_string(test_conn_str, "fsname", "dname")
427+
self._assert_devstore_conn_str(ddc)
428+
402429
@DataLakePreparer()
403430
@recorded_by_proxy_async
404431
async def test_azure_named_key_credential_access(self, **kwargs):

0 commit comments

Comments
 (0)