Support RTDB Emulator via FIREBASE_DATABASE_EMULATOR_HOST. (#313)

* Support RTDB Emulator via FIREBASE_DATABASE_EMULATOR_HOST.

* Fix linter issues.

* Defer ApplicationDefault init and remove FakeCredential class.

* Fix lazy initialization and tests.

* Address PR feedback and clean up URL parsing logic.

* Use non-global app for db tests.

* Simplify app project_id initialization logic.

* Docstring.

* Simplify parsing logic again!

* Docstring and indentation fix.

* Return!

Author: yuchenshi

CONTRIBUTING.md

@@ -195,6 +195,18 @@ Now you can invoke the integration test suite as follows:
 pytest integration/ --cert scripts/cert.json --apikey scripts/apikey.txt
 ```
 
+### Emulator-based Integration Testing
+
+Some integration tests can run against emulators. This allows local testing
+without using real projects or credentials. For now, only the RTDB Emulator
+is supported.
+
+First, install the Firebase CLI, then run:
+
+```
+firebase emulators:exec --only database --project fake-project-id 'pytest integration/test_db.py'
+```
+
 ### Test Coverage
 
 To review the test coverage, run `pytest` with the `--cov` flag. To view a detailed line by line

firebase_admin/__init__.py

@@ -215,39 +215,15 @@ def __init__(self, name, credential, options):
         self._options = _AppOptions(options)
         self._lock = threading.RLock()
         self._services = {}
-        self._project_id = App._lookup_project_id(self._credential, self._options)
 
-    @classmethod
-    def _lookup_project_id(cls, credential, options):
-        """Looks up the Firebase project ID associated with an App.
-
-        This method first inspects the app options for a ``projectId`` entry. Then it attempts to
-        get the project ID from the credential used to initialize the app. If that also fails,
-        attempts to look up the ``GOOGLE_CLOUD_PROJECT`` and ``GCLOUD_PROJECT`` environment
-        variables.
-
-        Args:
-            credential: A Firebase credential instance.
-            options: A Firebase AppOptions instance.
-
-        Returns:
-            str: A project ID string or None.
+        App._validate_project_id(self._options.get('projectId'))
+        self._project_id_initialized = False
 
-        Raises:
-            ValueError: If a non-string project ID value is specified.
-        """
-        project_id = options.get('projectId')
-        if not project_id:
-            try:
-                project_id = credential.project_id
-            except AttributeError:
-                pass
-        if not project_id:
-            project_id = os.environ.get('GOOGLE_CLOUD_PROJECT', os.environ.get('GCLOUD_PROJECT'))
+    @classmethod
+    def _validate_project_id(cls, project_id):
         if project_id is not None and not isinstance(project_id, six.string_types):
             raise ValueError(
                 'Invalid project ID: "{0}". project ID must be a string.'.format(project_id))
-        return project_id
 
     @property
     def name(self):
@@ -263,8 +239,34 @@ def options(self):
 
     @property
     def project_id(self):
+        if not self._project_id_initialized:
+            self._project_id = self._lookup_project_id()
+            self._project_id_initialized = True
         return self._project_id
 
+    def _lookup_project_id(self):
+        """Looks up the Firebase project ID associated with an App.
+
+        If a ``projectId`` is specified in app options, it is returned. Then tries to
+        get the project ID from the credential used to initialize the app. If that also fails,
+        attempts to look up the ``GOOGLE_CLOUD_PROJECT`` and ``GCLOUD_PROJECT`` environment
+        variables.
+
+        Returns:
+            str: A project ID string or None.
+        """
+        project_id = self._options.get('projectId')
+        if not project_id:
+            try:
+                project_id = self._credential.project_id
+            except AttributeError:
+                pass
+        if not project_id:
+            project_id = os.environ.get('GOOGLE_CLOUD_PROJECT',
+                                        os.environ.get('GCLOUD_PROJECT'))
+        App._validate_project_id(self._options.get('projectId'))
+        return project_id
+
     def _get_service(self, name, initializer):
         """Returns the service instance identified by the given name.
 

firebase_admin/credentials.py

@@ -123,26 +123,40 @@ class ApplicationDefault(Base):
     """A Google Application Default credential."""
 
     def __init__(self):
-        """Initializes the Application Default credentials for the current environment.
+        """Creates an instance that will use Application Default credentials.
 
-        Raises:
-          google.auth.exceptions.DefaultCredentialsError: If Application Default
-              credentials cannot be initialized in the current environment.
+        The credentials will be lazily initialized when get_credential() or
+        project_id() is called. See those methods for possible errors raised.
         """
         super(ApplicationDefault, self).__init__()
-        self._g_credential, self._project_id = google.auth.default(scopes=_scopes)
+        self._g_credential = None  # Will be lazily-loaded via _load_credential().
 
     def get_credential(self):
         """Returns the underlying Google credential.
 
+        Raises:
+          google.auth.exceptions.DefaultCredentialsError: If Application Default
+              credentials cannot be initialized in the current environment.
         Returns:
           google.auth.credentials.Credentials: A Google Auth credential instance."""
+        self._load_credential()
         return self._g_credential
 
     @property
     def project_id(self):
+        """Returns the project_id from the underlying Google credential.
+
+        Raises:
+          google.auth.exceptions.DefaultCredentialsError: If Application Default
+              credentials cannot be initialized in the current environment.
+        Returns:
+          str: The project id."""
+        self._load_credential()
         return self._project_id
 
+    def _load_credential(self):
+        if not self._g_credential:
+            self._g_credential, self._project_id = google.auth.default(scopes=_scopes)
 
 class RefreshToken(Base):
     """A credential initialized from an existing refresh token."""

firebase_admin/db.py

@@ -22,9 +22,11 @@
 
 import collections
 import json
+import os
 import sys
 import threading
 
+import google.auth
 import requests
 import six
 from six.moves import urllib
@@ -41,6 +43,7 @@
 _USER_AGENT = 'Firebase/HTTP/{0}/{1}.{2}/AdminPython'.format(
     firebase_admin.__version__, sys.version_info.major, sys.version_info.minor)
 _TRANSACTION_MAX_RETRIES = 25
+_EMULATOR_HOST_ENV_VAR = 'FIREBASE_DATABASE_EMULATOR_HOST'
 
 
 def reference(path='/', app=None, url=None):
@@ -768,46 +771,108 @@ class _DatabaseService(object):
     _DEFAULT_AUTH_OVERRIDE = '_admin_'
 
     def __init__(self, app):
-        self._credential = app.credential.get_credential()
+        self._credential = app.credential
         db_url = app.options.get('databaseURL')
         if db_url:
-            self._db_url = _DatabaseService._validate_url(db_url)
+            _DatabaseService._parse_db_url(db_url)  # Just for validation.
+            self._db_url = db_url
         else:
             self._db_url = None
         auth_override = _DatabaseService._get_auth_override(app)
         if auth_override != self._DEFAULT_AUTH_OVERRIDE and auth_override != {}:
-            encoded = json.dumps(auth_override, separators=(',', ':'))
-            self._auth_override = 'auth_variable_override={0}'.format(encoded)
+            self._auth_override = json.dumps(auth_override, separators=(',', ':'))
         else:
             self._auth_override = None
         self._timeout = app.options.get('httpTimeout')
         self._clients = {}
 
-    def get_client(self, base_url=None):
-        if base_url is None:
-            base_url = self._db_url
-        base_url = _DatabaseService._validate_url(base_url)
-        if base_url not in self._clients:
-            client = _Client(self._credential, base_url, self._auth_override, self._timeout)
-            self._clients[base_url] = client
-        return self._clients[base_url]
+        emulator_host = os.environ.get(_EMULATOR_HOST_ENV_VAR)
+        if emulator_host:
+            if '//' in emulator_host:
+                raise ValueError(
+                    'Invalid {0}: "{1}". It must follow format "host:port".'.format(
+                        _EMULATOR_HOST_ENV_VAR, emulator_host))
+            self._emulator_host = emulator_host
+        else:
+            self._emulator_host = None
+
+    def get_client(self, db_url=None):
+        """Creates a client based on the db_url. Clients may be cached."""
+        if db_url is None:
+            db_url = self._db_url
+
+        base_url, namespace = _DatabaseService._parse_db_url(db_url, self._emulator_host)
+        if base_url == 'https://{0}.firebaseio.com'.format(namespace):
+            # Production base_url. No need to specify namespace in query params.
+            params = {}
+            credential = self._credential.get_credential()
+        else:
+            # Emulator base_url. Use fake credentials and specify ?ns=foo in query params.
+            credential = _EmulatorAdminCredentials()
+            params = {'ns': namespace}
+        if self._auth_override:
+            params['auth_variable_override'] = self._auth_override
+
+        client_cache_key = (base_url, json.dumps(params, sort_keys=True))
+        if client_cache_key not in self._clients:
+            client = _Client(credential, base_url, self._timeout, params)
+            self._clients[client_cache_key] = client
+        return self._clients[client_cache_key]
 
     @classmethod
-    def _validate_url(cls, url):
-        """Parses and validates a given database URL."""
+    def _parse_db_url(cls, url, emulator_host=None):
+        """Parses (base_url, namespace) from a database URL.
+
+        The input can be either a production URL (https://foo-bar.firebaseio.com/)
+        or an Emulator URL (http://localhost:8080/?ns=foo-bar). In case of Emulator
+        URL, the namespace is extracted from the query param ns. The resulting
+        base_url never includes query params.
+
+        If url is a production URL and emulator_host is specified, the result
+        base URL will use emulator_host instead. emulator_host is ignored
+        if url is already an emulator URL.
+        """
         if not url or not isinstance(url, six.string_types):
             raise ValueError(
                 'Invalid database URL: "{0}". Database URL must be a non-empty '
                 'URL string.'.format(url))
-        parsed = urllib.parse.urlparse(url)
-        if parsed.scheme != 'https':
+        parsed_url = urllib.parse.urlparse(url)
+        if parsed_url.netloc.endswith('.firebaseio.com'):
+            return cls._parse_production_url(parsed_url, emulator_host)
+        else:
+            return cls._parse_emulator_url(parsed_url)
+
+    @classmethod
+    def _parse_production_url(cls, parsed_url, emulator_host):
+        """Parses production URL like https://foo-bar.firebaseio.com/"""
+        if parsed_url.scheme != 'https':
             raise ValueError(
-                'Invalid database URL: "{0}". Database URL must be an HTTPS URL.'.format(url))
-        elif not parsed.netloc.endswith('.firebaseio.com'):
+                'Invalid database URL scheme: "{0}". Database URL must be an HTTPS URL.'.format(
+                    parsed_url.scheme))
+        namespace = parsed_url.netloc.split('.')[0]
+        if not namespace:
             raise ValueError(
                 'Invalid database URL: "{0}". Database URL must be a valid URL to a '
-                'Firebase Realtime Database instance.'.format(url))
-        return 'https://{0}'.format(parsed.netloc)
+                'Firebase Realtime Database instance.'.format(parsed_url.geturl()))
+
+        if emulator_host:
+            base_url = 'http://{0}'.format(emulator_host)
+        else:
+            base_url = 'https://{0}'.format(parsed_url.netloc)
+        return base_url, namespace
+
+    @classmethod
+    def _parse_emulator_url(cls, parsed_url):
+        """Parses emulator URL like http://localhost:8080/?ns=foo-bar"""
+        query_ns = urllib.parse.parse_qs(parsed_url.query).get('ns')
+        if parsed_url.scheme != 'http' or (not query_ns or len(query_ns) != 1 or not query_ns[0]):
+            raise ValueError(
+                'Invalid database URL: "{0}". Database URL must be a valid URL to a '
+                'Firebase Realtime Database instance.'.format(parsed_url.geturl()))
+
+        namespace = query_ns[0]
+        base_url = '{0}://{1}'.format(parsed_url.scheme, parsed_url.netloc)
+        return base_url, namespace
 
     @classmethod
     def _get_auth_override(cls, app):
@@ -833,7 +898,7 @@ class _Client(_http_client.JsonHttpClient):
     marshalling and unmarshalling of JSON data.
     """
 
-    def __init__(self, credential, base_url, auth_override, timeout):
+    def __init__(self, credential, base_url, timeout, params=None):
         """Creates a new _Client from the given parameters.
 
         This exists primarily to enable testing. For regular use, obtain _Client instances by
@@ -843,22 +908,21 @@ def __init__(self, credential, base_url, auth_override, timeout):
           credential: A Google credential that can be used to authenticate requests.
           base_url: A URL prefix to be added to all outgoing requests. This is typically the
               Firebase Realtime Database URL.
-          auth_override: The encoded auth_variable_override query parameter to be included in
-              outgoing requests.
           timeout: HTTP request timeout in seconds. If not set connections will never
               timeout, which is the default behavior of the underlying requests library.
+          params: Dict of query parameters to add to all outgoing requests.
         """
         _http_client.JsonHttpClient.__init__(
             self, credential=credential, base_url=base_url, headers={'User-Agent': _USER_AGENT})
         self.credential = credential
-        self.auth_override = auth_override
         self.timeout = timeout
+        self.params = params if params else {}
 
     def request(self, method, url, **kwargs):
         """Makes an HTTP call using the Python requests library.
 
-        Extends the request() method of the parent JsonHttpClient class. Handles auth overrides,
-        and low-level exceptions.
+        Extends the request() method of the parent JsonHttpClient class. Handles default
+        params like auth overrides, and low-level exceptions.
 
         Args:
           method: HTTP method name as a string (e.g. get, post).
@@ -872,13 +936,15 @@ def request(self, method, url, **kwargs):
         Raises:
           ApiCallError: If an error occurs while making the HTTP call.
         """
-        if self.auth_override:
-            params = kwargs.get('params')
-            if params:
-                params += '&{0}'.format(self.auth_override)
+        query = '&'.join('{0}={1}'.format(key, self.params[key]) for key in self.params)
+        extra_params = kwargs.get('params')
+        if extra_params:
+            if query:
+                query = extra_params + '&' + query
             else:
-                params = self.auth_override
-            kwargs['params'] = params
+                query = extra_params
+        kwargs['params'] = query
+
         if self.timeout:
             kwargs['timeout'] = self.timeout
         try:
@@ -911,3 +977,12 @@ def extract_error_message(cls, error):
         except ValueError:
             pass
         return '{0}\nReason: {1}'.format(error, error.response.content.decode())
+
+
+class _EmulatorAdminCredentials(google.auth.credentials.Credentials):
+    def __init__(self):
+        google.auth.credentials.Credentials.__init__(self)
+        self.token = 'owner'
+
+    def refresh(self, request):
+        pass

integration/conftest.py

@@ -70,4 +70,3 @@ def api_key(request):
                          'command-line option.')
     with open(path) as keyfile:
         return keyfile.read().strip()
-