MHR replace GCP SA keys with container ADC for GCP API calls

jobs/permanent/assets-payment/src/assets_payment/config.py

@@ -37,6 +37,8 @@ class Config(BaseConfig):
     AUTH_SVC_URL = f"{AUTH_API_URL + AUTH_API_VERSION}"
     PAYMENT_SVC_URL = f"{PAY_API_URL + PAY_API_VERSION}"
     NOTIFY_SVC_URL = f"{NOTIFY_API_URL + NOTIFY_API_VERSION}"
+    if not NOTIFY_SVC_URL.endswith("/notify"):
+        NOTIFY_SVC_URL += "/notify"
 
     JWT_OIDC_TOKEN_URL = os.getenv("JWT_OIDC_TOKEN_URL")
     # service accounts

mhr-api/devops/vaults.gcp.env

@@ -20,7 +20,6 @@ PAY_API_URL="op://API/$APP_ENV/pay-api/PAY_API_URL"
 PAY_API_VERSION="op://API/$APP_ENV/pay-api/PAY_API_VERSION"
 REPORT_API_URL="op://ppr/$APP_ENV/mhr-api/REPORT_SVC_URL"
 REPORT_API_AUDIENCE="op://ppr/$APP_ENV/mhr-api/REPORT_API_AUDIENCE"
-GOOGLE_DEFAULT_SA="op://buckets/$APP_ENV/mhr-api/GOOGLE_DEFAULT_SA"
 GCP_CS_PROJECT_ID="op://ppr/$APP_ENV/mhr-api/GCP_PS_PROJECT_ID"
 GCP_CS_SA_SCOPES="op://buckets/$APP_ENV/mhr-api/GCP_CS_SA_SCOPES"
 GCP_CS_BUCKET_ID="op://buckets/$APP_ENV/mhr-api/GCP_CS_BUCKET_ID"

mhr-api/src/mhr_api/services/document_storage/storage_service.py

@@ -130,7 +130,7 @@ def __get_bucket_id(cls, doc_type: str = None):
     def __call_cs_api(cls, method: str, name: str, data=None, doc_type: str = None):
         """Call the Cloud Storage API."""
         credentials = GoogleAuthService.get_credentials()
-        storage_client = storage.Client(credentials=credentials)
+        storage_client = storage.Client(credentials=credentials) if credentials else storage.Client()
         bucket = storage_client.bucket(cls.__get_bucket_id(doc_type))
         blob = bucket.blob(name)
         if method == HTTP_POST:
@@ -147,13 +147,20 @@ def __call_cs_api(cls, method: str, name: str, data=None, doc_type: str = None):
     @classmethod
     def __call_cs_api_link(cls, name: str, data=None, doc_type: str = None, available_days: int = 1):
         """Call the Cloud Storage API, returning a time-limited download link."""
-        credentials = GoogleAuthService.get_credentials()
+        credentials = GoogleAuthService.get_cs_signed_credentials()
+        if not credentials:
+            logger.warning(f"No credentials to create signed storage link for {name}")
+            return ""
         storage_client = storage.Client(credentials=credentials)
         bucket = storage_client.bucket(cls.__get_bucket_id(doc_type))
         blob = bucket.blob(name)
         if data:
             blob.upload_from_string(data=data, content_type=CONTENT_TYPE_PDF)
         url = blob.generate_signed_url(
-            version="v4", expiration=datetime.timedelta(days=available_days, hours=0, minutes=0), method="GET"
+            version="v4",
+            expiration=datetime.timedelta(days=available_days, hours=0, minutes=0),
+            method="GET",
+            service_account_email=credentials.service_account_email,
+            access_token=credentials.token,
         )
         return url

mhr-api/src/mhr_api/services/gcp_auth/auth_service.py

@@ -37,10 +37,6 @@ class GoogleAuthService(AuthService):  # pylint: disable=too-few-public-methods
 
     service_account_info = None
     credentials = None
-    # Use service account env var if available.
-    if gcp_auth_key:
-        sa_bytes = bytes(gcp_auth_key, "utf-8")
-        service_account_info = json.loads(base64.b64decode(sa_bytes.decode("utf-8")))
 
     @staticmethod
     def init_app(app):
@@ -50,10 +46,18 @@ def init_app(app):
         if GoogleAuthService.gcp_auth_key:
             sa_bytes = bytes(GoogleAuthService.gcp_auth_key, "utf-8")
             GoogleAuthService.service_account_info = json.loads(base64.b64decode(sa_bytes.decode("utf-8")))
+            GoogleAuthService.credentials = service_account.Credentials.from_service_account_info(
+                GoogleAuthService.service_account_info, scopes=GoogleAuthService.gcp_sa_scopes
+            )
+        else:
+            logger.info("auth_service.init_app no SA info.")
 
     @classmethod
     def get_token(cls):
         """Generate an OAuth access token with cloud storage access."""
+        if not cls.gcp_auth_key or not cls.service_account_info:
+            return None
+
         if cls.credentials is None:
             cls.credentials = service_account.Credentials.from_service_account_info(
                 cls.service_account_info, scopes=cls.gcp_sa_scopes
@@ -77,9 +81,25 @@ def get_report_api_token(cls):
     @classmethod
     def get_credentials(cls):
         """Generate GCP auth credentials to pass to a GCP client."""
+        if not cls.gcp_auth_key or not cls.service_account_info:
+            return None
+
         if cls.credentials is None:
             cls.credentials = service_account.Credentials.from_service_account_info(
                 cls.service_account_info, scopes=cls.gcp_sa_scopes
             )
-        logger.info("Call successful: obtained credentials.")
+        logger.debug("Call successful: obtained credentials.")
         return cls.credentials
+
+    @classmethod
+    def get_cs_signed_credentials(cls):
+        """Extra steps for ADC cloud storage signed url - requires cert to sign."""
+        if cls.gcp_auth_key and cls.service_account_info:
+            return cls.get_credentials()
+
+        # Load default credentials
+        credentials, project = google.auth.default()  # pylint: disable=unused-variable; gcp api response
+        # Refresh credentials to ensure an access token is available
+        auth_request = google.auth.transport.requests.Request()
+        credentials.refresh(auth_request)
+        return credentials

mhr-api/src/mhr_api/services/queue_service.py

@@ -35,7 +35,9 @@ class GoogleQueueService:
     def init_app(app):
         """Initialize the publisher."""
         credentials = GoogleAuthService.get_credentials()
-        GoogleQueueService.publisher = pubsub_v1.PublisherClient(credentials=credentials)
+        GoogleQueueService.publisher = (
+            pubsub_v1.PublisherClient(credentials=credentials) if credentials else pubsub_v1.PublisherClient()
+        )
         project_id = str(app.config.get("GCP_PS_PROJECT_ID"))
         search_report_topic = str(app.config.get("GCP_PS_SEARCH_REPORT_TOPIC"))
         registration_report_topic = str(app.config.get("GCP_PS_REGISTRATION_REPORT_TOPIC"))

mhr-api/tests/unit/services/test_auth_service.py

@@ -19,63 +19,34 @@
 from flask import current_app
 
 from mhr_api.services.gcp_auth.auth_service import GoogleAuthService
+from mhr_api.utils.logging import logger
 
 
 def test_get_token(session, client, jwt):
     """Assert that the configuration to get a google storage token works as expected (no exceptions)."""
     token = GoogleAuthService.get_token()
-    print(token)
-    assert token
+    if current_app.config.get("GOOGLE_DEFAULT_SA"):
+        logger.debug(token)
+        assert token
+    else:
+        assert not token
 
 
 def test_get_credentials(session, client, jwt):
     """Assert that the configuration to get a google storage token works as expected (no exceptions)."""
     credentials = GoogleAuthService.get_credentials()
-    assert credentials
+    if current_app.config.get("GOOGLE_DEFAULT_SA"):
+        assert credentials
+        assert credentials.token
+        assert credentials.service_account_email
+    else:
+        assert not credentials
 
 
-def test_security_account(session, client, jwt):
-    """Assert that the configuration to get the GCP service account from the environment works as expected."""
-    decoded_sa = None
-    encoded_sa: bytes = None
-    default_sa = os.getenv('GOOGLE_DEFAULT_SERVICE_ACCOUNT')
-    if default_sa:
-        encoded_sa = bytes(default_sa, 'utf-8')
-    if not encoded_sa and os.getenv('GCP_CS_SA_CLIENT_ID'):  # local env testing only
-        current_app.logger.info('No GOOGLE_DEFAULT_SERVICE_ACCOUNT env var.')
-        sa_project_id = os.getenv('GCP_CS_PROJECT_ID')
-        sa_client_email = os.getenv('GCP_CS_SA_CLIENT_EMAIL')
-        sa_client_id = os.getenv('GCP_CS_SA_CLIENT_ID')
-        sa_private_key  = os.getenv('GCP_CS_SA_PRIVATE_KEY')
-        sa_private_key_id = os.getenv('GCP_CS_SA_PRIVATE_KEY_ID')
-        sa_cert_url = os.getenv('GCP_CS_SA_CERT_URL')
-        service_account_info = {
-            'type': 'service_account',
-            'project_id': sa_project_id,
-            'private_key_id': sa_private_key_id,
-            'private_key': str(sa_private_key).replace('\\n', '\n'),
-            'client_email': sa_client_email,
-            'client_id': sa_client_id,
-            'auth_uri': 'https://accounts.google.com/o/oauth2/auth',
-            'token_uri': 'https://oauth2.googleapis.com/token',
-            'auth_provider_x509_cert_url': 'https://www.googleapis.com/oauth2/v1/certs',
-            'client_x509_cert_url': sa_cert_url
-        }
-        encoded_sa = base64.b64encode(json.dumps(service_account_info).encode('utf-8'))
-        current_app.logger.debug(encoded_sa)
-
-    if default_sa:
-        assert encoded_sa
-        decoded_sa = json.loads(base64.b64decode(encoded_sa.decode('utf-8')))
-        # current_app.logger.debug(decoded_sa)
-        assert decoded_sa
-        assert decoded_sa.get('type')
-        assert decoded_sa.get('project_id')
-        assert decoded_sa.get('private_key_id')
-        assert decoded_sa.get('private_key')
-        assert decoded_sa.get('client_email')
-        assert decoded_sa.get('client_id')
-        assert decoded_sa.get('auth_uri')
-        assert decoded_sa.get('token_uri')
-        assert decoded_sa.get('auth_provider_x509_cert_url')
-        assert decoded_sa.get('client_x509_cert_url')
+def test_get_cs_signed_credentials(session, client, jwt):
+    """Assert that the configuration to get a google storage token works as expected (no exceptions)."""
+    if current_app.config.get("GOOGLE_DEFAULT_SA"):
+        credentials = GoogleAuthService.get_cs_signed_credentials()
+        assert credentials
+        assert credentials.token
+        assert credentials.service_account_email

mhr-api/tests/unit/services/test_notify_service.py

@@ -119,14 +119,14 @@ def test_review_notify(session, jwt, desc, has_env_var, has_email, status, appro
     if not has_env_var:
         current_app.config.update(NOTIFY_REVIEW_CONFIG="")
     reg_data = copy.deepcopy(TEST_TRANSFER)
-    if not has_email:
+    if not has_email or desc in ("Approved valid", "Declined valid"):
         del reg_data["submittingParty"]["emailAddress"]
     notify: Notify = Notify(**{"review": True})
     n_status = HTTPStatus.OK
     if approved:
         n_status = notify.send_review_approved(reg_data, "verify_url")
     else:
-        n_status = notify.send_review_declined(reg_data, "declined reason here")
+        n_status = notify.send_review_declined(reg_data, "declined reason here", "rejection_url")
     if env_var:
         current_app.config.update(NOTIFY_REVIEW_CONFIG=env_var)
     assert n_status == status

mhr-api/tests/unit/services/test_queue_service.py

@@ -46,19 +46,19 @@
 def test_publish_search_report(session):
     """Assert that enqueuing/publishing a search report event works as expected (no exception thrown)."""
     payload = TEST_PAYLOAD
-    apikey = current_app.config.get('SUBSCRIPTION_API_KEY')
-    if apikey:
+    if not is_ci_testing() and current_app.config.get("SUBSCRIPTION_API_KEY"):
+        apikey = current_app.config.get('SUBSCRIPTION_API_KEY')
         payload['apikey'] = apikey
-    GoogleQueueService().publish_search_report(payload)
+        GoogleQueueService().publish_search_report(payload)
 
 
 def test_publish_registration_report(session):
     """Assert that enqueuing/publishing a registration report event works as expected (no exception thrown)."""
     payload = TEST_PAYLOAD_REGISTRATION
-    apikey = current_app.config.get('SUBSCRIPTION_API_KEY')
-    if apikey:
+    if not is_ci_testing() and current_app.config.get("SUBSCRIPTION_API_KEY"):
+        apikey = current_app.config.get('SUBSCRIPTION_API_KEY')
         payload['apikey'] = apikey
-    GoogleQueueService().publish_registration_report(payload)
+        GoogleQueueService().publish_registration_report(payload)
 
 
 def test_publish_document_rec(session):
@@ -78,4 +78,6 @@ def test_publish_pay_completion(session):
 
 def is_ci_testing() -> bool:
     """Check unit test environment: exclude pub/sub for CI testing."""
+    if not current_app.config.get("GOOGLE_DEFAULT_SA"):
+        return True
     return  current_app.config.get("DEPLOYMENT_ENV", "testing") == "testing"

mhr-api/tests/unit/services/test_storage_service.py

@@ -33,6 +33,8 @@
 
 def test_cs_save_search_document(session):
     """Assert that saving a search bucket document to google cloud storage works as expected."""
+    if is_ci_testing():
+        return
     raw_data = None
     with open(TEST_DATAFILE, 'rb') as data_file:
         raw_data = data_file.read()
@@ -44,6 +46,8 @@ def test_cs_save_search_document(session):
 
 def test_cs_save_registration_document(session):
     """Assert that saving a registration verification statement to google cloud storage works as expected."""
+    if is_ci_testing():
+        return
     raw_data = None
     with open(TEST_REGISTRATION_DATAFILE, 'rb') as data_file:
         raw_data = data_file.read()
@@ -57,6 +61,8 @@ def test_cs_save_registration_document(session):
 
 def test_cs_get_search_document(session):
     """Assert that getting a search bucket document from google cloud storage works as expected."""
+    if is_ci_testing():
+        return
     raw_data = GoogleStorageService.get_document(TEST_SAVE_DOC_NAME, DocumentTypes.SEARCH_RESULTS)
     assert raw_data
     assert len(raw_data) > 0
@@ -67,6 +73,8 @@ def test_cs_get_search_document(session):
 
 def test_cs_get_registration_document(session):
     """Assert that getting a registration verification statement from google cloud storage works as expected."""
+    if is_ci_testing():
+        return
     raw_data = GoogleStorageService.get_document(TEST_REGISTRATION_SAVE_DOC_NAME, DocumentTypes.REGISTRATION)
     assert raw_data
     assert len(raw_data) > 0
@@ -77,6 +85,8 @@ def test_cs_get_registration_document(session):
 
 def test_cs_get_registration_document_link(session):
     """Assert that getting a document link from google cloud storage works as expected."""
+    if is_ci_testing():
+        return
     download_link = GoogleStorageService.get_document_link(TEST_REGISTRATION_SAVE_DOC_NAME,
                                                            DocumentTypes.REGISTRATION,
                                                            2)
@@ -85,11 +95,16 @@ def test_cs_get_registration_document_link(session):
 
 def test_cs_delete_search_document(session):
     """Assert that deleting a search bucket document from google cloud storage works as expected."""
+    if is_ci_testing():
+        return
     response = GoogleStorageService.delete_document(TEST_SAVE_DOC_NAME2, DocumentTypes.SEARCH_RESULTS)
+    assert not response
 
 
 def test_save_batch_registration_document(session):
     """Assert that saving a batch registration pdf to google cloud storage works as expected."""
+    if is_ci_testing():
+        return
     bucket: str = current_app.config.get('GCP_CS_BUCKET_ID_BATCH')
     current_app.logger.debug(f'Testing saving to bucket={bucket}')
     raw_data = None
@@ -104,6 +119,8 @@ def test_save_batch_registration_document(session):
 
 def test_cs_get_batch_registration_document(session):
     """Assert that getting a batch registration pdf from google cloud storage works as expected.""" 
+    if is_ci_testing():
+        return
     raw_data = GoogleStorageService.get_document(TEST_BATCH_REGISTRATION_SAVE_DOC_NAME,
                                                  DocumentTypes.BATCH_REGISTRATION)
     assert raw_data
@@ -115,10 +132,19 @@ def test_cs_get_batch_registration_document(session):
 
 def test_cs_get_service_agreement_document(session):
     """Assert that getting a qualified supplier service agreement pdf from google cloud storage works as expected."""
+    if is_ci_testing():
+        return
     raw_data = GoogleStorageService.get_document(TEST_SERVICE_AGREEMENT_SAVE_DOC_NAME,
                                                  DocumentTypes.SERVICE_AGREEMENT)
     assert raw_data
     assert len(raw_data) > 0
     with open(TEST_SERVICE_AGREEMENT_DATAFILE, "wb") as pdf_file:
         pdf_file.write(raw_data)
         pdf_file.close()
+
+
+def is_ci_testing() -> bool:
+    """Check unit test environment: exclude pub/sub for CI testing."""
+    if not current_app.config.get("GOOGLE_DEFAULT_SA"):
+        return True
+    return  current_app.config.get("DEPLOYMENT_ENV", "testing") == "testing"