From 08e1f3c0425b7edcef1cfda10c8451919cf6c800 Mon Sep 17 00:00:00 2001 From: Doug Lovett Date: Tue, 16 Jun 2026 14:27:12 -0700 Subject: [PATCH 1/3] MHR replace GCP SA keys with container ADC for GCP API calls Signed-off-by: Doug Lovett --- mhr-api/devops/vaults.gcp.env | 1 - .../document_storage/storage_service.py | 13 +++- .../mhr_api/services/gcp_auth/auth_service.py | 30 +++++++-- mhr-api/src/mhr_api/services/queue_service.py | 4 +- .../tests/unit/services/test_auth_service.py | 61 ++++++++++++++++--- .../unit/services/test_notify_service.py | 4 +- .../tests/unit/services/test_queue_service.py | 14 +++-- .../unit/services/test_storage_service.py | 26 ++++++++ 8 files changed, 127 insertions(+), 26 deletions(-) diff --git a/mhr-api/devops/vaults.gcp.env b/mhr-api/devops/vaults.gcp.env index 00fda8d00..7a0cf667e 100644 --- a/mhr-api/devops/vaults.gcp.env +++ b/mhr-api/devops/vaults.gcp.env @@ -20,7 +20,6 @@ PAY_API_URL="op://API/$APP_ENV/pay-api/PAY_API_URL" PAY_API_VERSION="op://API/$APP_ENV/pay-api/PAY_API_VERSION" REPORT_API_URL="op://ppr/$APP_ENV/mhr-api/REPORT_SVC_URL" REPORT_API_AUDIENCE="op://ppr/$APP_ENV/mhr-api/REPORT_API_AUDIENCE" -GOOGLE_DEFAULT_SA="op://buckets/$APP_ENV/mhr-api/GOOGLE_DEFAULT_SA" GCP_CS_PROJECT_ID="op://ppr/$APP_ENV/mhr-api/GCP_PS_PROJECT_ID" GCP_CS_SA_SCOPES="op://buckets/$APP_ENV/mhr-api/GCP_CS_SA_SCOPES" GCP_CS_BUCKET_ID="op://buckets/$APP_ENV/mhr-api/GCP_CS_BUCKET_ID" diff --git a/mhr-api/src/mhr_api/services/document_storage/storage_service.py b/mhr-api/src/mhr_api/services/document_storage/storage_service.py index d16886251..1f43ef4ef 100644 --- a/mhr-api/src/mhr_api/services/document_storage/storage_service.py +++ b/mhr-api/src/mhr_api/services/document_storage/storage_service.py @@ -130,7 +130,7 @@ def __get_bucket_id(cls, doc_type: str = None): def __call_cs_api(cls, method: str, name: str, data=None, doc_type: str = None): """Call the Cloud Storage API.""" credentials = GoogleAuthService.get_credentials() - storage_client = storage.Client(credentials=credentials) + storage_client = storage.Client(credentials=credentials) if credentials else storage.Client() bucket = storage_client.bucket(cls.__get_bucket_id(doc_type)) blob = bucket.blob(name) if method == HTTP_POST: @@ -147,13 +147,20 @@ def __call_cs_api(cls, method: str, name: str, data=None, doc_type: str = None): @classmethod def __call_cs_api_link(cls, name: str, data=None, doc_type: str = None, available_days: int = 1): """Call the Cloud Storage API, returning a time-limited download link.""" - credentials = GoogleAuthService.get_credentials() + credentials = GoogleAuthService.get_cs_signed_credentials() + if not credentials: + logger.warning(f"No credentials to create signed storage link for {name}") + return "" storage_client = storage.Client(credentials=credentials) bucket = storage_client.bucket(cls.__get_bucket_id(doc_type)) blob = bucket.blob(name) if data: blob.upload_from_string(data=data, content_type=CONTENT_TYPE_PDF) url = blob.generate_signed_url( - version="v4", expiration=datetime.timedelta(days=available_days, hours=0, minutes=0), method="GET" + version="v4", + expiration=datetime.timedelta(days=available_days, hours=0, minutes=0), + method="GET", + service_account_email=credentials.service_account_email, + access_token=credentials.token, ) return url diff --git a/mhr-api/src/mhr_api/services/gcp_auth/auth_service.py b/mhr-api/src/mhr_api/services/gcp_auth/auth_service.py index e52022892..5e494c1cf 100644 --- a/mhr-api/src/mhr_api/services/gcp_auth/auth_service.py +++ b/mhr-api/src/mhr_api/services/gcp_auth/auth_service.py @@ -37,10 +37,6 @@ class GoogleAuthService(AuthService): # pylint: disable=too-few-public-methods service_account_info = None credentials = None - # Use service account env var if available. - if gcp_auth_key: - sa_bytes = bytes(gcp_auth_key, "utf-8") - service_account_info = json.loads(base64.b64decode(sa_bytes.decode("utf-8"))) @staticmethod def init_app(app): @@ -50,10 +46,18 @@ def init_app(app): if GoogleAuthService.gcp_auth_key: sa_bytes = bytes(GoogleAuthService.gcp_auth_key, "utf-8") GoogleAuthService.service_account_info = json.loads(base64.b64decode(sa_bytes.decode("utf-8"))) + GoogleAuthService.credentials = service_account.Credentials.from_service_account_info( + GoogleAuthService.service_account_info, scopes=GoogleAuthService.gcp_sa_scopes + ) + else: + logger.info("auth_service.init_app no SA info.") @classmethod def get_token(cls): """Generate an OAuth access token with cloud storage access.""" + if not cls.gcp_auth_key or not cls.service_account_info: + return None + if cls.credentials is None: cls.credentials = service_account.Credentials.from_service_account_info( cls.service_account_info, scopes=cls.gcp_sa_scopes @@ -77,9 +81,25 @@ def get_report_api_token(cls): @classmethod def get_credentials(cls): """Generate GCP auth credentials to pass to a GCP client.""" + if not cls.gcp_auth_key or not cls.service_account_info: + return None + if cls.credentials is None: cls.credentials = service_account.Credentials.from_service_account_info( cls.service_account_info, scopes=cls.gcp_sa_scopes ) - logger.info("Call successful: obtained credentials.") + logger.debug("Call successful: obtained credentials.") return cls.credentials + + @classmethod + def get_cs_signed_credentials(cls): + """Extra steps for ADC cloud storage signed url - requires cert to sign.""" + if cls.gcp_auth_key and cls.service_account_info: + return cls.get_credentials() + + # Load default credentials + credentials, project = google.auth.default() # pylint: disable=unused-variable; gcp api response + # Refresh credentials to ensure an access token is available + auth_request = google.auth.transport.requests.Request() + credentials.refresh(auth_request) + return credentials diff --git a/mhr-api/src/mhr_api/services/queue_service.py b/mhr-api/src/mhr_api/services/queue_service.py index 8dc3eea65..18c2bea6d 100644 --- a/mhr-api/src/mhr_api/services/queue_service.py +++ b/mhr-api/src/mhr_api/services/queue_service.py @@ -35,7 +35,9 @@ class GoogleQueueService: def init_app(app): """Initialize the publisher.""" credentials = GoogleAuthService.get_credentials() - GoogleQueueService.publisher = pubsub_v1.PublisherClient(credentials=credentials) + GoogleQueueService.publisher = ( + pubsub_v1.PublisherClient(credentials=credentials) if credentials else pubsub_v1.PublisherClient() + ) project_id = str(app.config.get("GCP_PS_PROJECT_ID")) search_report_topic = str(app.config.get("GCP_PS_SEARCH_REPORT_TOPIC")) registration_report_topic = str(app.config.get("GCP_PS_REGISTRATION_REPORT_TOPIC")) diff --git a/mhr-api/tests/unit/services/test_auth_service.py b/mhr-api/tests/unit/services/test_auth_service.py index 7af631360..a42c69efa 100644 --- a/mhr-api/tests/unit/services/test_auth_service.py +++ b/mhr-api/tests/unit/services/test_auth_service.py @@ -19,30 +19,48 @@ from flask import current_app from mhr_api.services.gcp_auth.auth_service import GoogleAuthService +from mhr_api.utils.logging import logger def test_get_token(session, client, jwt): """Assert that the configuration to get a google storage token works as expected (no exceptions).""" token = GoogleAuthService.get_token() - print(token) - assert token + if current_app.config.get("GOOGLE_DEFAULT_SA"): + logger.debug(token) + assert token + else: + assert not token def test_get_credentials(session, client, jwt): """Assert that the configuration to get a google storage token works as expected (no exceptions).""" credentials = GoogleAuthService.get_credentials() - assert credentials + if current_app.config.get("GOOGLE_DEFAULT_SA"): + assert credentials + assert credentials.token + assert credentials.service_account_email + else: + assert not credentials -def test_security_account(session, client, jwt): +def test_get_cs_signed_credentials(session, client, jwt): + """Assert that the configuration to get a google storage token works as expected (no exceptions).""" + if current_app.config.get("GOOGLE_DEFAULT_SA"): + credentials = GoogleAuthService.get_cs_signed_credentials() + assert credentials + assert credentials.token + assert credentials.service_account_email + + +def test_service_account(session, client, jwt): """Assert that the configuration to get the GCP service account from the environment works as expected.""" decoded_sa = None encoded_sa: bytes = None - default_sa = os.getenv('GOOGLE_DEFAULT_SERVICE_ACCOUNT') + default_sa = current_app.config.get("GOOGLE_DEFAULT_SA") # os.getenv('GOOGLE_DEFAULT_SA') if default_sa: encoded_sa = bytes(default_sa, 'utf-8') if not encoded_sa and os.getenv('GCP_CS_SA_CLIENT_ID'): # local env testing only - current_app.logger.info('No GOOGLE_DEFAULT_SERVICE_ACCOUNT env var.') + logger.info('No GOOGLE_DEFAULT_SERVICE_ACCOUNT env var.') sa_project_id = os.getenv('GCP_CS_PROJECT_ID') sa_client_email = os.getenv('GCP_CS_SA_CLIENT_EMAIL') sa_client_id = os.getenv('GCP_CS_SA_CLIENT_ID') @@ -62,12 +80,12 @@ def test_security_account(session, client, jwt): 'client_x509_cert_url': sa_cert_url } encoded_sa = base64.b64encode(json.dumps(service_account_info).encode('utf-8')) - current_app.logger.debug(encoded_sa) + logger.info(encoded_sa) if default_sa: assert encoded_sa decoded_sa = json.loads(base64.b64decode(encoded_sa.decode('utf-8'))) - # current_app.logger.debug(decoded_sa) + logger.info(f"email={decoded_sa.get('client_email')} project={decoded_sa.get('project_id')}") assert decoded_sa assert decoded_sa.get('type') assert decoded_sa.get('project_id') @@ -79,3 +97,30 @@ def test_security_account(session, client, jwt): assert decoded_sa.get('token_uri') assert decoded_sa.get('auth_provider_x509_cert_url') assert decoded_sa.get('client_x509_cert_url') + + +def test_new_sa_key(session, client, jwt): + """Assert that storing sa info in the env var format works as expected.""" + sa_data = None + with open("tests/unit/services/test-sa.json", "r") as data_file: + text_data = data_file.read() + data_file.close() + sa_data = json.loads(text_data) + logger.info(f"sa_data= {sa_data}") + encoded_sa: bytes = base64.b64encode(json.dumps(sa_data).encode('utf-8')) + logger.info(encoded_sa) + decoded = base64.b64decode(encoded_sa.decode('utf-8')) + # logger.info(f"decoded= {decoded}") + decoded_sa = json.loads(decoded) + logger.info(f"email={decoded_sa.get('client_email')} project={decoded_sa.get('project_id')}") + assert decoded_sa + assert decoded_sa.get('type') + assert decoded_sa.get('project_id') + assert decoded_sa.get('private_key_id') + assert decoded_sa.get('private_key') + assert decoded_sa.get('client_email') + assert decoded_sa.get('client_id') + assert decoded_sa.get('auth_uri') + assert decoded_sa.get('token_uri') + assert decoded_sa.get('auth_provider_x509_cert_url') + assert decoded_sa.get('client_x509_cert_url') diff --git a/mhr-api/tests/unit/services/test_notify_service.py b/mhr-api/tests/unit/services/test_notify_service.py index 70e48e1d6..f54c6d76b 100644 --- a/mhr-api/tests/unit/services/test_notify_service.py +++ b/mhr-api/tests/unit/services/test_notify_service.py @@ -119,14 +119,14 @@ def test_review_notify(session, jwt, desc, has_env_var, has_email, status, appro if not has_env_var: current_app.config.update(NOTIFY_REVIEW_CONFIG="") reg_data = copy.deepcopy(TEST_TRANSFER) - if not has_email: + if not has_email or desc in ("Approved valid", "Declined valid"): del reg_data["submittingParty"]["emailAddress"] notify: Notify = Notify(**{"review": True}) n_status = HTTPStatus.OK if approved: n_status = notify.send_review_approved(reg_data, "verify_url") else: - n_status = notify.send_review_declined(reg_data, "declined reason here") + n_status = notify.send_review_declined(reg_data, "declined reason here", "rejection_url") if env_var: current_app.config.update(NOTIFY_REVIEW_CONFIG=env_var) assert n_status == status diff --git a/mhr-api/tests/unit/services/test_queue_service.py b/mhr-api/tests/unit/services/test_queue_service.py index 5176794ca..9768f6d44 100644 --- a/mhr-api/tests/unit/services/test_queue_service.py +++ b/mhr-api/tests/unit/services/test_queue_service.py @@ -46,19 +46,19 @@ def test_publish_search_report(session): """Assert that enqueuing/publishing a search report event works as expected (no exception thrown).""" payload = TEST_PAYLOAD - apikey = current_app.config.get('SUBSCRIPTION_API_KEY') - if apikey: + if not is_ci_testing() and current_app.config.get("SUBSCRIPTION_API_KEY"): + apikey = current_app.config.get('SUBSCRIPTION_API_KEY') payload['apikey'] = apikey - GoogleQueueService().publish_search_report(payload) + GoogleQueueService().publish_search_report(payload) def test_publish_registration_report(session): """Assert that enqueuing/publishing a registration report event works as expected (no exception thrown).""" payload = TEST_PAYLOAD_REGISTRATION - apikey = current_app.config.get('SUBSCRIPTION_API_KEY') - if apikey: + if not is_ci_testing() and current_app.config.get("SUBSCRIPTION_API_KEY"): + apikey = current_app.config.get('SUBSCRIPTION_API_KEY') payload['apikey'] = apikey - GoogleQueueService().publish_registration_report(payload) + GoogleQueueService().publish_registration_report(payload) def test_publish_document_rec(session): @@ -78,4 +78,6 @@ def test_publish_pay_completion(session): def is_ci_testing() -> bool: """Check unit test environment: exclude pub/sub for CI testing.""" + if not current_app.config.get("GOOGLE_DEFAULT_SA"): + return True return current_app.config.get("DEPLOYMENT_ENV", "testing") == "testing" diff --git a/mhr-api/tests/unit/services/test_storage_service.py b/mhr-api/tests/unit/services/test_storage_service.py index 90c28b4a2..e9c6b692f 100644 --- a/mhr-api/tests/unit/services/test_storage_service.py +++ b/mhr-api/tests/unit/services/test_storage_service.py @@ -33,6 +33,8 @@ def test_cs_save_search_document(session): """Assert that saving a search bucket document to google cloud storage works as expected.""" + if is_ci_testing(): + return raw_data = None with open(TEST_DATAFILE, 'rb') as data_file: raw_data = data_file.read() @@ -44,6 +46,8 @@ def test_cs_save_search_document(session): def test_cs_save_registration_document(session): """Assert that saving a registration verification statement to google cloud storage works as expected.""" + if is_ci_testing(): + return raw_data = None with open(TEST_REGISTRATION_DATAFILE, 'rb') as data_file: raw_data = data_file.read() @@ -57,6 +61,8 @@ def test_cs_save_registration_document(session): def test_cs_get_search_document(session): """Assert that getting a search bucket document from google cloud storage works as expected.""" + if is_ci_testing(): + return raw_data = GoogleStorageService.get_document(TEST_SAVE_DOC_NAME, DocumentTypes.SEARCH_RESULTS) assert raw_data assert len(raw_data) > 0 @@ -67,6 +73,8 @@ def test_cs_get_search_document(session): def test_cs_get_registration_document(session): """Assert that getting a registration verification statement from google cloud storage works as expected.""" + if is_ci_testing(): + return raw_data = GoogleStorageService.get_document(TEST_REGISTRATION_SAVE_DOC_NAME, DocumentTypes.REGISTRATION) assert raw_data assert len(raw_data) > 0 @@ -77,6 +85,8 @@ def test_cs_get_registration_document(session): def test_cs_get_registration_document_link(session): """Assert that getting a document link from google cloud storage works as expected.""" + if is_ci_testing(): + return download_link = GoogleStorageService.get_document_link(TEST_REGISTRATION_SAVE_DOC_NAME, DocumentTypes.REGISTRATION, 2) @@ -85,11 +95,16 @@ def test_cs_get_registration_document_link(session): def test_cs_delete_search_document(session): """Assert that deleting a search bucket document from google cloud storage works as expected.""" + if is_ci_testing(): + return response = GoogleStorageService.delete_document(TEST_SAVE_DOC_NAME2, DocumentTypes.SEARCH_RESULTS) + assert not response def test_save_batch_registration_document(session): """Assert that saving a batch registration pdf to google cloud storage works as expected.""" + if is_ci_testing(): + return bucket: str = current_app.config.get('GCP_CS_BUCKET_ID_BATCH') current_app.logger.debug(f'Testing saving to bucket={bucket}') raw_data = None @@ -104,6 +119,8 @@ def test_save_batch_registration_document(session): def test_cs_get_batch_registration_document(session): """Assert that getting a batch registration pdf from google cloud storage works as expected.""" + if is_ci_testing(): + return raw_data = GoogleStorageService.get_document(TEST_BATCH_REGISTRATION_SAVE_DOC_NAME, DocumentTypes.BATCH_REGISTRATION) assert raw_data @@ -115,6 +132,8 @@ def test_cs_get_batch_registration_document(session): def test_cs_get_service_agreement_document(session): """Assert that getting a qualified supplier service agreement pdf from google cloud storage works as expected.""" + if is_ci_testing(): + return raw_data = GoogleStorageService.get_document(TEST_SERVICE_AGREEMENT_SAVE_DOC_NAME, DocumentTypes.SERVICE_AGREEMENT) assert raw_data @@ -122,3 +141,10 @@ def test_cs_get_service_agreement_document(session): with open(TEST_SERVICE_AGREEMENT_DATAFILE, "wb") as pdf_file: pdf_file.write(raw_data) pdf_file.close() + + +def is_ci_testing() -> bool: + """Check unit test environment: exclude pub/sub for CI testing.""" + if not current_app.config.get("GOOGLE_DEFAULT_SA"): + return True + return current_app.config.get("DEPLOYMENT_ENV", "testing") == "testing" From b55f993a16b9385c91ef8e077eaf309061b01dc4 Mon Sep 17 00:00:00 2001 From: Doug Lovett Date: Tue, 16 Jun 2026 14:32:54 -0700 Subject: [PATCH 2/3] MHR replace GCP SA keys with container ADC for GCP API calls Signed-off-by: Doug Lovett --- .../tests/unit/services/test_auth_service.py | 74 ------------------- 1 file changed, 74 deletions(-) diff --git a/mhr-api/tests/unit/services/test_auth_service.py b/mhr-api/tests/unit/services/test_auth_service.py index a42c69efa..b55f5e39b 100644 --- a/mhr-api/tests/unit/services/test_auth_service.py +++ b/mhr-api/tests/unit/services/test_auth_service.py @@ -50,77 +50,3 @@ def test_get_cs_signed_credentials(session, client, jwt): assert credentials assert credentials.token assert credentials.service_account_email - - -def test_service_account(session, client, jwt): - """Assert that the configuration to get the GCP service account from the environment works as expected.""" - decoded_sa = None - encoded_sa: bytes = None - default_sa = current_app.config.get("GOOGLE_DEFAULT_SA") # os.getenv('GOOGLE_DEFAULT_SA') - if default_sa: - encoded_sa = bytes(default_sa, 'utf-8') - if not encoded_sa and os.getenv('GCP_CS_SA_CLIENT_ID'): # local env testing only - logger.info('No GOOGLE_DEFAULT_SERVICE_ACCOUNT env var.') - sa_project_id = os.getenv('GCP_CS_PROJECT_ID') - sa_client_email = os.getenv('GCP_CS_SA_CLIENT_EMAIL') - sa_client_id = os.getenv('GCP_CS_SA_CLIENT_ID') - sa_private_key = os.getenv('GCP_CS_SA_PRIVATE_KEY') - sa_private_key_id = os.getenv('GCP_CS_SA_PRIVATE_KEY_ID') - sa_cert_url = os.getenv('GCP_CS_SA_CERT_URL') - service_account_info = { - 'type': 'service_account', - 'project_id': sa_project_id, - 'private_key_id': sa_private_key_id, - 'private_key': str(sa_private_key).replace('\\n', '\n'), - 'client_email': sa_client_email, - 'client_id': sa_client_id, - 'auth_uri': 'https://accounts.google.com/o/oauth2/auth', - 'token_uri': 'https://oauth2.googleapis.com/token', - 'auth_provider_x509_cert_url': 'https://www.googleapis.com/oauth2/v1/certs', - 'client_x509_cert_url': sa_cert_url - } - encoded_sa = base64.b64encode(json.dumps(service_account_info).encode('utf-8')) - logger.info(encoded_sa) - - if default_sa: - assert encoded_sa - decoded_sa = json.loads(base64.b64decode(encoded_sa.decode('utf-8'))) - logger.info(f"email={decoded_sa.get('client_email')} project={decoded_sa.get('project_id')}") - assert decoded_sa - assert decoded_sa.get('type') - assert decoded_sa.get('project_id') - assert decoded_sa.get('private_key_id') - assert decoded_sa.get('private_key') - assert decoded_sa.get('client_email') - assert decoded_sa.get('client_id') - assert decoded_sa.get('auth_uri') - assert decoded_sa.get('token_uri') - assert decoded_sa.get('auth_provider_x509_cert_url') - assert decoded_sa.get('client_x509_cert_url') - - -def test_new_sa_key(session, client, jwt): - """Assert that storing sa info in the env var format works as expected.""" - sa_data = None - with open("tests/unit/services/test-sa.json", "r") as data_file: - text_data = data_file.read() - data_file.close() - sa_data = json.loads(text_data) - logger.info(f"sa_data= {sa_data}") - encoded_sa: bytes = base64.b64encode(json.dumps(sa_data).encode('utf-8')) - logger.info(encoded_sa) - decoded = base64.b64decode(encoded_sa.decode('utf-8')) - # logger.info(f"decoded= {decoded}") - decoded_sa = json.loads(decoded) - logger.info(f"email={decoded_sa.get('client_email')} project={decoded_sa.get('project_id')}") - assert decoded_sa - assert decoded_sa.get('type') - assert decoded_sa.get('project_id') - assert decoded_sa.get('private_key_id') - assert decoded_sa.get('private_key') - assert decoded_sa.get('client_email') - assert decoded_sa.get('client_id') - assert decoded_sa.get('auth_uri') - assert decoded_sa.get('token_uri') - assert decoded_sa.get('auth_provider_x509_cert_url') - assert decoded_sa.get('client_x509_cert_url') From d84f24e8162795ccdbebc99b6247b76a676c0810 Mon Sep 17 00:00:00 2001 From: Doug Lovett Date: Wed, 17 Jun 2026 08:57:55 -0700 Subject: [PATCH 3/3] Assets payment tweak email notification set up. Signed-off-by: Doug Lovett --- jobs/permanent/assets-payment/src/assets_payment/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jobs/permanent/assets-payment/src/assets_payment/config.py b/jobs/permanent/assets-payment/src/assets_payment/config.py index 7313b1f3d..6289ddae9 100644 --- a/jobs/permanent/assets-payment/src/assets_payment/config.py +++ b/jobs/permanent/assets-payment/src/assets_payment/config.py @@ -37,6 +37,8 @@ class Config(BaseConfig): AUTH_SVC_URL = f"{AUTH_API_URL + AUTH_API_VERSION}" PAYMENT_SVC_URL = f"{PAY_API_URL + PAY_API_VERSION}" NOTIFY_SVC_URL = f"{NOTIFY_API_URL + NOTIFY_API_VERSION}" + if not NOTIFY_SVC_URL.endswith("/notify"): + NOTIFY_SVC_URL += "/notify" JWT_OIDC_TOKEN_URL = os.getenv("JWT_OIDC_TOKEN_URL") # service accounts