diff --git a/jobs/permanent/assets-payment/src/assets_payment/config.py b/jobs/permanent/assets-payment/src/assets_payment/config.py index 7313b1f3d..6289ddae9 100644 --- a/jobs/permanent/assets-payment/src/assets_payment/config.py +++ b/jobs/permanent/assets-payment/src/assets_payment/config.py @@ -37,6 +37,8 @@ class Config(BaseConfig): AUTH_SVC_URL = f"{AUTH_API_URL + AUTH_API_VERSION}" PAYMENT_SVC_URL = f"{PAY_API_URL + PAY_API_VERSION}" NOTIFY_SVC_URL = f"{NOTIFY_API_URL + NOTIFY_API_VERSION}" + if not NOTIFY_SVC_URL.endswith("/notify"): + NOTIFY_SVC_URL += "/notify" JWT_OIDC_TOKEN_URL = os.getenv("JWT_OIDC_TOKEN_URL") # service accounts diff --git a/mhr-api/devops/vaults.gcp.env b/mhr-api/devops/vaults.gcp.env index 00fda8d00..7a0cf667e 100644 --- a/mhr-api/devops/vaults.gcp.env +++ b/mhr-api/devops/vaults.gcp.env @@ -20,7 +20,6 @@ PAY_API_URL="op://API/$APP_ENV/pay-api/PAY_API_URL" PAY_API_VERSION="op://API/$APP_ENV/pay-api/PAY_API_VERSION" REPORT_API_URL="op://ppr/$APP_ENV/mhr-api/REPORT_SVC_URL" REPORT_API_AUDIENCE="op://ppr/$APP_ENV/mhr-api/REPORT_API_AUDIENCE" -GOOGLE_DEFAULT_SA="op://buckets/$APP_ENV/mhr-api/GOOGLE_DEFAULT_SA" GCP_CS_PROJECT_ID="op://ppr/$APP_ENV/mhr-api/GCP_PS_PROJECT_ID" GCP_CS_SA_SCOPES="op://buckets/$APP_ENV/mhr-api/GCP_CS_SA_SCOPES" GCP_CS_BUCKET_ID="op://buckets/$APP_ENV/mhr-api/GCP_CS_BUCKET_ID" diff --git a/mhr-api/src/mhr_api/services/document_storage/storage_service.py b/mhr-api/src/mhr_api/services/document_storage/storage_service.py index d16886251..1f43ef4ef 100644 --- a/mhr-api/src/mhr_api/services/document_storage/storage_service.py +++ b/mhr-api/src/mhr_api/services/document_storage/storage_service.py @@ -130,7 +130,7 @@ def __get_bucket_id(cls, doc_type: str = None): def __call_cs_api(cls, method: str, name: str, data=None, doc_type: str = None): """Call the Cloud Storage API.""" credentials = GoogleAuthService.get_credentials() - storage_client = storage.Client(credentials=credentials) + storage_client = storage.Client(credentials=credentials) if credentials else storage.Client() bucket = storage_client.bucket(cls.__get_bucket_id(doc_type)) blob = bucket.blob(name) if method == HTTP_POST: @@ -147,13 +147,20 @@ def __call_cs_api(cls, method: str, name: str, data=None, doc_type: str = None): @classmethod def __call_cs_api_link(cls, name: str, data=None, doc_type: str = None, available_days: int = 1): """Call the Cloud Storage API, returning a time-limited download link.""" - credentials = GoogleAuthService.get_credentials() + credentials = GoogleAuthService.get_cs_signed_credentials() + if not credentials: + logger.warning(f"No credentials to create signed storage link for {name}") + return "" storage_client = storage.Client(credentials=credentials) bucket = storage_client.bucket(cls.__get_bucket_id(doc_type)) blob = bucket.blob(name) if data: blob.upload_from_string(data=data, content_type=CONTENT_TYPE_PDF) url = blob.generate_signed_url( - version="v4", expiration=datetime.timedelta(days=available_days, hours=0, minutes=0), method="GET" + version="v4", + expiration=datetime.timedelta(days=available_days, hours=0, minutes=0), + method="GET", + service_account_email=credentials.service_account_email, + access_token=credentials.token, ) return url diff --git a/mhr-api/src/mhr_api/services/gcp_auth/auth_service.py b/mhr-api/src/mhr_api/services/gcp_auth/auth_service.py index e52022892..5e494c1cf 100644 --- a/mhr-api/src/mhr_api/services/gcp_auth/auth_service.py +++ b/mhr-api/src/mhr_api/services/gcp_auth/auth_service.py @@ -37,10 +37,6 @@ class GoogleAuthService(AuthService): # pylint: disable=too-few-public-methods service_account_info = None credentials = None - # Use service account env var if available. - if gcp_auth_key: - sa_bytes = bytes(gcp_auth_key, "utf-8") - service_account_info = json.loads(base64.b64decode(sa_bytes.decode("utf-8"))) @staticmethod def init_app(app): @@ -50,10 +46,18 @@ def init_app(app): if GoogleAuthService.gcp_auth_key: sa_bytes = bytes(GoogleAuthService.gcp_auth_key, "utf-8") GoogleAuthService.service_account_info = json.loads(base64.b64decode(sa_bytes.decode("utf-8"))) + GoogleAuthService.credentials = service_account.Credentials.from_service_account_info( + GoogleAuthService.service_account_info, scopes=GoogleAuthService.gcp_sa_scopes + ) + else: + logger.info("auth_service.init_app no SA info.") @classmethod def get_token(cls): """Generate an OAuth access token with cloud storage access.""" + if not cls.gcp_auth_key or not cls.service_account_info: + return None + if cls.credentials is None: cls.credentials = service_account.Credentials.from_service_account_info( cls.service_account_info, scopes=cls.gcp_sa_scopes @@ -77,9 +81,25 @@ def get_report_api_token(cls): @classmethod def get_credentials(cls): """Generate GCP auth credentials to pass to a GCP client.""" + if not cls.gcp_auth_key or not cls.service_account_info: + return None + if cls.credentials is None: cls.credentials = service_account.Credentials.from_service_account_info( cls.service_account_info, scopes=cls.gcp_sa_scopes ) - logger.info("Call successful: obtained credentials.") + logger.debug("Call successful: obtained credentials.") return cls.credentials + + @classmethod + def get_cs_signed_credentials(cls): + """Extra steps for ADC cloud storage signed url - requires cert to sign.""" + if cls.gcp_auth_key and cls.service_account_info: + return cls.get_credentials() + + # Load default credentials + credentials, project = google.auth.default() # pylint: disable=unused-variable; gcp api response + # Refresh credentials to ensure an access token is available + auth_request = google.auth.transport.requests.Request() + credentials.refresh(auth_request) + return credentials diff --git a/mhr-api/src/mhr_api/services/queue_service.py b/mhr-api/src/mhr_api/services/queue_service.py index 8dc3eea65..18c2bea6d 100644 --- a/mhr-api/src/mhr_api/services/queue_service.py +++ b/mhr-api/src/mhr_api/services/queue_service.py @@ -35,7 +35,9 @@ class GoogleQueueService: def init_app(app): """Initialize the publisher.""" credentials = GoogleAuthService.get_credentials() - GoogleQueueService.publisher = pubsub_v1.PublisherClient(credentials=credentials) + GoogleQueueService.publisher = ( + pubsub_v1.PublisherClient(credentials=credentials) if credentials else pubsub_v1.PublisherClient() + ) project_id = str(app.config.get("GCP_PS_PROJECT_ID")) search_report_topic = str(app.config.get("GCP_PS_SEARCH_REPORT_TOPIC")) registration_report_topic = str(app.config.get("GCP_PS_REGISTRATION_REPORT_TOPIC")) diff --git a/mhr-api/tests/unit/services/test_auth_service.py b/mhr-api/tests/unit/services/test_auth_service.py index 7af631360..b55f5e39b 100644 --- a/mhr-api/tests/unit/services/test_auth_service.py +++ b/mhr-api/tests/unit/services/test_auth_service.py @@ -19,63 +19,34 @@ from flask import current_app from mhr_api.services.gcp_auth.auth_service import GoogleAuthService +from mhr_api.utils.logging import logger def test_get_token(session, client, jwt): """Assert that the configuration to get a google storage token works as expected (no exceptions).""" token = GoogleAuthService.get_token() - print(token) - assert token + if current_app.config.get("GOOGLE_DEFAULT_SA"): + logger.debug(token) + assert token + else: + assert not token def test_get_credentials(session, client, jwt): """Assert that the configuration to get a google storage token works as expected (no exceptions).""" credentials = GoogleAuthService.get_credentials() - assert credentials + if current_app.config.get("GOOGLE_DEFAULT_SA"): + assert credentials + assert credentials.token + assert credentials.service_account_email + else: + assert not credentials -def test_security_account(session, client, jwt): - """Assert that the configuration to get the GCP service account from the environment works as expected.""" - decoded_sa = None - encoded_sa: bytes = None - default_sa = os.getenv('GOOGLE_DEFAULT_SERVICE_ACCOUNT') - if default_sa: - encoded_sa = bytes(default_sa, 'utf-8') - if not encoded_sa and os.getenv('GCP_CS_SA_CLIENT_ID'): # local env testing only - current_app.logger.info('No GOOGLE_DEFAULT_SERVICE_ACCOUNT env var.') - sa_project_id = os.getenv('GCP_CS_PROJECT_ID') - sa_client_email = os.getenv('GCP_CS_SA_CLIENT_EMAIL') - sa_client_id = os.getenv('GCP_CS_SA_CLIENT_ID') - sa_private_key = os.getenv('GCP_CS_SA_PRIVATE_KEY') - sa_private_key_id = os.getenv('GCP_CS_SA_PRIVATE_KEY_ID') - sa_cert_url = os.getenv('GCP_CS_SA_CERT_URL') - service_account_info = { - 'type': 'service_account', - 'project_id': sa_project_id, - 'private_key_id': sa_private_key_id, - 'private_key': str(sa_private_key).replace('\\n', '\n'), - 'client_email': sa_client_email, - 'client_id': sa_client_id, - 'auth_uri': 'https://accounts.google.com/o/oauth2/auth', - 'token_uri': 'https://oauth2.googleapis.com/token', - 'auth_provider_x509_cert_url': 'https://www.googleapis.com/oauth2/v1/certs', - 'client_x509_cert_url': sa_cert_url - } - encoded_sa = base64.b64encode(json.dumps(service_account_info).encode('utf-8')) - current_app.logger.debug(encoded_sa) - - if default_sa: - assert encoded_sa - decoded_sa = json.loads(base64.b64decode(encoded_sa.decode('utf-8'))) - # current_app.logger.debug(decoded_sa) - assert decoded_sa - assert decoded_sa.get('type') - assert decoded_sa.get('project_id') - assert decoded_sa.get('private_key_id') - assert decoded_sa.get('private_key') - assert decoded_sa.get('client_email') - assert decoded_sa.get('client_id') - assert decoded_sa.get('auth_uri') - assert decoded_sa.get('token_uri') - assert decoded_sa.get('auth_provider_x509_cert_url') - assert decoded_sa.get('client_x509_cert_url') +def test_get_cs_signed_credentials(session, client, jwt): + """Assert that the configuration to get a google storage token works as expected (no exceptions).""" + if current_app.config.get("GOOGLE_DEFAULT_SA"): + credentials = GoogleAuthService.get_cs_signed_credentials() + assert credentials + assert credentials.token + assert credentials.service_account_email diff --git a/mhr-api/tests/unit/services/test_notify_service.py b/mhr-api/tests/unit/services/test_notify_service.py index 70e48e1d6..f54c6d76b 100644 --- a/mhr-api/tests/unit/services/test_notify_service.py +++ b/mhr-api/tests/unit/services/test_notify_service.py @@ -119,14 +119,14 @@ def test_review_notify(session, jwt, desc, has_env_var, has_email, status, appro if not has_env_var: current_app.config.update(NOTIFY_REVIEW_CONFIG="") reg_data = copy.deepcopy(TEST_TRANSFER) - if not has_email: + if not has_email or desc in ("Approved valid", "Declined valid"): del reg_data["submittingParty"]["emailAddress"] notify: Notify = Notify(**{"review": True}) n_status = HTTPStatus.OK if approved: n_status = notify.send_review_approved(reg_data, "verify_url") else: - n_status = notify.send_review_declined(reg_data, "declined reason here") + n_status = notify.send_review_declined(reg_data, "declined reason here", "rejection_url") if env_var: current_app.config.update(NOTIFY_REVIEW_CONFIG=env_var) assert n_status == status diff --git a/mhr-api/tests/unit/services/test_queue_service.py b/mhr-api/tests/unit/services/test_queue_service.py index 5176794ca..9768f6d44 100644 --- a/mhr-api/tests/unit/services/test_queue_service.py +++ b/mhr-api/tests/unit/services/test_queue_service.py @@ -46,19 +46,19 @@ def test_publish_search_report(session): """Assert that enqueuing/publishing a search report event works as expected (no exception thrown).""" payload = TEST_PAYLOAD - apikey = current_app.config.get('SUBSCRIPTION_API_KEY') - if apikey: + if not is_ci_testing() and current_app.config.get("SUBSCRIPTION_API_KEY"): + apikey = current_app.config.get('SUBSCRIPTION_API_KEY') payload['apikey'] = apikey - GoogleQueueService().publish_search_report(payload) + GoogleQueueService().publish_search_report(payload) def test_publish_registration_report(session): """Assert that enqueuing/publishing a registration report event works as expected (no exception thrown).""" payload = TEST_PAYLOAD_REGISTRATION - apikey = current_app.config.get('SUBSCRIPTION_API_KEY') - if apikey: + if not is_ci_testing() and current_app.config.get("SUBSCRIPTION_API_KEY"): + apikey = current_app.config.get('SUBSCRIPTION_API_KEY') payload['apikey'] = apikey - GoogleQueueService().publish_registration_report(payload) + GoogleQueueService().publish_registration_report(payload) def test_publish_document_rec(session): @@ -78,4 +78,6 @@ def test_publish_pay_completion(session): def is_ci_testing() -> bool: """Check unit test environment: exclude pub/sub for CI testing.""" + if not current_app.config.get("GOOGLE_DEFAULT_SA"): + return True return current_app.config.get("DEPLOYMENT_ENV", "testing") == "testing" diff --git a/mhr-api/tests/unit/services/test_storage_service.py b/mhr-api/tests/unit/services/test_storage_service.py index 90c28b4a2..e9c6b692f 100644 --- a/mhr-api/tests/unit/services/test_storage_service.py +++ b/mhr-api/tests/unit/services/test_storage_service.py @@ -33,6 +33,8 @@ def test_cs_save_search_document(session): """Assert that saving a search bucket document to google cloud storage works as expected.""" + if is_ci_testing(): + return raw_data = None with open(TEST_DATAFILE, 'rb') as data_file: raw_data = data_file.read() @@ -44,6 +46,8 @@ def test_cs_save_search_document(session): def test_cs_save_registration_document(session): """Assert that saving a registration verification statement to google cloud storage works as expected.""" + if is_ci_testing(): + return raw_data = None with open(TEST_REGISTRATION_DATAFILE, 'rb') as data_file: raw_data = data_file.read() @@ -57,6 +61,8 @@ def test_cs_save_registration_document(session): def test_cs_get_search_document(session): """Assert that getting a search bucket document from google cloud storage works as expected.""" + if is_ci_testing(): + return raw_data = GoogleStorageService.get_document(TEST_SAVE_DOC_NAME, DocumentTypes.SEARCH_RESULTS) assert raw_data assert len(raw_data) > 0 @@ -67,6 +73,8 @@ def test_cs_get_search_document(session): def test_cs_get_registration_document(session): """Assert that getting a registration verification statement from google cloud storage works as expected.""" + if is_ci_testing(): + return raw_data = GoogleStorageService.get_document(TEST_REGISTRATION_SAVE_DOC_NAME, DocumentTypes.REGISTRATION) assert raw_data assert len(raw_data) > 0 @@ -77,6 +85,8 @@ def test_cs_get_registration_document(session): def test_cs_get_registration_document_link(session): """Assert that getting a document link from google cloud storage works as expected.""" + if is_ci_testing(): + return download_link = GoogleStorageService.get_document_link(TEST_REGISTRATION_SAVE_DOC_NAME, DocumentTypes.REGISTRATION, 2) @@ -85,11 +95,16 @@ def test_cs_get_registration_document_link(session): def test_cs_delete_search_document(session): """Assert that deleting a search bucket document from google cloud storage works as expected.""" + if is_ci_testing(): + return response = GoogleStorageService.delete_document(TEST_SAVE_DOC_NAME2, DocumentTypes.SEARCH_RESULTS) + assert not response def test_save_batch_registration_document(session): """Assert that saving a batch registration pdf to google cloud storage works as expected.""" + if is_ci_testing(): + return bucket: str = current_app.config.get('GCP_CS_BUCKET_ID_BATCH') current_app.logger.debug(f'Testing saving to bucket={bucket}') raw_data = None @@ -104,6 +119,8 @@ def test_save_batch_registration_document(session): def test_cs_get_batch_registration_document(session): """Assert that getting a batch registration pdf from google cloud storage works as expected.""" + if is_ci_testing(): + return raw_data = GoogleStorageService.get_document(TEST_BATCH_REGISTRATION_SAVE_DOC_NAME, DocumentTypes.BATCH_REGISTRATION) assert raw_data @@ -115,6 +132,8 @@ def test_cs_get_batch_registration_document(session): def test_cs_get_service_agreement_document(session): """Assert that getting a qualified supplier service agreement pdf from google cloud storage works as expected.""" + if is_ci_testing(): + return raw_data = GoogleStorageService.get_document(TEST_SERVICE_AGREEMENT_SAVE_DOC_NAME, DocumentTypes.SERVICE_AGREEMENT) assert raw_data @@ -122,3 +141,10 @@ def test_cs_get_service_agreement_document(session): with open(TEST_SERVICE_AGREEMENT_DATAFILE, "wb") as pdf_file: pdf_file.write(raw_data) pdf_file.close() + + +def is_ci_testing() -> bool: + """Check unit test environment: exclude pub/sub for CI testing.""" + if not current_app.config.get("GOOGLE_DEFAULT_SA"): + return True + return current_app.config.get("DEPLOYMENT_ENV", "testing") == "testing"