diff --git a/Backend/emr-cluster-ops/src/conf/common/common-tags.json b/Backend/emr-cluster-ops/src/conf/common/common-tags.json index affdf78..ee54894 100644 --- a/Backend/emr-cluster-ops/src/conf/common/common-tags.json +++ b/Backend/emr-cluster-ops/src/conf/common/common-tags.json @@ -1,38 +1,18 @@ [ { - "Key": "intuit:billing:component", - "Value": "__role__" + "Key": "component", + "Value": "EMR" }, { - "Key": "intuit:billing:user-app", - "Value": "__name__" + "Key": "Org", + "Value": "Intuit" }, { - "Key": "intuit:billing:appenv", - "Value": "prd" - }, - { - "Key": "bu", - "Value": "sbseg" - }, - { - "Key": "env", - "Value": "prod" + "Key": "Team", + "Value": "Platform" }, { "Key": "app", - "Value": "__name__" - }, - { - "Key": "Name", - "Value": "__name__-EMR" - }, - { - "Key": "type", - "Value": "__type__" - }, - { - "Key": "ami:restack:date", - "Value": "__restack__" + "Value": "EMR" } ] \ No newline at end of file diff --git a/Backend/emr-cluster-ops/src/conf/example_account/common-tags.json b/Backend/emr-cluster-ops/src/conf/example_account/common-tags.json new file mode 100644 index 0000000..ee54894 --- /dev/null +++ b/Backend/emr-cluster-ops/src/conf/example_account/common-tags.json @@ -0,0 +1,18 @@ +[ + { + "Key": "component", + "Value": "EMR" + }, + { + "Key": "Org", + "Value": "Intuit" + }, + { + "Key": "Team", + "Value": "Platform" + }, + { + "Key": "app", + "Value": "EMR" + } +] \ No newline at end of file diff --git a/Backend/emr-cluster-ops/src/conf/example_account/configurations.json b/Backend/emr-cluster-ops/src/conf/example_account/configurations.json new file mode 100644 index 0000000..1f80165 --- /dev/null +++ b/Backend/emr-cluster-ops/src/conf/example_account/configurations.json @@ -0,0 +1,59 @@ +[ + { + "Classification": "hive-site", + "Properties": { + "hive.tez.container.size": "3584", + "hive.exec.reducers.max": "10009", + "hive.driver.parallel.compilation": "true", + "hive.exec.reducers.bytes.per.reducer": "158998272", + "hive.blobstore.optimizations.enabled": "false", + "hive.cbo.enable": "true", + "hive.compute.query.using.stats": "true", + "hive.exec.dynamic.partition ": " true", + "hive.exec.dynamic.partition.mode": "nonstrict", + "hive.exec.parallel": "true", + "hive.exec.stagingdir": "/tmp/hive/.staging", + "hive.execution.engine": "tez", + "hive.optimize.ppd": "true", + "hive.server2.logging.operation.level": "VERBOSE", + "hive.stats.dbclass": "fs", + "hive.stats.fetch.column.stats": "true", + "hive.stats.fetch.partition.stats": "true", + "hive.default.fileformat": "Parquet", + "hive.default.fileformat.managed": "Parquet" + } + }, + { + "Classification": "tez-site", + "Properties": { + "tez.am.resource.memory.mb": "3584", + "tez.container.max.java.heap.fraction": "0.8", + "tez.grouping.max-size": "573741824", + "tez.grouping.min-size": "67108864" + } + }, + { + "Classification": "yarn-site", + "Properties": { + "yarn.timeline-service.http-authentication.type": "simple", + "yarn.nodemanager.localizer.cache.target-size-mb": "4096", + "yarn.nodemanager.localizer.cache.cleanup.interval-ms": "300000", + "yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage": "99.99" + } + }, + + { + "Classification": "hiveserver2-site", + "Properties": { + "hive.security.authorization.sqlstd.confwhitelist.append": "tez\\.grouping\\..*", + "hive.async.log.enabled": "false", + "hive.server2.in.place.progress": "true" + } + }, + { + "Classification": "emrfs-site", + "Properties": { + "fs.s3.maxConnections": "50000" + } + } +] diff --git a/Backend/emr-cluster-ops/src/util/emrlib.py b/Backend/emr-cluster-ops/src/util/emrlib.py index 1f3eacf..becd057 100644 --- a/Backend/emr-cluster-ops/src/util/emrlib.py +++ b/Backend/emr-cluster-ops/src/util/emrlib.py @@ -299,7 +299,14 @@ def get_emr_cluster_settings(user_inputs: dict, config_file: str, metadata_file: # BOOTSTRAP ACTIONS are going to be a list, similar to steps. # required settings/configurations should be inserted at the start of the src_dir = os.path.dirname(os.path.dirname(__file__)) - emr_sw_config_file = f'{src_dir}/conf/common/configurations.json' + try: + account_id = get_account_id() + emr_sw_config_file = f'{src_dir}/conf/{account_id}/configurations.json' + if not os.path.isfile(emr_sw_config_file): + emr_sw_config_file = f'{src_dir}/conf/common/configurations.json' + except Exception as error: + logger.error(f"Unable to parse common tag file {str(error)}") + raise with open(emr_sw_config_file) as config_json: configuration = json.load(config_json) @@ -567,6 +574,20 @@ def tag_list(role_config: dict, metadata_config: dict, cluster_name: str): Returns: Returns list of dict, each dict is an ec2 tag object. """ + + try: + account_id = get_account_id() + src_dir = os.path.dirname(os.path.dirname(__file__)) + tag_file = f'{src_dir}/conf/{account_id}/common-tags.json' + if not os.path.isfile(tag_file): + tag_file = f'{src_dir}/conf/common/common-tags.json' + + with open(tag_file) as f: + tags = json.load(f) + except Exception as error: + logger.error(f"Unable to parse common tag file {str(error)}") + raise + role_tags = [] for tag in role_config.get('tags', []): @@ -1102,3 +1123,24 @@ def get_network_interface_association(sg_group_id: str): else: return False +def get_account_id(): + """Get aws account id + + Raises: + botocore.exceptions.ClientError: Raise exception for client errors + + Returns: + [str]: [account id] + """ + sts = boto3.client('sts') + account_id = None + + try: + response = sts.get_caller_identity() + account_id = response.get('Account') + + except Exception as exception: + log_error(exception) + raise botocore.exceptions.ClientError + + return account_id \ No newline at end of file