Rendered Template
job_flow_overrides
("{'Name': 'vd-uat-viooh-exchange-trade-summary_v1', 'LogUri': "
"'s3n://viooh-emr-logs-lab-cn/bigdata/emr/', 'ReleaseLabel': 'emr-5.35.0', "
"'Instances': {'InstanceFleets': [{'Name': 'Masterfleet', "
"'InstanceFleetType': 'MASTER', 'TargetOnDemandCapacity': 1, "
"'InstanceTypeConfigs': [{'InstanceType': 'm5.xlarge'}, {'InstanceType': "
"'m5.2xlarge'}, {'InstanceType': 'm5.4xlarge'}]}, {'Name': 'Corefleet', "
"'InstanceFleetType': 'CORE', 'TargetOnDemandCapacity': 5, "
"'InstanceTypeConfigs': [{'InstanceType': 'm5.xlarge'}, {'InstanceType': "
"'m5.2xlarge'}, {'InstanceType': 'm5.4xlarge'}]}], 'Ec2KeyName': "
"'data.engineer.lab', 'KeepJobFlowAliveWhenNoSteps': False, "
"'TerminationProtected': False, 'Ec2SubnetId': 'subnet-09a80b42d0e6dedd8', "
"'EmrManagedMasterSecurityGroup': 'sg-0da002beeed87fb1a', "
"'EmrManagedSlaveSecurityGroup': 'sg-0f1a0b7517309fcf0', "
"'ServiceAccessSecurityGroup': 'sg-009528c399781cc69'}, 'BootstrapActions': "
"[], 'Applications': [{'Name': 'Spark'}, {'Name': 'Hadoop'}, {'Name': "
"'Ganglia'}, {'Name': 'Zeppelin'}], 'VisibleToAllUsers': True, 'Steps': "
"[{'Name': 'Setup Hadoop Debugging', 'ActionOnFailure': 'TERMINATE_JOB_FLOW', "
"'HadoopJarStep': {'Jar': 'command-runner.jar', 'Args': "
"['state-pusher-script']}}, {'Name': 'SupplyDenormalizationJob', "
"'ActionOnFailure': 'TERMINATE_CLUSTER', 'HadoopJarStep': {'Args': "
"['spark-submit', '--deploy-mode', 'cluster', '--master', 'yarn', '--conf', "
"'spark.yarn.submit.waitAppCompletion=true', '--conf', "
"'spark.driver.memory=6g', '--conf', 'spark.app.env=uat', '--conf', "
"'spark.dag.execution.time=2024-06-06T00:00:00+00:00', '--class', "
"'com.viooh.smex.supply.denormalization.SupplyDataDenormalization', "
"'s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar'], "
"'Jar': 'command-runner.jar'}}, {'Name': 'DemandDenormalizationJob', "
"'ActionOnFailure': 'TERMINATE_CLUSTER', 'HadoopJarStep': {'Args': "
"['spark-submit', '--deploy-mode', 'cluster', '--master', 'yarn', '--conf', "
"'spark.yarn.submit.waitAppCompletion=true', '--conf', "
"'spark.driver.memory=6g', '--conf', 'spark.app.env=uat', '--conf', "
"'spark.dag.execution.time=2024-06-06T00:00:00+00:00', '--class', "
"'com.viooh.smex.demand.denormalization.DemandDataDenormalization', "
"'s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar'], "
"'Jar': 'command-runner.jar'}}, {'Name': 'DealSyncDenormalizationJob', "
"'ActionOnFailure': 'TERMINATE_CLUSTER', 'HadoopJarStep': {'Args': "
"['spark-submit', '--deploy-mode', 'cluster', '--master', 'yarn', '--conf', "
"'spark.yarn.submit.waitAppCompletion=true', '--conf', "
"'spark.driver.memory=6g', '--conf', 'spark.app.env=uat', '--conf', "
"'spark.dag.execution.time=2024-06-06T00:00:00+00:00', '--class', "
"'com.viooh.smex.dealsync.denormalization.DealSyncDenormalization', "
"'s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar'], "
"'Jar': 'command-runner.jar'}}, {'Name': 'ModerationDenormalizationJob', "
"'ActionOnFailure': 'TERMINATE_CLUSTER', 'HadoopJarStep': {'Args': "
"['spark-submit', '--deploy-mode', 'cluster', '--master', 'yarn', '--conf', "
"'spark.yarn.submit.waitAppCompletion=true', '--conf', "
"'spark.driver.memory=6g', '--conf', 'spark.app.env=uat', '--conf', "
"'spark.dag.execution.time=2024-06-06T00:00:00+00:00', '--class', "
"'com.viooh.smex.moderation.denormalization.ModerationDenormalization', "
"'--jars', '/usr/lib/spark/external/lib/spark-avro.jar', "
"'s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar'], "
"'Jar': 'command-runner.jar'}}, {'Name': 'DealDenormalizationJob', "
"'ActionOnFailure': 'TERMINATE_CLUSTER', 'HadoopJarStep': {'Args': "
"['spark-submit', '--deploy-mode', 'cluster', '--master', 'yarn', '--conf', "
"'spark.yarn.submit.waitAppCompletion=true', '--conf', "
"'spark.driver.memory=6g', '--conf', 'spark.app.env=uat', '--conf', "
"'spark.dag.execution.time=2024-06-06T00:00:00+00:00', '--class', "
"'com.viooh.smex.deal.denormalization.DealDenormalization', '--jars', "
"'/usr/lib/spark/external/lib/spark-avro.jar', "
"'s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar'], "
"'Jar': 'command-runner.jar'}}, {'Name': 'ExchangeTradeSummaryJob', "
"'ActionOnFailure': 'TERMINATE_CLUSTER', 'HadoopJarStep': {'Args': "
"['spark-submit', '--deploy-mode', 'cluster', '--master', 'yarn', '--conf', "
"'spark.yarn.submit.waitAppCompletion=true', '--conf', "
"'spark.driver.memory=6g', '--conf', "
"'spark.serializer=org.apache.spark.serializer.KryoSerializer', '--conf', "
"'spark.task.maxFailures=20', '--conf', 'spark.yarn.maxAppAttempts=10', "
"'--conf', 'spark.stage.maxConsecutiveAttempts=20', '--conf', "
"'spark.app.config.key=exchange-trade-summary', '--conf', "
"'spark.app.env=uat', '--conf', "
"'spark.dag.execution.time=2024-06-06T00:00:00+00:00', '--class', "
"'com.viooh.smex.trade.summary.ExchangeTradeSummary', "
"'s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar'], "
"'Jar': 'command-runner.jar'}}, {'Name': 'CopySmartExchangeMediaOwnerReport', "
"'ActionOnFailure': 'TERMINATE_JOB_FLOW', 'HadoopJarStep': {'Jar': "
"'s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar', "
"'Args': "
"['s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh', "
"'uat', '2024-06-06', '1', 'mediaowner-trade-report', "
"'trade-summary-reports', 'v1', 'csv', 'cn-northwest-1']}}, {'Name': "
"'CopyExchangeLiteMediaOwnerReport', 'ActionOnFailure': 'TERMINATE_JOB_FLOW', "
"'HadoopJarStep': {'Jar': "
"'s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar', "
"'Args': "
"['s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh', "
"'uat', '2024-06-06', '0', 'exchange-lite-mediaowner-trade-report', "
"'trade-summary-reports', 'v1', 'csv', 'cn-northwest-1']}}, {'Name': "
"'CopySmartExchangeFrameLevelMediaOwnerReport', 'ActionOnFailure': "
"'TERMINATE_JOB_FLOW', 'HadoopJarStep': {'Jar': "
"'s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar', "
"'Args': "
"['s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh', "
"'uat', '2024-06-06', '1', 'frame-level-mediaowner-trade-report', "
"'frame-level-media-owner-reports', 'v1', 'csv', 'cn-northwest-1']}}, "
"{'Name': 'CopyExchangeLiteFrameLevelMediaOwnerReport', 'ActionOnFailure': "
"'TERMINATE_JOB_FLOW', 'HadoopJarStep': {'Jar': "
"'s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar', "
"'Args': "
"['s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh', "
"'uat', '2024-06-06', '0', "
"'frame-level-exchange-lite-mediaowner-trade-report', "
"'frame-level-media-owner-reports', 'v1', 'csv', 'cn-northwest-1']}}, "
"{'Name': 'CopySmartExchangeMediaOwnerReportV2', 'ActionOnFailure': "
"'TERMINATE_JOB_FLOW', 'HadoopJarStep': {'Jar': "
"'s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar', "
"'Args': "
"['s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh', "
"'uat', '2024-06-06', '1', 'mediaowner-trade-report', "
"'trade-summary-reports', 'v2', 'json', 'cn-northwest-1']}}, {'Name': "
"'CopyExchangeLiteMediaOwnerReportV2', 'ActionOnFailure': "
"'TERMINATE_JOB_FLOW', 'HadoopJarStep': {'Jar': "
"'s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar', "
"'Args': "
"['s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh', "
"'uat', '2024-06-06', '0', 'exchange-lite-mediaowner-trade-report', "
"'trade-summary-reports', 'v2', 'json', 'cn-northwest-1']}}, {'Name': "
"'ExportVioohHourLevelReportToDbV2', 'ActionOnFailure': 'TERMINATE_JOB_FLOW', "
"'HadoopJarStep': {'Jar': "
"'s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar', "
"'Args': "
"['s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh', "
"'uat', '0.0.2', 'db-import-viooh-hourly-ui-report-v2', '1', "
"'2024-06-06T00:00:00+00:00', 'viooh-spark-artifacts-lab-cn', "
"'scripts/exchange-trade-summary/sspui_reporting', 'deal', "
"'cn-northwest-1']}}, {'Name': 'ExportVioohHourlyCreativeLevelReportToDb', "
"'ActionOnFailure': 'TERMINATE_JOB_FLOW', 'HadoopJarStep': {'Jar': "
"'s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar', "
"'Args': "
"['s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh', "
"'uat', '0.0.2', 'db-import-viooh-hourly-creative-report', '1', "
"'2024-06-06T00:00:00+00:00', 'viooh-spark-artifacts-lab-cn', "
"'scripts/exchange-trade-summary/sspui_reporting', 'creative', "
"'cn-northwest-1']}}], 'JobFlowRole': 'DeEmrInstanceRole', 'ServiceRole': "
"'DeEmrServiceRole', 'SecurityConfiguration': 'bigdata_emr_cn_sec_conf', "
"'Tags': [{'Key': 'environment', 'Value': 'uat'}, {'Key': 'role', 'Value': "
"'analysis'}, {'Key': 'application', 'Value': 'emr'}, {'Key': 'project', "
"'Value': 'exchange-reporting'}, {'Key': 'TerminationPolicy', 'Value': "
"'OFFICE-HOURS'}, {'Key': 'ContactEmail', 'Value': 'viooh.data@viooh.com'}, "
"{'Key': 'team', 'Value': 'bigdata'}, {'Key': 'Name', 'Value': "
"'vd-uat-viooh-exchange-trade-summary_v1'}], 'Configurations': "
"[{'Classification': 'yarn-site', 'Properties': "
"{'yarn.resourcemanager.am.max-attempts': '2'}}]}")