| return_value |
{'Name': 'vd-uat-viooh-exchange-trade-summary-backfill_all', 'LogUri': 's3n://viooh-emr-logs-lab-cn/bigdata/emr/', 'ReleaseLabel': 'emr-5.35.0', 'Instances': {'InstanceFleets': [{'Name': 'Masterfleet', 'InstanceFleetType': 'MASTER', 'TargetOnDemandCapacity': 1, 'InstanceTypeConfigs': [{'InstanceType': 'm5.2xlarge'}, {'InstanceType': 'm5.4xlarge'}]}, {'Name': 'Corefleet', 'InstanceFleetType': 'CORE', 'TargetOnDemandCapacity': 20, 'InstanceTypeConfigs': [{'InstanceType': 'm5.2xlarge', 'EbsConfiguration': {'EbsBlockDeviceConfigs': [{'VolumeSpecification': {'VolumeType': 'gp2', 'SizeInGB': 200}, 'VolumesPerInstance': 1}], 'EbsOptimized': True}}, {'InstanceType': 'm5.4xlarge', 'EbsConfiguration': {'EbsBlockDeviceConfigs': [{'VolumeSpecification': {'VolumeType': 'gp2', 'SizeInGB': 200}, 'VolumesPerInstance': 1}], 'EbsOptimized': True}}]}], 'Ec2KeyName': 'data.engineer.lab', 'KeepJobFlowAliveWhenNoSteps': False, 'TerminationProtected': False, 'Ec2SubnetId': 'subnet-06c5b5a18c64a0316', 'EmrManagedMasterSecurityGroup': 'sg-0da002beeed87fb1a', 'EmrManagedSlaveSecurityGroup': 'sg-0f1a0b7517309fcf0', 'ServiceAccessSecurityGroup': 'sg-009528c399781cc69'}, 'BootstrapActions': [], 'Applications': [{'Name': 'Spark'}, {'Name': 'Hadoop'}, {'Name': 'Ganglia'}, {'Name': 'Zeppelin'}], 'VisibleToAllUsers': True, 'Steps': [{'Name': 'Setup Hadoop Debugging', 'ActionOnFailure': 'TERMINATE_JOB_FLOW', 'HadoopJarStep': {'Jar': 'command-runner.jar', 'Args': ['state-pusher-script']}}, {'Name': 'SupplyDenormalizationJobBackfill', 'ActionOnFailure': 'TERMINATE_CLUSTER', 'HadoopJarStep': {'Args': ['spark-submit', '--deploy-mode', 'cluster', '--master', 'yarn', '--conf', 'spark.yarn.submit.waitAppCompletion=true', '--conf', 'spark.driver.memory=20g', '--conf', 'spark.driver.cores=5', '--conf', 'spark.serializer=org.apache.spark.serializer.KryoSerializer', '--conf', 'spark.task.maxFailures=20', '--conf', 'spark.yarn.maxAppAttempts=10', '--conf', 'spark.stage.maxConsecutiveAttempts=20', '--conf', 'spark.app.env=uat', '--conf', 'spark.dag.execution.time=2023-11-10T09:56:28.500821+00:00', '--class', 'com.viooh.smex.supply.denormalization.SupplyDataDenormalizationBackfill', '--jars', '/usr/lib/spark/external/lib/spark-avro.jar', 's3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.9.5.develop/supply-denormalization-2.9.5.develop-jar-with-dependencies.jar'], 'Jar': 'command-runner.jar'}}, {'Name': 'DemandDenormalizationJobBackfill', 'ActionOnFailure': 'TERMINATE_CLUSTER', 'HadoopJarStep': {'Args': ['spark-submit', '--deploy-mode', 'cluster', '--master', 'yarn', '--conf', 'spark.yarn.submit.waitAppCompletion=true', '--conf', 'spark.driver.memory=20g', '--conf', 'spark.driver.cores=5', '--conf', 'spark.serializer=org.apache.spark.serializer.KryoSerializer', '--conf', 'spark.task.maxFailures=20', '--conf', 'spark.yarn.maxAppAttempts=10', '--conf', 'spark.stage.maxConsecutiveAttempts=20', '--conf', 'spark.app.env=uat', '--conf', 'spark.dag.execution.time=2023-11-10T09:56:28.500821+00:00', '--class', 'com.viooh.smex.demand.denormalization.DemandDataDenormalizationBackfill', '--jars', '/usr/lib/spark/external/lib/spark-avro.jar', 's3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.9.2.develop/demand-denormalization-2.9.2.develop-jar-with-dependencies.jar'], 'Jar': 'command-runner.jar'}}, {'Name': 'DealsyncDenormalizationJobBackfill', 'ActionOnFailure': 'TERMINATE_CLUSTER', 'HadoopJarStep': {'Args': ['spark-submit', '--deploy-mode', 'cluster', '--master', 'yarn', '--conf', 'spark.yarn.submit.waitAppCompletion=true', '--conf', 'spark.driver.memory=20g', '--conf', 'spark.driver.cores=5', '--conf', 'spark.serializer=org.apache.spark.serializer.KryoSerializer', '--conf', 'spark.task.maxFailures=20', '--conf', 'spark.yarn.maxAppAttempts=10', '--conf', 'spark.stage.maxConsecutiveAttempts=20', '--conf', 'spark.app.env=uat', '--conf', 'spark.dag.execution.time=2023-11-10T09:56:28.500821+00:00', '--class', 'com.viooh.smex.dealsync.denormalization.DealSyncDenormalizationBackfill', 's3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.8.8/dealsync-denormalization-2.8.8-jar-with-dependencies.jar'], 'Jar': 'command-runner.jar'}}, {'Name': 'ModerationDenormalizationJobBackfill', 'ActionOnFailure': 'TERMINATE_CLUSTER', 'HadoopJarStep': {'Args': ['spark-submit', '--deploy-mode', 'cluster', '--master', 'yarn', '--conf', 'spark.yarn.submit.waitAppCompletion=true', '--conf', 'spark.driver.memory=20g', '--conf', 'spark.driver.cores=5', '--conf', 'spark.serializer=org.apache.spark.serializer.KryoSerializer', '--conf', 'spark.task.maxFailures=20', '--conf', 'spark.yarn.maxAppAttempts=10', '--conf', 'spark.stage.maxConsecutiveAttempts=20', '--conf', 'spark.app.env=uat', '--conf', 'spark.dag.execution.time=2023-11-10T09:56:28.500821+00:00', '--class', 'com.viooh.smex.moderation.denormalization.ModerationDenormalizationBackfill', '--jars', '/usr/lib/spark/external/lib/spark-avro.jar', 's3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.0.0.develop/moderation-denormalization-2.0.0.develop-jar-with-dependencies.jar'], 'Jar': 'command-runner.jar'}}, {'Name': 'DealDenormalizationJobBackfill', 'ActionOnFailure': 'TERMINATE_CLUSTER', 'HadoopJarStep': {'Args': ['spark-submit', '--deploy-mode', 'cluster', '--master', 'yarn', '--conf', 'spark.yarn.submit.waitAppCompletion=true', '--conf', 'spark.driver.memory=20g', '--conf', 'spark.driver.cores=5', '--conf', 'spark.serializer=org.apache.spark.serializer.KryoSerializer', '--conf', 'spark.task.maxFailures=20', '--conf', 'spark.yarn.maxAppAttempts=10', '--conf', 'spark.stage.maxConsecutiveAttempts=20', '--conf', 'spark.app.env=uat', '--conf', 'spark.dag.execution.time=2023-11-10T09:56:28.500821+00:00', '--class', 'com.viooh.smex.deal.denormalization.DealDenormalizationBackfill', '--jars', '/usr/lib/spark/external/lib/spark-avro.jar', 's3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.1.4.develop/deal-denormalization-1.1.4.develop-jar-with-dependencies.jar'], 'Jar': 'command-runner.jar'}}, {'Name': 'ExchangeTradeSummaryJobBackfill', 'ActionOnFailure': 'TERMINATE_CLUSTER', 'HadoopJarStep': {'Args': ['spark-submit', '--deploy-mode', 'cluster', '--master', 'yarn', '--conf', 'spark.yarn.submit.waitAppCompletion=true', '--conf', 'spark.driver.memory=16g', '--conf', 'spark.serializer=org.apache.spark.serializer.KryoSerializer', '--conf', 'spark.task.maxFailures=20', '--conf', 'spark.yarn.maxAppAttempts=10', '--conf', 'spark.stage.maxConsecutiveAttempts=20', '--conf', 'spark.app.env=uat', '--conf', 'spark.dag.execution.time=2023-11-10T09:56:28.500821+00:00', '--class', 'com.viooh.smex.trade.summary.ExchangeTradeSummaryBackfill', 's3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.1.1.develop/exchange-trade-summary-3.1.1.develop-jar-with-dependencies.jar'], 'Jar': 'command-runner.jar'}}, {'Name': 'ExportVioohHourLevelReportToDbV2Backfill', 'ActionOnFailure': 'TERMINATE_JOB_FLOW', 'HadoopJarStep': {'Jar': 's3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar', 'Args': ['s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh', 'uat', '0.0.2', 'db-import-viooh-hourly-ui-report-v2', 'backfill-frame', '2023-11-10T09:56:28.500821+00:00', 'viooh-spark-artifacts-lab-cn', 'scripts/exchange-trade-summary/sspui_reporting', 'backfill-frame', 'cn-northwest-1']}}, {'Name': 'ExportVioohHourLevelReportToDbV2Backfill', 'ActionOnFailure': 'TERMINATE_JOB_FLOW', 'HadoopJarStep': {'Jar': 's3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar', 'Args': ['s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh', 'uat', '0.0.2', 'db-import-viooh-hourly-creative-report', 'backfill-creative', '2023-11-10T09:56:28.500821+00:00', 'viooh-spark-artifacts-lab-cn', 'scripts/exchange-trade-summary/sspui_reporting', 'backfill-creative', 'cn-northwest-1']}}], 'JobFlowRole': 'DeEmrInstanceRole', 'ServiceRole': 'DeEmrServiceRole', 'SecurityConfiguration': 'bigdata_emr_cn_sec_conf', 'Tags': [{'Key': 'environment', 'Value': 'uat'}, {'Key': 'role', 'Value': 'analysis'}, {'Key': 'application', 'Value': 'emr'}, {'Key': 'project', 'Value': ''}, {'Key': 'TerminationPolicy', 'Value': 'OFFICE-HOURS'}, {'Key': 'ContactEmail', 'Value': 'viooh.data@viooh.com'}, {'Key': 'team', 'Value': 'bigdata'}, {'Key': 'Name', 'Value': 'vd-uat-viooh-exchange-trade-summary-backfill_all'}], 'Configurations': [{'Classification': 'yarn-site', 'Properties': {'yarn.resourcemanager.am.max-attempts': '1'}}]} |