DAG: campaign-versioning-job_v002

schedule: None


Task Instance: create_cluster_and_add_emr_steps


Rendered Template

job_flow_overrides
("{'Name': 'vd-uat-viooh-campaign-versioning-job', 'LogUri': "
 "'s3n://viooh-emr-logs-lab-cn/bigdata/emr/', 'ReleaseLabel': 'emr-5.21.0', "
 "'Instances': {'InstanceFleets': [{'Name': 'Masterfleet', "
 "'InstanceFleetType': 'MASTER', 'TargetOnDemandCapacity': 1, "
 "'InstanceTypeConfigs': [{'InstanceType': 'm5.2xlarge'}, {'InstanceType': "
 "'m5.4xlarge'}]}, {'Name': 'Corefleet', 'InstanceFleetType': 'CORE', "
 "'TargetOnDemandCapacity': 3, 'InstanceTypeConfigs': [{'InstanceType': "
 "'m5.2xlarge'}, {'InstanceType': 'm5.4xlarge'}]}, {'Name': 'Taskfleet', "
 "'InstanceFleetType': 'TASK', 'TargetSpotCapacity': 3, "
 "'LaunchSpecifications': {'SpotSpecification': {'TimeoutDurationMinutes': "
 "120, 'TimeoutAction': 'SWITCH_TO_ON_DEMAND'}}, 'InstanceTypeConfigs': "
 "[{'InstanceType': 'm5.2xlarge', 'BidPriceAsPercentageOfOnDemandPrice': 90}, "
 "{'InstanceType': 'm5.4xlarge', 'BidPriceAsPercentageOfOnDemandPrice': "
 "90}]}], 'Ec2KeyName': 'data.engineer', 'KeepJobFlowAliveWhenNoSteps': False, "
 "'TerminationProtected': False, 'Ec2SubnetId': 'subnet-0e214a494e67e2266', "
 "'EmrManagedMasterSecurityGroup': 'sg-0da002beeed87fb1a', "
 "'EmrManagedSlaveSecurityGroup': 'sg-0f1a0b7517309fcf0', "
 "'ServiceAccessSecurityGroup': 'sg-009528c399781cc69'}, 'BootstrapActions': "
 "[], 'Applications': [{'Name': 'Spark'}, {'Name': 'Hadoop'}, {'Name': "
 "'Ganglia'}, {'Name': 'Zeppelin'}], 'VisibleToAllUsers': True, 'Steps': "
 "[{'Name': 'Setup Hadoop Debugging', 'ActionOnFailure': 'TERMINATE_JOB_FLOW', "
 "'HadoopJarStep': {'Jar': 'command-runner.jar', 'Args': "
 "['state-pusher-script']}}, {'Name': 'CampaignProcessingJob', "
 "'ActionOnFailure': 'TERMINATE_CLUSTER', 'HadoopJarStep': {'Args': "
 "['spark-submit', '--deploy-mode', 'cluster', '--master', 'yarn', '--conf', "
 "'spark.yarn.submit.waitAppCompletion=true', '--conf', "
 "'spark.driver.memory=20g', '--conf', 'spark.app.env=uat', '--conf', "
 "'spark.dag.execution.time=2020-10-26T09:28:31.689257+00:00', '--class', "
 "'com.viooh.CampaignProcessingMain', "
 "'s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.5/campaign-processing-job-1.3.5.jar'], "
 "'Jar': 'command-runner.jar'}}], 'JobFlowRole': 'DeEmrInstanceRole', "
 "'ServiceRole': 'DeEmrServiceRole', 'SecurityConfiguration': "
 "'bigdata_emr_cn_sec_conf', 'Tags': [{'Key': 'environment', 'Value': 'uat'}, "
 "{'Key': 'role', 'Value': 'analysis'}, {'Key': 'application', 'Value': "
 "'emr'}, {'Key': 'project', 'Value': 'ccd'}, {'Key': 'TerminationPolicy', "
 "'Value': 'OFFICE-HOURS'}, {'Key': 'ContactEmail', 'Value': "
 "'viooh.data@viooh.com'}, {'Key': 'team', 'Value': 'bigdata'}, {'Key': "
 "'Name', 'Value': 'vd-uat-viooh-campaign-versioning-job'}]}")