| Key | Value | Timestamp | Execution Date | Task Id | Dag Id | ||
|---|---|---|---|---|---|---|---|
| return_value | j-17QY3OLYFFBU6 | 2021-04-10 18:11:58.300721+00:00 | 2021-04-10 17:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-EHGOUZ2K623B | 2021-04-10 18:12:00.253353+00:00 | 2021-04-10 17:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T18:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}]} | 2021-04-10 19:00:22.711482+00:00 | 2021-04-10 18:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-39Z7YHGGXZ9IG | 2021-04-10 19:00:49.512737+00:00 | 2021-04-10 18:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T18:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}]} | 2021-04-10 19:05:27.511493+00:00 | 2021-04-10 18:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-ERER64DJ1EDJ | 2021-04-10 19:05:48.149021+00:00 | 2021-04-10 18:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T18:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T18:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2021-04-10 19:10:38.727654+00:00 | 2021-04-10 18:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T18:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}]} | 2021-04-10 19:10:40.595296+00:00 | 2021-04-10 18:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | j-311YFPREDH4ML | 2021-04-10 19:10:57.393679+00:00 | 2021-04-10 18:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-2GT7X9SYTUUVR | 2021-04-10 19:11:01.164581+00:00 | 2021-04-10 18:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T19:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}]} | 2021-04-10 20:00:13.623104+00:00 | 2021-04-10 19:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-2DEPG1KLHKTI4 | 2021-04-10 20:00:32.621452+00:00 | 2021-04-10 19:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T19:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}]} | 2021-04-10 20:05:08.040612+00:00 | 2021-04-10 19:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-VF5855QR9ESU | 2021-04-10 20:05:33.181079+00:00 | 2021-04-10 19:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T19:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T19:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2021-04-10 20:10:55.917463+00:00 | 2021-04-10 19:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T19:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}]} | 2021-04-10 20:10:57.464672+00:00 | 2021-04-10 19:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | j-2CZ7DFL6WR78A | 2021-04-10 20:11:21.540154+00:00 | 2021-04-10 19:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-35CUO9X2P1RN | 2021-04-10 20:11:29.721676+00:00 | 2021-04-10 19:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T20:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}]} | 2021-04-10 21:00:11.275749+00:00 | 2021-04-10 20:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-2LZPCNQD1X130 | 2021-04-10 21:00:32.478982+00:00 | 2021-04-10 20:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T20:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}]} | 2021-04-10 21:05:27.338414+00:00 | 2021-04-10 20:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3JQ3COGYMGBO0 | 2021-04-10 21:05:48.593869+00:00 | 2021-04-10 20:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T20:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T20:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2021-04-10 21:11:08.291913+00:00 | 2021-04-10 20:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T20:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}]} | 2021-04-10 21:11:20.347599+00:00 | 2021-04-10 20:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | j-8IM0EW8KCYUA | 2021-04-10 21:11:28.995635+00:00 | 2021-04-10 20:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-UCKBMYM3TP2R | 2021-04-10 21:11:41.130916+00:00 | 2021-04-10 20:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T21:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}]} | 2021-04-10 22:00:14.054857+00:00 | 2021-04-10 21:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-1H8HDRW45Z6CE | 2021-04-10 22:00:35.176773+00:00 | 2021-04-10 21:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T21:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}]} | 2021-04-10 22:05:10.739951+00:00 | 2021-04-10 21:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2YGSB7NZ2NXK | 2021-04-10 22:05:29.514608+00:00 | 2021-04-10 21:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T21:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}]} | 2021-04-10 22:09:14.843012+00:00 | 2021-04-10 21:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T21:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T21:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2021-04-10 22:09:15.384996+00:00 | 2021-04-10 21:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-2G3OR5GPJYY6X | 2021-04-10 22:09:36.080149+00:00 | 2021-04-10 21:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-2PYO86T7C4M65 | 2021-04-10 22:09:36.207846+00:00 | 2021-04-10 21:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T22:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}]} | 2021-04-10 23:00:19.062345+00:00 | 2021-04-10 22:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-3JFXXLSHZ8TSD | 2021-04-10 23:00:38.215868+00:00 | 2021-04-10 22:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T22:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}]} | 2021-04-10 23:05:39.276074+00:00 | 2021-04-10 22:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-A91K28CYT4NM | 2021-04-10 23:06:00.724609+00:00 | 2021-04-10 22:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T22:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T22:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2021-04-10 23:10:24.853262+00:00 | 2021-04-10 22:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T22:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}]} | 2021-04-10 23:10:27.915235+00:00 | 2021-04-10 22:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | j-3QFTEBKG23PIX | 2021-04-10 23:10:44.564070+00:00 | 2021-04-10 22:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-6MSNQ2BM85MY | 2021-04-10 23:10:48.499829+00:00 | 2021-04-10 22:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T23:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}]} | 2021-04-11 00:00:12.015998+00:00 | 2021-04-10 23:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-1ZH11XZ4G3MW | 2021-04-11 00:00:30.505426+00:00 | 2021-04-10 23:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T23:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}]} | 2021-04-11 00:05:26.486416+00:00 | 2021-04-10 23:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-25SNR8GO1Q8I8 | 2021-04-11 00:05:47.123122+00:00 | 2021-04-10 23:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T23:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}]} | 2021-04-11 00:10:33.192648+00:00 | 2021-04-10 23:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T23:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-10T23:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2021-04-11 00:10:37.073307+00:00 | 2021-04-10 23:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-1IT7P2ZY8THFQ | 2021-04-11 00:10:53.588993+00:00 | 2021-04-10 23:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | j-35Z72M3K80Q7D | 2021-04-11 00:10:57.649128+00:00 | 2021-04-10 23:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T00:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}]} | 2021-04-11 01:00:14.699966+00:00 | 2021-04-11 00:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-QMO6B95D1KZN | 2021-04-11 01:00:35.542379+00:00 | 2021-04-11 00:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T00:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}]} | 2021-04-11 01:05:25.057649+00:00 | 2021-04-11 00:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-14YELEZYLZHG2 | 2021-04-11 01:05:46.147395+00:00 | 2021-04-11 00:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T00:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}]} | 2021-04-11 01:10:29.549366+00:00 | 2021-04-11 00:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T00:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T00:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2021-04-11 01:10:40.495723+00:00 | 2021-04-11 00:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-2W352VEWFTN4Y | 2021-04-11 01:10:51.054831+00:00 | 2021-04-11 00:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | j-2M6RZIS308BSA | 2021-04-11 01:11:00.656762+00:00 | 2021-04-11 00:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T01:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}]} | 2021-04-11 02:00:20.735733+00:00 | 2021-04-11 01:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-2FC9GBV0G44Z8 | 2021-04-11 02:00:39.377128+00:00 | 2021-04-11 01:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T01:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}]} | 2021-04-11 02:05:27.085181+00:00 | 2021-04-11 01:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2VMLKHN2H8T04 | 2021-04-11 02:05:47.491029+00:00 | 2021-04-11 01:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T01:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}]} | 2021-04-11 02:11:34.086527+00:00 | 2021-04-11 01:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T01:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T01:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2021-04-11 02:11:41.687230+00:00 | 2021-04-11 01:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-1UTW1PB3D1HLE | 2021-04-11 02:11:54.244817+00:00 | 2021-04-11 01:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | j-BA5CP42XKMZD | 2021-04-11 02:12:02.556159+00:00 | 2021-04-11 01:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T02:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}]} | 2021-04-11 03:00:18.218698+00:00 | 2021-04-11 02:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-1SL9D18KMZEWW | 2021-04-11 03:00:38.452988+00:00 | 2021-04-11 02:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T02:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}]} | 2021-04-11 03:05:26.191544+00:00 | 2021-04-11 02:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1UC7AO7P0P6QN | 2021-04-11 03:05:46.932391+00:00 | 2021-04-11 02:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T02:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}]} | 2021-04-11 03:09:24.641534+00:00 | 2021-04-11 02:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T02:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T02:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2021-04-11 03:09:26.878107+00:00 | 2021-04-11 02:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-A8HZZTO36VM5 | 2021-04-11 03:09:45.538818+00:00 | 2021-04-11 02:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | j-4P6SSIC80GC1 | 2021-04-11 03:09:47.380766+00:00 | 2021-04-11 02:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T03:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}]} | 2021-04-11 04:00:16.620387+00:00 | 2021-04-11 03:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-2S3PBYQ5IFQBG | 2021-04-11 04:00:37.307655+00:00 | 2021-04-11 03:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T03:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}]} | 2021-04-11 04:05:18.502212+00:00 | 2021-04-11 03:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-31RMWKV8AQKN1 | 2021-04-11 04:05:39.708355+00:00 | 2021-04-11 03:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T03:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T03:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2021-04-11 04:10:19.855835+00:00 | 2021-04-11 03:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T03:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}]} | 2021-04-11 04:10:31.238259+00:00 | 2021-04-11 03:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | j-RQWBV8CNOSQK | 2021-04-11 04:10:38.519232+00:00 | 2021-04-11 03:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-80Q0F7OOYA39 | 2021-04-11 04:10:52.181453+00:00 | 2021-04-11 03:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T04:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}]} | 2021-04-11 05:00:10.927771+00:00 | 2021-04-11 04:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-263BAEW6WZBR2 | 2021-04-11 05:00:31.538193+00:00 | 2021-04-11 04:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T04:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}]} | 2021-04-11 05:05:16.977777+00:00 | 2021-04-11 04:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1VKREWP8UOG8L | 2021-04-11 05:05:37.775709+00:00 | 2021-04-11 04:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T04:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}]} | 2021-04-11 05:19:25.005198+00:00 | 2021-04-11 04:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T04:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T04:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2021-04-11 05:19:27.324197+00:00 | 2021-04-11 04:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-1YT9SW7XX5X9Z | 2021-04-11 05:19:46.090073+00:00 | 2021-04-11 04:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | j-20JDWDHMHMP1S | 2021-04-11 05:19:48.243039+00:00 | 2021-04-11 04:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T05:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}]} | 2021-04-11 06:00:22.821685+00:00 | 2021-04-11 05:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-27PDSSYZQE6VQ | 2021-04-11 06:00:42.600683+00:00 | 2021-04-11 05:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T05:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}]} | 2021-04-11 06:05:18.371334+00:00 | 2021-04-11 05:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2QZH4S19YCY4Y | 2021-04-11 06:05:41.386420+00:00 | 2021-04-11 05:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T05:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}]} | 2021-04-11 06:09:37.024258+00:00 | 2021-04-11 05:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T05:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T05:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2021-04-11 06:09:43.203257+00:00 | 2021-04-11 05:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-2HWAO1VVD7EI | 2021-04-11 06:09:57.736933+00:00 | 2021-04-11 05:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | j-3VL8ON2T9SOMV | 2021-04-11 06:10:04.349366+00:00 | 2021-04-11 05:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-04-11T06:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}]} | 2021-04-11 07:00:24.259617+00:00 | 2021-04-11 06:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-2YTT6MEWQ18EM | 2021-04-11 07:00:44.768712+00:00 | 2021-04-11 06:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 |