| Key | Value | Timestamp | Execution Date | Task Id | Dag Id | ||
|---|---|---|---|---|---|---|---|
| return_value | j-HDWXHI207VKZ | 2021-05-28 13:00:28.817363+00:00 | 2021-05-28 12:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | j-16B6C01IPQ58X | 2021-05-29 11:00:49.899183+00:00 | 2021-05-29 10:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | j-EOIAULWZ34TJ | 2021-05-31 09:00:46.287954+00:00 | 2021-05-31 08:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | j-114OAX7DM4ZO4 | 2021-06-01 10:00:46.907131+00:00 | 2021-06-01 09:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T13:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T13:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-30 14:00:14.134427+00:00 | 2020-10-30 13:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-19M98M09OZU75 | 2020-10-30 14:00:22.974781+00:00 | 2020-10-30 13:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T14:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T14:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-30 15:00:13.642431+00:00 | 2020-10-30 14:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-KNHYK13D9MVS | 2020-10-30 15:00:22.334139+00:00 | 2020-10-30 14:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T15:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T15:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-30 16:00:08.545378+00:00 | 2020-10-30 15:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-2PR76ISNSVFC2 | 2020-10-30 16:00:17.131531+00:00 | 2020-10-30 15:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T16:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T16:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-30 17:00:15.625521+00:00 | 2020-10-30 16:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-1JCW8OJMADGD5 | 2020-10-30 17:00:24.384282+00:00 | 2020-10-30 16:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-N9L56PBQRNL0 | 2021-05-25 13:00:31.185100+00:00 | 2021-05-25 12:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-1XNI3YNG3YXZ3 | 2021-05-27 12:00:31.460250+00:00 | 2021-05-27 11:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-2D1W0IZWPUEMQ | 2021-05-28 13:00:37.286101+00:00 | 2021-05-28 12:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-FAUM3HZND91 | 2021-05-29 12:00:30.505226+00:00 | 2021-05-29 11:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | j-11FHAI2MFZE26 | 2021-05-31 10:00:28.715899+00:00 | 2021-05-31 09:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | j-173SKGLX4V4GJ | 2021-06-01 11:00:41.592419+00:00 | 2021-06-01 10:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T17:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T17:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-30 18:00:08.672486+00:00 | 2020-10-30 17:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-YZDMR2YYG6MP | 2020-10-30 18:00:17.625380+00:00 | 2020-10-30 17:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T18:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T18:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-30 19:00:09.623156+00:00 | 2020-10-30 18:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-2YZW5FD7ZNMW4 | 2020-10-30 19:00:16.627811+00:00 | 2020-10-30 18:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T19:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T19:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-30 20:00:10.908254+00:00 | 2020-10-30 19:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-3K6F28EPJGVXN | 2020-10-30 20:00:19.507629+00:00 | 2020-10-30 19:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T20:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T20:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-30 21:00:14.038568+00:00 | 2020-10-30 20:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-3KA0ZRQ4FRJ2S | 2020-10-30 21:00:22.974169+00:00 | 2020-10-30 20:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-1YIZHQA9ZUC9P | 2021-05-25 13:00:37.383636+00:00 | 2021-05-25 12:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | j-13D5XYSZ5DP7V | 2021-05-27 12:00:48.825445+00:00 | 2021-05-27 11:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | j-HC9PY5GQYTSW | 2021-05-28 14:00:43.288090+00:00 | 2021-05-28 13:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | j-16M6BQNRA6SKP | 2021-05-29 12:00:44.525664+00:00 | 2021-05-29 11:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-1I4JRUG5WGWU5 | 2021-05-31 10:00:38.730254+00:00 | 2021-05-31 09:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-ALTDSC3D4NN4 | 2021-06-01 11:00:47.631069+00:00 | 2021-06-01 10:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T21:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T21:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-30 22:00:15.534658+00:00 | 2020-10-30 21:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-28VGMVTU13KDT | 2020-10-30 22:00:24.377809+00:00 | 2020-10-30 21:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T22:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T22:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-30 23:00:10.196484+00:00 | 2020-10-30 22:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-delta-past-delete-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignDeltaPastDeleteJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=10g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-29T23:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaPastDeleteMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-delta-past-delete-job"}]} | 2020-10-30 23:00:14.068921+00:00 | 2020-10-29 23:00:00+00:00 | create_emr_steps | campaign-delta-past-delete-job_v002 | ||
| return_value | j-39NMSMHKMK6VL | 2020-10-30 23:00:19.103589+00:00 | 2020-10-30 22:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-3CW4UD4X4N65L | 2020-10-30 23:00:22.906428+00:00 | 2020-10-29 23:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-delta-past-delete-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T23:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T23:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-31 00:00:13.454197+00:00 | 2020-10-30 23:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-1176GK5XED7UP | 2020-10-31 00:00:30.116560+00:00 | 2020-10-30 23:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-J0V2GIG6W09I | 2021-05-25 14:00:33.350760+00:00 | 2021-05-25 13:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-8ETFIE5F1XK6 | 2021-05-27 13:00:34.509828+00:00 | 2021-05-27 12:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | j-2CKK8YUQ99D2F | 2021-05-28 14:00:47.093168+00:00 | 2021-05-28 13:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-1YPKICCHA0OQ1 | 2021-05-29 13:00:47.042483+00:00 | 2021-05-29 12:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | j-FZO39QB1JMKV | 2021-05-31 11:00:30.336933+00:00 | 2021-05-31 10:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-1FXM79OTVFKEZ | 2021-06-01 12:00:42.849541+00:00 | 2021-06-01 11:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T00:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T00:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-31 01:00:22.510496+00:00 | 2020-10-31 00:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-3AT96RA3IKKTH | 2020-10-31 01:00:39.075969+00:00 | 2020-10-31 00:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T01:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T01:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-31 02:00:33.596263+00:00 | 2020-10-31 01:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-3L7ZXNRWVV56Z | 2020-10-31 02:00:42.455385+00:00 | 2020-10-31 01:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T02:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T02:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-31 03:00:13.038988+00:00 | 2020-10-31 02:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-31GG2RFID7449 | 2020-10-31 03:00:21.869237+00:00 | 2020-10-31 02:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T03:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T03:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-31 04:00:14.682494+00:00 | 2020-10-31 03:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-3J0IX51SLLH98 | 2020-10-31 04:00:21.582956+00:00 | 2020-10-31 03:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-7AQTXBI3VNE5 | 2021-05-25 14:00:41.648430+00:00 | 2021-05-25 13:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | j-2WF1XQ1CCPBAH | 2021-05-27 13:00:44.514452+00:00 | 2021-05-27 12:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-3HGUHACEOLD3G | 2021-05-28 15:00:31.940016+00:00 | 2021-05-28 14:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-1M0J2NL62N7CY | 2021-05-29 13:00:47.232927+00:00 | 2021-05-29 12:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-2OEYPLKSD7U2G | 2021-05-31 11:00:44.058873+00:00 | 2021-05-31 10:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | j-HO816X7GGVTW | 2021-06-01 12:00:49.016030+00:00 | 2021-06-01 11:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T04:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T04:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-31 05:00:15.344591+00:00 | 2020-10-31 04:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-9JO1IM2ES3S6 | 2020-10-31 05:00:24.313318+00:00 | 2020-10-31 04:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T05:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T05:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-31 06:00:08.903795+00:00 | 2020-10-31 05:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-1ABATWCLI1OPW | 2020-10-31 06:00:21.564289+00:00 | 2020-10-31 05:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T06:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T06:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-31 07:00:09.806630+00:00 | 2020-10-31 06:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-3N74G6TL4IZVC | 2020-10-31 07:00:18.750248+00:00 | 2020-10-31 06:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-missing-dr-check-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "MissingDRCheckJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-30T07:30:00+00:00", "--class", "com.viooh.MissingDRCheck", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/compliance-daily-report/1.3.4/compliance-daily-report-1.3.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-missing-dr-check-job"}]} | 2020-10-31 07:30:06.677564+00:00 | 2020-10-30 07:30:00+00:00 | create_emr_steps | missing-dr-check-job_v002 | ||
| return_value | j-22LQDXIJEUC20 | 2020-10-31 07:30:15.845589+00:00 | 2020-10-30 07:30:00+00:00 | create_cluster_and_add_emr_steps | missing-dr-check-job_v002 | ||
| return_value | j-2OUB1GR29VLJP | 2021-05-25 15:00:37.923364+00:00 | 2021-05-25 14:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-36W54VQUZNFL8 | 2021-05-27 14:00:42.905229+00:00 | 2021-05-27 13:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | j-3OUBAFVFYYI00 | 2021-05-28 15:00:47.688506+00:00 | 2021-05-28 14:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | j-15DEZKBEIIVEF | 2021-05-29 14:00:32.563709+00:00 | 2021-05-29 13:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | j-3GQS1ACXZP27N | 2021-05-31 12:00:39.564727+00:00 | 2021-05-31 11:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-2L2IKXFLCMLY5 | 2021-06-01 13:00:38.882571+00:00 | 2021-06-01 12:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | j-8LZ39CVN6C79 | 2021-06-02 08:00:35.893362+00:00 | 2021-06-02 07:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-LXNVONR3KWSA | 2021-06-16 04:00:40.526176+00:00 | 2021-06-16 03:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T07:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T07:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-31 08:00:14.672788+00:00 | 2020-10-31 07:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-B681ZEB58NGQ | 2020-10-31 08:00:21.633446+00:00 | 2020-10-31 07:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T08:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T08:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-31 09:00:15.139650+00:00 | 2020-10-31 08:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-379ZWGM4U0B3E | 2020-10-31 09:00:34.042765+00:00 | 2020-10-31 08:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T09:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T09:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-31 10:00:10.266257+00:00 | 2020-10-31 09:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-3MV108PRS66WU | 2020-10-31 10:00:17.198347+00:00 | 2020-10-31 09:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T10:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T10:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-31 11:00:08.134165+00:00 | 2020-10-31 10:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-2WQM51WZD64IT | 2020-10-31 11:00:16.967781+00:00 | 2020-10-31 10:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-8F0XFY45LUL0 | 2021-05-25 15:00:43.920757+00:00 | 2021-05-25 14:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | j-1LXHE8TGFOTEL | 2021-05-27 14:00:44.899427+00:00 | 2021-05-27 13:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-2TPGGDP8I5I4L | 2021-05-28 16:00:35.639321+00:00 | 2021-05-28 15:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | j-3SVSZQ7SS78MH | 2021-05-29 14:00:49.053423+00:00 | 2021-05-29 13:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-1KF9A34UYBT6Z | 2021-05-31 12:00:41.890800+00:00 | 2021-05-31 11:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 | ||
| return_value | j-3IFWCYQ3Y1E1V | 2021-06-01 13:00:42.808486+00:00 | 2021-06-01 12:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T11:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T11:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-31 12:00:11.457317+00:00 | 2020-10-31 11:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-3N4BHXDL0VD2B | 2020-10-31 12:00:18.234884+00:00 | 2020-10-31 11:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T12:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T12:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-31 13:00:12.571253+00:00 | 2020-10-31 12:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-23KSRV36ZXFD3 | 2020-10-31 13:00:21.263072+00:00 | 2020-10-31 12:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T13:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T13:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-31 14:00:07.774711+00:00 | 2020-10-31 13:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-2T9JFR4B5XTXR | 2020-10-31 14:00:16.511277+00:00 | 2020-10-31 13:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T14:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2020-10-31T14:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.4/pandora-campaign-delta-1.4.4-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "ccd"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}]} | 2020-10-31 15:00:08.901503+00:00 | 2020-10-31 14:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-1QUON68QDJKNI | 2020-10-31 15:00:17.632072+00:00 | 2020-10-31 14:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-1ISGK4WADCRWX | 2021-05-25 16:00:40.932722+00:00 | 2021-05-25 15:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-16QDJAEUKGBE | 2021-05-27 15:00:36.149047+00:00 | 2021-05-27 14:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-job_v002 |