| Key | Value | Timestamp | Execution Date | Task Id | Dag Id | ||
|---|---|---|---|---|---|---|---|
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T12:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-17 13:00:23.895874+00:00 | 2021-05-17 12:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-3F6E4UGXI5BK8 | 2021-05-17 13:00:44.101321+00:00 | 2021-05-17 12:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T12:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-17 13:05:38.145863+00:00 | 2021-05-17 12:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-DTUEV4NYFWE8 | 2021-05-17 13:05:56.919731+00:00 | 2021-05-17 12:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T12:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-17 13:20:24.401548+00:00 | 2021-05-17 12:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T12:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T12:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-17 13:20:29.033704+00:00 | 2021-05-17 12:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-3FAI4K5HLDAF7 | 2021-05-17 13:20:44.414665+00:00 | 2021-05-17 12:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | j-CDMB1CGZFXCC | 2021-05-17 13:20:46.256213+00:00 | 2021-05-17 12:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T13:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-17 14:00:25.736760+00:00 | 2021-05-17 13:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-HRMF4ACJO96X | 2021-05-17 14:00:43.997175+00:00 | 2021-05-17 13:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T13:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-17 14:05:19.840529+00:00 | 2021-05-17 13:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1DW80ONK1ADR3 | 2021-05-17 14:05:40.370917+00:00 | 2021-05-17 13:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T13:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-17 14:11:27.518851+00:00 | 2021-05-17 13:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T13:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T13:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-17 14:11:31.656905+00:00 | 2021-05-17 13:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-1Y3YEBSM535PT | 2021-05-17 14:11:47.137053+00:00 | 2021-05-17 13:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | j-136CW5ZEUCTVK | 2021-05-17 14:11:51.666598+00:00 | 2021-05-17 13:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T14:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-17 15:00:25.325475+00:00 | 2021-05-17 14:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-3AQXIZIP06F5C | 2021-05-17 15:00:45.682982+00:00 | 2021-05-17 14:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T14:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-17 15:05:25.378580+00:00 | 2021-05-17 14:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-49ASLFDUHL7F | 2021-05-17 15:05:45.893481+00:00 | 2021-05-17 14:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T14:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-17 15:10:53.714495+00:00 | 2021-05-17 14:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T14:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T14:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-17 15:10:58.173074+00:00 | 2021-05-17 14:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-3OPCWUEQPB19C | 2021-05-17 15:11:14.689681+00:00 | 2021-05-17 14:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | j-3R7QYR3ICY4FP | 2021-05-17 15:11:18.525908+00:00 | 2021-05-17 14:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T15:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-17 16:00:18.567073+00:00 | 2021-05-17 15:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-137K73FRNN9E2 | 2021-05-17 16:00:38.898096+00:00 | 2021-05-17 15:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T15:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-17 16:05:22.951980+00:00 | 2021-05-17 15:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2RF0FHPDKWEFB | 2021-05-17 16:05:43.110627+00:00 | 2021-05-17 15:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T15:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-17 16:11:43.713788+00:00 | 2021-05-17 15:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T15:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T15:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-17 16:11:45.910642+00:00 | 2021-05-17 15:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-3MR1LOG5HMG4X | 2021-05-17 16:12:04.399182+00:00 | 2021-05-17 15:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | j-24D2WY68TYOSS | 2021-05-17 16:12:06.395424+00:00 | 2021-05-17 15:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T16:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-17 17:11:36.802512+00:00 | 2021-05-17 16:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-G1KYUGDK1I4S | 2021-05-17 17:11:53.057993+00:00 | 2021-05-17 16:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3NO27AA82N4L8 | 2021-05-17 17:11:55.376242+00:00 | 2021-05-17 16:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T16:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-17 17:21:40.029224+00:00 | 2021-05-17 16:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T16:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T16:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-17 17:21:54.081068+00:00 | 2021-05-17 16:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-3VQYPF2A9VH4H | 2021-05-17 17:22:02.870431+00:00 | 2021-05-17 16:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | j-GRM2YOVMDR64 | 2021-05-17 17:22:14.325388+00:00 | 2021-05-17 16:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T17:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-17 18:00:20.611410+00:00 | 2021-05-17 17:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-3POSUW2DD7E9K | 2021-05-17 18:00:41.348013+00:00 | 2021-05-17 17:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-I8P8FJ4IHPRT | 2021-05-17 18:05:38.090557+00:00 | 2021-05-17 17:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T17:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T17:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-17 18:09:32.152984+00:00 | 2021-05-17 17:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T17:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-17 18:09:41.435572+00:00 | 2021-05-17 17:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | j-2SMY6UDHTKAF6 | 2021-05-17 18:09:52.077477+00:00 | 2021-05-17 17:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-ATIDHW2U881V | 2021-05-17 18:10:00.341397+00:00 | 2021-05-17 17:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T18:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-17 19:00:25.469111+00:00 | 2021-05-17 18:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-2P1PZE727Q4PI | 2021-05-17 19:00:53.910898+00:00 | 2021-05-17 18:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T18:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-17 19:05:25.761127+00:00 | 2021-05-17 18:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1AIB8R9V8WTF2 | 2021-05-17 19:05:46.656103+00:00 | 2021-05-17 18:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T18:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T18:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-17 19:10:03.105609+00:00 | 2021-05-17 18:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T18:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-17 19:10:08.815735+00:00 | 2021-05-17 18:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | j-T1H05APQ3CNA | 2021-05-17 19:10:21.196540+00:00 | 2021-05-17 18:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-6IZ5M0FHDZ7R | 2021-05-17 19:10:29.926014+00:00 | 2021-05-17 18:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T19:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-17 20:00:13.059020+00:00 | 2021-05-17 19:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-28956R253MT1S | 2021-05-17 20:00:33.401576+00:00 | 2021-05-17 19:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T19:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-17 20:05:17.510501+00:00 | 2021-05-17 19:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2RNP4L8GD6YIA | 2021-05-17 20:05:37.806971+00:00 | 2021-05-17 19:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T19:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T19:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-17 20:10:40.231865+00:00 | 2021-05-17 19:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T19:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-17 20:10:41.897199+00:00 | 2021-05-17 19:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | j-1W185M3S16MF9 | 2021-05-17 20:10:58.863591+00:00 | 2021-05-17 19:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-2N7VSP1F5N6LX | 2021-05-17 20:11:00.726731+00:00 | 2021-05-17 19:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T20:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-17 21:00:25.340788+00:00 | 2021-05-17 20:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-3DZY99VBQEFPK | 2021-05-17 21:00:47.978809+00:00 | 2021-05-17 20:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T20:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-17 21:05:11.711548+00:00 | 2021-05-17 20:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-RWIPBFVT8S0Q | 2021-05-17 21:05:30.832623+00:00 | 2021-05-17 20:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T20:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T20:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-17 21:10:46.717583+00:00 | 2021-05-17 20:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T20:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-17 21:10:54.408701+00:00 | 2021-05-17 20:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | j-12CXT0F7V4GDL | 2021-05-17 21:11:05.129437+00:00 | 2021-05-17 20:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-2NQNYKKL7T3JU | 2021-05-17 21:11:14.996219+00:00 | 2021-05-17 20:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T21:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-17 22:00:14.322615+00:00 | 2021-05-17 21:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-2YVTYS4X1FB0Y | 2021-05-17 22:00:35.292546+00:00 | 2021-05-17 21:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T21:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-17 22:05:12.718869+00:00 | 2021-05-17 21:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-KL1VG5JAQ41W | 2021-05-17 22:05:31.371524+00:00 | 2021-05-17 21:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T21:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T21:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-17 22:10:33.924823+00:00 | 2021-05-17 21:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T21:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-17 22:10:41.393957+00:00 | 2021-05-17 21:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | j-QFYHUREEZDJS | 2021-05-17 22:10:53.767806+00:00 | 2021-05-17 21:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-17X4Z27U2UTCO | 2021-05-17 22:11:02.637777+00:00 | 2021-05-17 21:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T22:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-17 23:00:18.710946+00:00 | 2021-05-17 22:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-delta-past-delete-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignDeltaPastDeleteJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-16T23:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaPastDeleteMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-delta-past-delete-job"}], "Configurations": []} | 2021-05-17 23:00:20.566613+00:00 | 2021-05-16 23:00:00+00:00 | create_emr_steps | campaign-delta-past-delete-job_v002 | ||
| return_value | j-1AN9G5ZL3HRCY | 2021-05-17 23:00:39.103017+00:00 | 2021-05-17 22:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-19JI4DC6IJGOF | 2021-05-17 23:00:43.022596+00:00 | 2021-05-16 23:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-delta-past-delete-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T22:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-17 23:05:19.135362+00:00 | 2021-05-17 22:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-C0XWBL6OZ5HI | 2021-05-17 23:05:39.802890+00:00 | 2021-05-17 22:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T22:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T22:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-17 23:09:31.634826+00:00 | 2021-05-17 22:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T22:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-17 23:09:37.403197+00:00 | 2021-05-17 22:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | j-1QN2LOQJOK91N | 2021-05-17 23:09:52.486645+00:00 | 2021-05-17 22:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-17MMUH39Q6KW1 | 2021-05-17 23:09:58.081657+00:00 | 2021-05-17 22:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T23:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-18 00:00:27.367449+00:00 | 2021-05-17 23:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-GM5N84T81QKL | 2021-05-18 00:00:45.852312+00:00 | 2021-05-17 23:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T23:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-18 00:05:15.679077+00:00 | 2021-05-17 23:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3B1NEGZ9JGKSO | 2021-05-18 00:05:40.074346+00:00 | 2021-05-17 23:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T23:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T23:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-18 00:09:40.200720+00:00 | 2021-05-17 23:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-17T23:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-18 00:09:50.019645+00:00 | 2021-05-17 23:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | j-3JSO5TX7FZFJC | 2021-05-18 00:09:58.783228+00:00 | 2021-05-17 23:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-38CT3NF1IYW4I | 2021-05-18 00:10:10.657385+00:00 | 2021-05-17 23:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-18T00:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-18 01:00:13.696329+00:00 | 2021-05-18 00:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-36RJWDVS15SA | 2021-05-18 01:00:32.535939+00:00 | 2021-05-18 00:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-18T00:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-18 01:05:26.144153+00:00 | 2021-05-18 00:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3K907V6F4J8Z4 | 2021-05-18 01:05:46.864540+00:00 | 2021-05-18 00:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 |