| Key | Value | Timestamp | Execution Date | Task Id | Dag Id | ||
|---|---|---|---|---|---|---|---|
| return_value | j-1VYEON9BZQDEP | 2021-05-01 00:10:56.442062+00:00 | 2021-04-30 23:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-1YITAX0B09ORO | 2021-05-01 00:11:12.461470+00:00 | 2021-04-30 23:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T00:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-01 01:00:08.940610+00:00 | 2021-05-01 00:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-3F2LJXARMC86D | 2021-05-01 01:00:27.526691+00:00 | 2021-05-01 00:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T00:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-01 01:05:21.199507+00:00 | 2021-05-01 00:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1FS5NO4T2NX6R | 2021-05-01 01:05:41.767508+00:00 | 2021-05-01 00:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T00:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T00:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-01 01:10:22.073084+00:00 | 2021-05-01 00:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T00:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-01 01:10:34.003173+00:00 | 2021-05-01 00:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | j-3QSTIIB27907K | 2021-05-01 01:10:42.486771+00:00 | 2021-05-01 00:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-2YAZZ302OOPHM | 2021-05-01 01:10:54.539835+00:00 | 2021-05-01 00:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T01:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-01 02:00:18.213367+00:00 | 2021-05-01 01:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-23H92RR25OOG | 2021-05-01 02:00:38.471383+00:00 | 2021-05-01 01:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T01:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-01 02:05:24.251611+00:00 | 2021-05-01 01:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-26VFERH1UGYRN | 2021-05-01 02:05:43.133718+00:00 | 2021-05-01 01:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T01:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T01:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-01 02:09:36.922858+00:00 | 2021-05-01 01:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T01:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-01 02:09:38.670596+00:00 | 2021-05-01 01:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | j-2PSMCNAZUUGC5 | 2021-05-01 02:09:57.686557+00:00 | 2021-05-01 01:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | j-3ALE2B79P0CTZ | 2021-05-01 02:09:57.860261+00:00 | 2021-05-01 01:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T02:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-01 03:03:43.018722+00:00 | 2021-05-01 02:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-YX03TN8Y8ZH5 | 2021-05-01 03:04:03.797499+00:00 | 2021-05-01 02:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T02:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-01 03:05:27.039424+00:00 | 2021-05-01 02:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2QMRVCCOQX4B6 | 2021-05-01 03:05:46.170678+00:00 | 2021-05-01 02:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T02:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-01 03:14:47.937075+00:00 | 2021-05-01 02:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T02:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T02:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-01 03:14:49.698916+00:00 | 2021-05-01 02:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-2HUUIN82TM96X | 2021-05-01 03:15:04.987410+00:00 | 2021-05-01 02:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | j-3HNNXL6CYF7I9 | 2021-05-01 03:15:09.006889+00:00 | 2021-05-01 02:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T03:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-01 04:00:27.461441+00:00 | 2021-05-01 03:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-19J6Z9ZQL7SW2 | 2021-05-01 04:00:46.310876+00:00 | 2021-05-01 03:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T03:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-01 04:05:50.093956+00:00 | 2021-05-01 03:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3MP465MFDIBT6 | 2021-05-01 04:06:08.700552+00:00 | 2021-05-01 03:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T03:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T03:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-01 04:09:40.555565+00:00 | 2021-05-01 03:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T03:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-01 04:09:46.458427+00:00 | 2021-05-01 03:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | j-1I8JEIGOD1B1P | 2021-05-01 04:09:59.143873+00:00 | 2021-05-01 03:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-27JIX4NVJPCEF | 2021-05-01 04:10:05.412378+00:00 | 2021-05-01 03:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T04:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-01 05:00:15.409481+00:00 | 2021-05-01 04:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-2MOBQBN4X8HXG | 2021-05-01 05:00:38.077089+00:00 | 2021-05-01 04:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T04:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-01 05:05:13.831997+00:00 | 2021-05-01 04:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-9ZFF5RXTCRPG | 2021-05-01 05:05:34.446168+00:00 | 2021-05-01 04:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T04:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T04:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-01 05:10:57.335572+00:00 | 2021-05-01 04:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T04:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-01 05:11:10.525530+00:00 | 2021-05-01 04:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | j-J1R3O47IAFIM | 2021-05-01 05:11:17.187523+00:00 | 2021-05-01 04:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-2OCLTQAWJL824 | 2021-05-01 05:11:29.144634+00:00 | 2021-05-01 04:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T05:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-01 06:00:24.139993+00:00 | 2021-05-01 05:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-34H6J2GQKFHVZ | 2021-05-01 06:00:44.486776+00:00 | 2021-05-01 05:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T05:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-01 06:05:10.245970+00:00 | 2021-05-01 05:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-36KC61RO5B6TS | 2021-05-01 06:05:30.803061+00:00 | 2021-05-01 05:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T05:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T05:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-01 06:16:46.362272+00:00 | 2021-05-01 05:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T05:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-01 06:16:54.114589+00:00 | 2021-05-01 05:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | j-1N84D8S3C5RJZ | 2021-05-01 06:17:07.307428+00:00 | 2021-05-01 05:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-2HUTMK1G0GCPJ | 2021-05-01 06:17:13.328134+00:00 | 2021-05-01 05:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T06:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-01 07:00:12.729794+00:00 | 2021-05-01 06:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-3AW0G2JLKRHLE | 2021-05-01 07:00:33.347322+00:00 | 2021-05-01 06:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T06:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-01 07:05:23.172828+00:00 | 2021-05-01 06:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2Z3UEHR4OZ8JS | 2021-05-01 07:05:43.759391+00:00 | 2021-05-01 06:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T06:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T06:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-01 07:10:13.891627+00:00 | 2021-05-01 06:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T06:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-01 07:10:21.538860+00:00 | 2021-05-01 06:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | j-1O00SNKAVBDMW | 2021-05-01 07:10:32.327731+00:00 | 2021-05-01 06:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-S7IXBAS46YSG | 2021-05-01 07:10:40.140434+00:00 | 2021-05-01 06:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T07:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-01 08:00:15.790945+00:00 | 2021-05-01 07:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-30X0EW0ZSGSD6 | 2021-05-01 08:00:34.319960+00:00 | 2021-05-01 07:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T07:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-01 08:05:14.409897+00:00 | 2021-05-01 07:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-7ZY1RQ2ICB91 | 2021-05-01 08:05:35.003600+00:00 | 2021-05-01 07:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T07:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T07:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-01 08:11:25.304680+00:00 | 2021-05-01 07:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T07:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-01 08:11:38.965327+00:00 | 2021-05-01 07:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | j-187QSRXKJITSW | 2021-05-01 08:11:45.466917+00:00 | 2021-05-01 07:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-2NNAMO0QHUPHT | 2021-05-01 08:11:59.717479+00:00 | 2021-05-01 07:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T08:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-01 09:00:20.148902+00:00 | 2021-05-01 08:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-3385TF3J7A4HI | 2021-05-01 09:00:40.501254+00:00 | 2021-05-01 08:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T08:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-01 09:05:16.191021+00:00 | 2021-05-01 08:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2SP6BMX47UTHF | 2021-05-01 09:05:37.181877+00:00 | 2021-05-01 08:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T08:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-01 09:10:43.291497+00:00 | 2021-05-01 08:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T08:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T08:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-01 09:10:45.298575+00:00 | 2021-05-01 08:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-X7L51BYFPTNX | 2021-05-01 09:11:04.222343+00:00 | 2021-05-01 08:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | j-O254SZ1W46ZU | 2021-05-01 09:11:06.569992+00:00 | 2021-05-01 08:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T09:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-01 10:00:08.982039+00:00 | 2021-05-01 09:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-32GK30PG2QAO1 | 2021-05-01 10:00:29.368850+00:00 | 2021-05-01 09:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T09:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-01 10:05:18.798490+00:00 | 2021-05-01 09:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1MK2QQ9MXAR02 | 2021-05-01 10:05:39.679962+00:00 | 2021-05-01 09:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T09:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-01 10:10:39.379682+00:00 | 2021-05-01 09:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T09:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T09:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-01 10:10:39.797881+00:00 | 2021-05-01 09:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-1CCS54GMI3MY4 | 2021-05-01 10:10:58.804391+00:00 | 2021-05-01 09:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-2DXB41UWZJ690 | 2021-05-01 10:11:00.855199+00:00 | 2021-05-01 09:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T10:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-01 11:00:11.002022+00:00 | 2021-05-01 10:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-1F7AYR5PX66UB | 2021-05-01 11:00:29.502158+00:00 | 2021-05-01 10:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T10:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-01 11:05:27.031809+00:00 | 2021-05-01 10:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3MIN4SCXSFILF | 2021-05-01 11:05:47.985729+00:00 | 2021-05-01 10:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T10:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-01 11:10:08.065838+00:00 | 2021-05-01 10:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T10:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T10:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-01 11:10:10.030195+00:00 | 2021-05-01 10:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-25K4OQSET6ZCH | 2021-05-01 11:10:28.500184+00:00 | 2021-05-01 10:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-2D1EV2WTPDRPR | 2021-05-01 11:10:28.610852+00:00 | 2021-05-01 10:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T11:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-01 12:00:16.113504+00:00 | 2021-05-01 11:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-348IZ89ZM85A | 2021-05-01 12:00:37.190948+00:00 | 2021-05-01 11:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T11:05:00+00:00", "--class", "VioohCentralAssetsMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.6.3/daily-import-1.6.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2021-05-01 12:05:14.677301+00:00 | 2021-05-01 11:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1I8GLY1RWC2DU | 2021-05-01 12:05:35.739436+00:00 | 2021-05-01 11:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-history-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignHistoryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T11:00:00+00:00", "--class", "com.viooh.campaign.history.campaignHistoryMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-history/1.2.3/campaign-history-1.2.3-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-history-job"}], "Configurations": []} | 2021-05-01 12:11:45.491958+00:00 | 2021-05-01 11:00:00+00:00 | create_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 7, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 100}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 1, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T11:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.3.6/campaign-processing-job-1.3.6.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T11:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.4.9/pandora-campaign-delta-1.4.9-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2021-05-01 12:11:48.086568+00:00 | 2021-05-01 11:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-1GB6809ULYGA9 | 2021-05-01 12:12:08.563912+00:00 | 2021-05-01 11:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | j-12OUZ03Q89B31 | 2021-05-01 12:12:08.632790+00:00 | 2021-05-01 11:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-history-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-flow-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.21.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Taskfleet", "InstanceFleetType": "TASK", "TargetSpotCapacity": 3, "LaunchSpecifications": {"SpotSpecification": {"TimeoutDurationMinutes": 120, "TimeoutAction": "SWITCH_TO_ON_DEMAND"}}, "InstanceTypeConfigs": [{"InstanceType": "r5.2xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}, {"InstanceType": "m5.4xlarge", "BidPriceAsPercentageOfOnDemandPrice": 90}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=40g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2021-05-01T12:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--packages", "org.apache.spark:spark-avro_2.11:2.4.0", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.0.0/campaign-extractor-1.0.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-flow-job"}], "Configurations": []} | 2021-05-01 13:00:11.717922+00:00 | 2021-05-01 12:00:00+00:00 | create_emr_steps | campaign-flow-job_v002 | ||
| return_value | j-20HUEQ0S6U74G | 2021-05-01 13:00:29.853273+00:00 | 2021-05-01 12:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-flow-job_v002 |