| Key | Value | Timestamp | Execution Date | Task Id | Dag Id | ||
|---|---|---|---|---|---|---|---|
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T08:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.10.0/daily-import-1.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-06-01 09:05:07.705673+00:00 | 2024-06-01 08:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-NYUL2H688PLK | 2024-06-01 09:05:32.170090+00:00 | 2024-06-01 08:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T09:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T09:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T09:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T09:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T09:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T09:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "10", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "9", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "10", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "9", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "10", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "9", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-06-01T09:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-06-01T09:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-06-01 10:00:33.938696+00:00 | 2024-06-01 09:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "LivePOPShuffle", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T09:00:00+00:00", "--class", "com.viooh.pop.data.live.shuffle.RawLivePOPShuffleMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-shuffle-live/1.12.0/pop-shuffle-live-1.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "PopAggregator", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T09:00:00+00:00", "--class", "com.viooh.pop.aggregator.livepop.LivePOPAggregatorMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-data-aggregator/1.13.0/pop-data-aggregator-1.13.0.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModelTransform", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T09:00:00+00:00", "--class", "uk.co.viooh.job.modeltransform.ModelTransform", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/uk/co/viooh/pandora-model-transform/1.5.0/pandora-model-transform-1.5.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform"}], "Configurations": []} | 2024-06-01 10:00:51.739997+00:00 | 2024-06-01 09:00:00+00:00 | create_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | j-3O7Z3D6DOSYMQ | 2024-06-01 10:00:52.075506+00:00 | 2024-06-01 09:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-PZH0LTPP8J9V | 2024-06-01 10:01:13.000442+00:00 | 2024-06-01 09:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T09:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.10.0/daily-import-1.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-06-01 10:05:16.175434+00:00 | 2024-06-01 09:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2F3AB0LC3P4Y | 2024-06-01 10:05:39.543259+00:00 | 2024-06-01 09:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T10:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T10:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T10:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T10:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T10:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T10:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "11", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "10", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "11", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "10", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "11", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "10", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-06-01T10:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-06-01T10:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-06-01 11:00:40.499763+00:00 | 2024-06-01 10:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "LivePOPShuffle", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T10:00:00+00:00", "--class", "com.viooh.pop.data.live.shuffle.RawLivePOPShuffleMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-shuffle-live/1.12.0/pop-shuffle-live-1.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "PopAggregator", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T10:00:00+00:00", "--class", "com.viooh.pop.aggregator.livepop.LivePOPAggregatorMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-data-aggregator/1.13.0/pop-data-aggregator-1.13.0.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModelTransform", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T10:00:00+00:00", "--class", "uk.co.viooh.job.modeltransform.ModelTransform", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/uk/co/viooh/pandora-model-transform/1.5.0/pandora-model-transform-1.5.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform"}], "Configurations": []} | 2024-06-01 11:00:52.012864+00:00 | 2024-06-01 10:00:00+00:00 | create_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | j-EF2QTBYPBF18 | 2024-06-01 11:00:59.804912+00:00 | 2024-06-01 10:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-2JW7HIB18FMIH | 2024-06-01 11:01:13.770542+00:00 | 2024-06-01 10:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T10:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.10.0/daily-import-1.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-06-01 11:05:09.997086+00:00 | 2024-06-01 10:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-7UQZNXOFJWUT | 2024-06-01 11:05:35.425135+00:00 | 2024-06-01 10:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T11:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T11:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T11:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T11:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T11:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T11:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "12", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "11", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "12", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "11", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "12", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "11", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-06-01T11:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-06-01T11:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-06-01 12:00:35.721242+00:00 | 2024-06-01 11:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "LivePOPShuffle", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T11:00:00+00:00", "--class", "com.viooh.pop.data.live.shuffle.RawLivePOPShuffleMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-shuffle-live/1.12.0/pop-shuffle-live-1.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "PopAggregator", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T11:00:00+00:00", "--class", "com.viooh.pop.aggregator.livepop.LivePOPAggregatorMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-data-aggregator/1.13.0/pop-data-aggregator-1.13.0.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModelTransform", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T11:00:00+00:00", "--class", "uk.co.viooh.job.modeltransform.ModelTransform", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/uk/co/viooh/pandora-model-transform/1.5.0/pandora-model-transform-1.5.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform"}], "Configurations": []} | 2024-06-01 12:00:52.631597+00:00 | 2024-06-01 11:00:00+00:00 | create_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | j-3QSNZB1OI74XW | 2024-06-01 12:00:56.332681+00:00 | 2024-06-01 11:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-UBELUEDN5E1N | 2024-06-01 12:01:14.897667+00:00 | 2024-06-01 11:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T11:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.10.0/daily-import-1.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-06-01 12:05:19.869854+00:00 | 2024-06-01 11:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2CVEN6UIMR95R | 2024-06-01 12:05:43.449820+00:00 | 2024-06-01 11:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T12:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T12:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T12:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T12:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T12:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T12:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "13", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "12", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "13", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "12", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "13", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "12", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-06-01T12:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-06-01T12:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-06-01 13:00:24.815116+00:00 | 2024-06-01 12:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-2M1ECOTEWPZZH | 2024-06-01 13:00:44.560242+00:00 | 2024-06-01 12:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "LivePOPShuffle", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T12:00:00+00:00", "--class", "com.viooh.pop.data.live.shuffle.RawLivePOPShuffleMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-shuffle-live/1.12.0/pop-shuffle-live-1.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "PopAggregator", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T12:00:00+00:00", "--class", "com.viooh.pop.aggregator.livepop.LivePOPAggregatorMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-data-aggregator/1.13.0/pop-data-aggregator-1.13.0.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModelTransform", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T12:00:00+00:00", "--class", "uk.co.viooh.job.modeltransform.ModelTransform", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/uk/co/viooh/pandora-model-transform/1.5.0/pandora-model-transform-1.5.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform"}], "Configurations": []} | 2024-06-01 13:00:58.654771+00:00 | 2024-06-01 12:00:00+00:00 | create_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | j-34LANX7ZKERBX | 2024-06-01 13:01:19.868553+00:00 | 2024-06-01 12:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T12:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.10.0/daily-import-1.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-06-01 13:05:16.071259+00:00 | 2024-06-01 12:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2LJ4XM0M17LIF | 2024-06-01 13:05:39.726077+00:00 | 2024-06-01 12:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T13:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T13:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T13:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T13:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T13:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T13:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "14", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "13", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "14", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "13", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "14", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "13", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-06-01T13:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-06-01T13:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-06-01 14:00:32.293718+00:00 | 2024-06-01 13:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "LivePOPShuffle", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T13:00:00+00:00", "--class", "com.viooh.pop.data.live.shuffle.RawLivePOPShuffleMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-shuffle-live/1.12.0/pop-shuffle-live-1.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "PopAggregator", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T13:00:00+00:00", "--class", "com.viooh.pop.aggregator.livepop.LivePOPAggregatorMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-data-aggregator/1.13.0/pop-data-aggregator-1.13.0.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModelTransform", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T13:00:00+00:00", "--class", "uk.co.viooh.job.modeltransform.ModelTransform", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/uk/co/viooh/pandora-model-transform/1.5.0/pandora-model-transform-1.5.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform"}], "Configurations": []} | 2024-06-01 14:00:43.818342+00:00 | 2024-06-01 13:00:00+00:00 | create_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | j-2T9J1ZQZQH9RZ | 2024-06-01 14:00:53.005480+00:00 | 2024-06-01 13:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-230EVCVWT851A | 2024-06-01 14:01:03.771506+00:00 | 2024-06-01 13:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T13:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.10.0/daily-import-1.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-06-01 14:05:24.248685+00:00 | 2024-06-01 13:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-8VNIHMOO9SJB | 2024-06-01 14:05:45.488531+00:00 | 2024-06-01 13:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T14:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T14:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T14:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T14:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T14:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T14:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "15", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "14", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "15", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "14", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "15", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "14", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-06-01T14:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-06-01T14:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-06-01 15:00:42.468323+00:00 | 2024-06-01 14:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-KX4FL52NK9TZ | 2024-06-01 15:00:55.057761+00:00 | 2024-06-01 14:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "LivePOPShuffle", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T14:00:00+00:00", "--class", "com.viooh.pop.data.live.shuffle.RawLivePOPShuffleMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-shuffle-live/1.12.0/pop-shuffle-live-1.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "PopAggregator", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T14:00:00+00:00", "--class", "com.viooh.pop.aggregator.livepop.LivePOPAggregatorMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-data-aggregator/1.13.0/pop-data-aggregator-1.13.0.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModelTransform", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T14:00:00+00:00", "--class", "uk.co.viooh.job.modeltransform.ModelTransform", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/uk/co/viooh/pandora-model-transform/1.5.0/pandora-model-transform-1.5.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform"}], "Configurations": []} | 2024-06-01 15:01:00.213260+00:00 | 2024-06-01 14:00:00+00:00 | create_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | j-2W4MWT62XDCOA | 2024-06-01 15:01:20.789390+00:00 | 2024-06-01 14:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T14:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.10.0/daily-import-1.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-06-01 15:05:16.245233+00:00 | 2024-06-01 14:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-15QZJYMJJ61RU | 2024-06-01 15:05:39.487235+00:00 | 2024-06-01 14:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T15:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T15:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T15:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T15:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T15:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T15:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "16", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "15", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "16", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "15", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "16", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "15", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-06-01T15:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-06-01T15:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-06-01 16:00:32.082121+00:00 | 2024-06-01 15:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-2U87BDHJP1O17 | 2024-06-01 16:00:51.992909+00:00 | 2024-06-01 15:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "LivePOPShuffle", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T15:00:00+00:00", "--class", "com.viooh.pop.data.live.shuffle.RawLivePOPShuffleMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-shuffle-live/1.12.0/pop-shuffle-live-1.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "PopAggregator", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T15:00:00+00:00", "--class", "com.viooh.pop.aggregator.livepop.LivePOPAggregatorMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-data-aggregator/1.13.0/pop-data-aggregator-1.13.0.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModelTransform", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T15:00:00+00:00", "--class", "uk.co.viooh.job.modeltransform.ModelTransform", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/uk/co/viooh/pandora-model-transform/1.5.0/pandora-model-transform-1.5.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform"}], "Configurations": []} | 2024-06-01 16:00:56.588994+00:00 | 2024-06-01 15:00:00+00:00 | create_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | j-1HR0NDN9QLDIV | 2024-06-01 16:01:17.023257+00:00 | 2024-06-01 15:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T15:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.10.0/daily-import-1.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-06-01 16:05:18.210155+00:00 | 2024-06-01 15:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3C5SCQWFXYCO | 2024-06-01 16:05:39.860475+00:00 | 2024-06-01 15:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T16:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T16:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T16:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T16:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T16:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T16:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "17", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "16", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "17", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "16", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "17", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "16", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-06-01T16:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-06-01T16:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-06-01 17:00:22.636595+00:00 | 2024-06-01 16:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-3RYYA1CQYS1WF | 2024-06-01 17:00:43.204915+00:00 | 2024-06-01 16:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "LivePOPShuffle", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T16:00:00+00:00", "--class", "com.viooh.pop.data.live.shuffle.RawLivePOPShuffleMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-shuffle-live/1.12.0/pop-shuffle-live-1.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "PopAggregator", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T16:00:00+00:00", "--class", "com.viooh.pop.aggregator.livepop.LivePOPAggregatorMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-data-aggregator/1.13.0/pop-data-aggregator-1.13.0.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModelTransform", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T16:00:00+00:00", "--class", "uk.co.viooh.job.modeltransform.ModelTransform", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/uk/co/viooh/pandora-model-transform/1.5.0/pandora-model-transform-1.5.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform"}], "Configurations": []} | 2024-06-01 17:00:46.198485+00:00 | 2024-06-01 16:00:00+00:00 | create_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | j-1G97FOHL1VXGV | 2024-06-01 17:01:07.523790+00:00 | 2024-06-01 16:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T16:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.10.0/daily-import-1.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-06-01 17:05:06.047800+00:00 | 2024-06-01 16:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-Y7LNSD1K8IQZ | 2024-06-01 17:05:31.883228+00:00 | 2024-06-01 16:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T17:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T17:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T17:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T17:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T17:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T17:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "18", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "17", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "18", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "17", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "18", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "17", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-06-01T17:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-06-01T17:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-06-01 18:00:24.835219+00:00 | 2024-06-01 17:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-2X2D485FTZ14H | 2024-06-01 18:00:42.593166+00:00 | 2024-06-01 17:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "LivePOPShuffle", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T17:00:00+00:00", "--class", "com.viooh.pop.data.live.shuffle.RawLivePOPShuffleMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-shuffle-live/1.12.0/pop-shuffle-live-1.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "PopAggregator", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T17:00:00+00:00", "--class", "com.viooh.pop.aggregator.livepop.LivePOPAggregatorMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-data-aggregator/1.13.0/pop-data-aggregator-1.13.0.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModelTransform", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T17:00:00+00:00", "--class", "uk.co.viooh.job.modeltransform.ModelTransform", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/uk/co/viooh/pandora-model-transform/1.5.0/pandora-model-transform-1.5.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform"}], "Configurations": []} | 2024-06-01 18:00:45.571506+00:00 | 2024-06-01 17:00:00+00:00 | create_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | j-38G0TOTBA50IU | 2024-06-01 18:01:05.180299+00:00 | 2024-06-01 17:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T17:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.10.0/daily-import-1.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-06-01 18:05:11.742296+00:00 | 2024-06-01 17:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-23XMO2XBTRKAL | 2024-06-01 18:05:35.528760+00:00 | 2024-06-01 17:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T18:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T18:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T18:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T18:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T18:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T18:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "19", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "18", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "19", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "18", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "19", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "18", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-06-01T18:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-06-01T18:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-06-01 19:00:38.749040+00:00 | 2024-06-01 18:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "LivePOPShuffle", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T18:00:00+00:00", "--class", "com.viooh.pop.data.live.shuffle.RawLivePOPShuffleMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-shuffle-live/1.12.0/pop-shuffle-live-1.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "PopAggregator", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T18:00:00+00:00", "--class", "com.viooh.pop.aggregator.livepop.LivePOPAggregatorMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-data-aggregator/1.13.0/pop-data-aggregator-1.13.0.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModelTransform", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T18:00:00+00:00", "--class", "uk.co.viooh.job.modeltransform.ModelTransform", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/uk/co/viooh/pandora-model-transform/1.5.0/pandora-model-transform-1.5.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform"}], "Configurations": []} | 2024-06-01 19:00:45.915021+00:00 | 2024-06-01 18:00:00+00:00 | create_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | j-2D6F3C7O3HLQ0 | 2024-06-01 19:00:56.382811+00:00 | 2024-06-01 18:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1UPWXE3TO2Z19 | 2024-06-01 19:01:06.879675+00:00 | 2024-06-01 18:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T18:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.10.0/daily-import-1.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-06-01 19:05:16.773664+00:00 | 2024-06-01 18:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3A09SJ57FBL6 | 2024-06-01 19:05:36.636957+00:00 | 2024-06-01 18:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T19:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T19:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T19:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T19:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T19:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T19:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "20", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "19", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "20", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "19", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "20", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "19", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-06-01T19:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-06-01T19:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-06-01 20:00:36.141975+00:00 | 2024-06-01 19:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-AAOMM3WTKFH | 2024-06-01 20:00:55.745777+00:00 | 2024-06-01 19:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "LivePOPShuffle", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T19:00:00+00:00", "--class", "com.viooh.pop.data.live.shuffle.RawLivePOPShuffleMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-shuffle-live/1.12.0/pop-shuffle-live-1.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "PopAggregator", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T19:00:00+00:00", "--class", "com.viooh.pop.aggregator.livepop.LivePOPAggregatorMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-data-aggregator/1.13.0/pop-data-aggregator-1.13.0.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModelTransform", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T19:00:00+00:00", "--class", "uk.co.viooh.job.modeltransform.ModelTransform", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/uk/co/viooh/pandora-model-transform/1.5.0/pandora-model-transform-1.5.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform"}], "Configurations": []} | 2024-06-01 20:00:57.370491+00:00 | 2024-06-01 19:00:00+00:00 | create_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | j-2F0QRYBOJSFQ5 | 2024-06-01 20:01:16.686347+00:00 | 2024-06-01 19:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T19:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.10.0/daily-import-1.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-06-01 20:05:30.259928+00:00 | 2024-06-01 19:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-IKIOFS5WTBFB | 2024-06-01 20:05:51.837091+00:00 | 2024-06-01 19:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T20:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T20:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T20:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T20:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T20:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T20:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "21", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "20", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "21", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "20", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "21", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "20", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-06-01T20:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-06-01T20:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-06-01 21:00:30.606876+00:00 | 2024-06-01 20:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "LivePOPShuffle", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T20:00:00+00:00", "--class", "com.viooh.pop.data.live.shuffle.RawLivePOPShuffleMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-shuffle-live/1.12.0/pop-shuffle-live-1.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "PopAggregator", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T20:00:00+00:00", "--class", "com.viooh.pop.aggregator.livepop.LivePOPAggregatorMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-data-aggregator/1.13.0/pop-data-aggregator-1.13.0.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModelTransform", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T20:00:00+00:00", "--class", "uk.co.viooh.job.modeltransform.ModelTransform", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/uk/co/viooh/pandora-model-transform/1.5.0/pandora-model-transform-1.5.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform"}], "Configurations": []} | 2024-06-01 21:00:49.054310+00:00 | 2024-06-01 20:00:00+00:00 | create_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | j-JAMKPIZMT4S5 | 2024-06-01 21:00:50.919232+00:00 | 2024-06-01 20:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-2KW5P4FYSP9ZO | 2024-06-01 21:01:06.952472+00:00 | 2024-06-01 20:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T20:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.10.0/daily-import-1.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-06-01 21:05:14.510009+00:00 | 2024-06-01 20:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2B6SK6YW4YVTM | 2024-06-01 21:05:36.239731+00:00 | 2024-06-01 20:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T21:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T21:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T21:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T21:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T21:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T21:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "22", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "21", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "22", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "21", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "22", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "21", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-06-01T21:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-06-01T21:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-06-01 22:00:36.230800+00:00 | 2024-06-01 21:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-39O7HQGJ1OYPH | 2024-06-01 22:00:54.124222+00:00 | 2024-06-01 21:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "LivePOPShuffle", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T21:00:00+00:00", "--class", "com.viooh.pop.data.live.shuffle.RawLivePOPShuffleMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-shuffle-live/1.12.0/pop-shuffle-live-1.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "PopAggregator", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T21:00:00+00:00", "--class", "com.viooh.pop.aggregator.livepop.LivePOPAggregatorMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-data-aggregator/1.13.0/pop-data-aggregator-1.13.0.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModelTransform", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T21:00:00+00:00", "--class", "uk.co.viooh.job.modeltransform.ModelTransform", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/uk/co/viooh/pandora-model-transform/1.5.0/pandora-model-transform-1.5.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform"}], "Configurations": []} | 2024-06-01 22:00:56.664175+00:00 | 2024-06-01 21:00:00+00:00 | create_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | j-2T6C8OMH7GBPP | 2024-06-01 22:01:16.771478+00:00 | 2024-06-01 21:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T21:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.10.0/daily-import-1.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-06-01 22:05:14.943776+00:00 | 2024-06-01 21:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2QYCXP9RENZ8T | 2024-06-01 22:05:38.036236+00:00 | 2024-06-01 21:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-delta-past-delete-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignDeltaPastDeleteJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-05-31T23:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaPastDeleteMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.16.0/pandora-campaign-delta-1.16.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-delta-past-delete-job"}], "Configurations": []} | 2024-06-01 23:00:26.485060+00:00 | 2024-05-31 23:00:00+00:00 | create_emr_steps | campaign-delta-past-delete-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T22:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T22:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T22:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T22:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T22:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T22:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "23", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "22", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "23", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "22", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "23", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "22", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-06-01T22:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-06-01T22:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-06-01 23:00:30.212392+00:00 | 2024-06-01 22:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1L95I2OOSYW8 | 2024-06-01 23:00:44.409205+00:00 | 2024-06-01 22:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "LivePOPShuffle", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T22:00:00+00:00", "--class", "com.viooh.pop.data.live.shuffle.RawLivePOPShuffleMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-shuffle-live/1.12.0/pop-shuffle-live-1.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "PopAggregator", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T22:00:00+00:00", "--class", "com.viooh.pop.aggregator.livepop.LivePOPAggregatorMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-data-aggregator/1.13.0/pop-data-aggregator-1.13.0.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModelTransform", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T22:00:00+00:00", "--class", "uk.co.viooh.job.modeltransform.ModelTransform", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/uk/co/viooh/pandora-model-transform/1.5.0/pandora-model-transform-1.5.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform"}], "Configurations": []} | 2024-06-01 23:00:47.134210+00:00 | 2024-06-01 22:00:00+00:00 | create_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | j-2I6ULOY58C7GJ | 2024-06-01 23:00:48.383507+00:00 | 2024-05-31 23:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-delta-past-delete-job_v002 | ||
| return_value | j-38NR9Z3WE8O4L | 2024-06-01 23:01:07.883476+00:00 | 2024-06-01 22:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T22:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.10.0/daily-import-1.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-06-01 23:05:19.662917+00:00 | 2024-06-01 22:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-17MNPNUFCXH2U | 2024-06-01 23:05:43.118749+00:00 | 2024-06-01 22:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T23:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T23:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T23:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T23:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T23:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T23:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-02", "0", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "23", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-02", "0", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "23", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-02", "0", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-01", "23", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-06-01T23:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-06-01T23:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-06-02 00:00:23.349874+00:00 | 2024-06-01 23:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-58FHBOJ66MT9 | 2024-06-02 00:00:43.712876+00:00 | 2024-06-01 23:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "LivePOPShuffle", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T23:00:00+00:00", "--class", "com.viooh.pop.data.live.shuffle.RawLivePOPShuffleMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-shuffle-live/1.12.0/pop-shuffle-live-1.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "PopAggregator", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T23:00:00+00:00", "--class", "com.viooh.pop.aggregator.livepop.LivePOPAggregatorMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-data-aggregator/1.13.0/pop-data-aggregator-1.13.0.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModelTransform", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T23:00:00+00:00", "--class", "uk.co.viooh.job.modeltransform.ModelTransform", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/uk/co/viooh/pandora-model-transform/1.5.0/pandora-model-transform-1.5.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform"}], "Configurations": []} | 2024-06-02 00:00:50.688384+00:00 | 2024-06-01 23:00:00+00:00 | create_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | j-GRT32YW7UBH6 | 2024-06-02 00:01:09.329383+00:00 | 2024-06-01 23:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-01T23:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.10.0/daily-import-1.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-06-02 00:05:15.756549+00:00 | 2024-06-01 23:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3CY99XDF3VFPP | 2024-06-02 00:05:39.915448+00:00 | 2024-06-01 23:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-02T00:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-02T00:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-02T00:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-02T00:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-02T00:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-02T00:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-02", "1", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-02", "0", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-02", "1", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-02", "0", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-02", "1", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-06-02", "0", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-06-02T00:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-06-02T00:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-06-02 01:00:42.764814+00:00 | 2024-06-02 00:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "LivePOPShuffle", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-02T00:00:00+00:00", "--class", "com.viooh.pop.data.live.shuffle.RawLivePOPShuffleMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-shuffle-live/1.12.0/pop-shuffle-live-1.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "PopAggregator", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-02T00:00:00+00:00", "--class", "com.viooh.pop.aggregator.livepop.LivePOPAggregatorMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pop/pop-data-aggregator/1.13.0/pop-data-aggregator-1.13.0.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModelTransform", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-02T00:00:00+00:00", "--class", "uk.co.viooh.job.modeltransform.ModelTransform", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/uk/co/viooh/pandora-model-transform/1.5.0/pandora-model-transform-1.5.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-livepop-shuffle-aggregator-modeltransform"}], "Configurations": []} | 2024-06-02 01:00:48.434769+00:00 | 2024-06-02 00:00:00+00:00 | create_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | j-3LXS08ODDJ9TG | 2024-06-02 01:01:04.164304+00:00 | 2024-06-02 00:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1R0LV58B6QYBF | 2024-06-02 01:01:09.597901+00:00 | 2024-06-02 00:00:00+00:00 | create_cluster_and_add_emr_steps | livepop-shuffle-aggregator-modeltransform_v004 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-06-02T00:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.10.0/daily-import-1.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-06-02 01:05:06.284549+00:00 | 2024-06-02 00:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2PUJPEGTF7FIB | 2024-06-02 01:05:37.096949+00:00 | 2024-06-02 00:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 |