| Key | Value | Timestamp | Execution Date | Task Id | Dag Id | ||
|---|---|---|---|---|---|---|---|
| return_value | j-337XNM2KCPYV2 | 2024-08-07 21:00:46.080793+00:00 | 2024-08-07 20:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T20:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-07 21:05:12.013145+00:00 | 2024-08-07 20:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3PFE5GYSGW3EM | 2024-08-07 21:05:36.164273+00:00 | 2024-08-07 20:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T21:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T21:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T21:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T21:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T21:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T21:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-07", "22", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-07", "21", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-07", "22", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-07", "21", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-07", "22", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-07", "21", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-07T21:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-07T21:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-07 22:00:11.789510+00:00 | 2024-08-07 21:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-3NWXYUM6VLS3A | 2024-08-07 22:00:32.288889+00:00 | 2024-08-07 21:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-12T22:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-12T22:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-12T22:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-12T22:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-12T22:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-12T22:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-12", "23", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-12", "22", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-12", "23", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-12", "22", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-12", "23", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-12", "22", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-12T22:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-12T22:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-13 02:51:39.186353+00:00 | 2024-09-12 22:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T21:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-07 22:05:16.424422+00:00 | 2024-08-07 21:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-JOGBHJ8XF6UP | 2024-08-07 22:05:36.720215+00:00 | 2024-08-07 21:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-delta-past-delete-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignDeltaPastDeleteJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-06T23:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaPastDeleteMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.22.0/pandora-campaign-delta-1.22.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-delta-past-delete-job"}], "Configurations": []} | 2024-08-07 23:00:09.695361+00:00 | 2024-08-06 23:00:00+00:00 | create_emr_steps | campaign-delta-past-delete-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T22:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T22:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T22:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T22:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T22:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T22:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-07", "23", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-07", "22", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-07", "23", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-07", "22", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-07", "23", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-07", "22", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-07T22:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-07T22:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-07 23:00:16.654154+00:00 | 2024-08-07 22:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-HU4D0MJ38A6S | 2024-08-07 23:00:33.751014+00:00 | 2024-08-06 23:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-delta-past-delete-job_v002 | ||
| return_value | j-3MY6KANNXSKQH | 2024-08-07 23:00:35.763576+00:00 | 2024-08-07 22:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T22:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-07 23:05:21.848864+00:00 | 2024-08-07 22:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-VXZW1HR17CCV | 2024-08-07 23:05:43.466059+00:00 | 2024-08-07 22:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T23:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T23:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T23:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T23:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T23:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T23:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "0", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-07", "23", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "0", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-07", "23", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "0", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-07", "23", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-07T23:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-07T23:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 00:00:27.389354+00:00 | 2024-08-07 23:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-36SNR3MDR0I2E | 2024-08-08 00:00:52.486740+00:00 | 2024-08-07 23:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T23:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 00:05:21.948475+00:00 | 2024-08-07 23:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2648B3S5HSPT5 | 2024-08-08 00:05:43.559624+00:00 | 2024-08-07 23:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T00:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T00:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T00:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T00:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T00:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T00:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "1", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "0", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "1", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "0", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "1", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "0", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T00:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T00:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 01:00:25.863460+00:00 | 2024-08-08 00:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-3PZJMLJVLRQQF | 2024-08-08 01:00:44.590922+00:00 | 2024-08-08 00:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T00:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 01:05:25.633345+00:00 | 2024-08-08 00:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-13MKZ3U5Q19T6 | 2024-08-08 01:05:47.323088+00:00 | 2024-08-08 00:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-by-hour-model-transform", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "BookingByHour-ModelTransform", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=20g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T01:30:00+00:00", "--class", "com.viooh.transform.BookingModelTransform", "s3://viooh-spark-artifacts-lab-cn/releases/uk/co/viooh/pandora-model-transform/1.6.0/pandora-model-transform-1.6.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-by-hour-model-transform"}], "Configurations": []} | 2024-08-08 01:30:17.530202+00:00 | 2024-08-07 01:30:00+00:00 | create_emr_steps | booking-by-hour-model-transform_v002 | ||
| return_value | j-18DALJ9ODCFRA | 2024-08-08 01:30:39.340165+00:00 | 2024-08-07 01:30:00+00:00 | create_cluster_and_add_emr_steps | booking-by-hour-model-transform_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T01:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T01:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T01:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T01:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T01:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T01:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "2", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "1", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "2", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "1", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "2", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "1", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T01:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T01:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 02:00:10.678930+00:00 | 2024-08-08 01:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1SPW98IBT7VL6 | 2024-08-08 02:00:31.027779+00:00 | 2024-08-08 01:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T01:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 02:05:12.166926+00:00 | 2024-08-08 01:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2WUD8TARHYOIM | 2024-08-08 02:05:32.451789+00:00 | 2024-08-08 01:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T02:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T02:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T02:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T02:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T02:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T02:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "3", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "2", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "3", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "2", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "3", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "2", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T02:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T02:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 03:00:23.717353+00:00 | 2024-08-08 02:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-3364D7JN53VRY | 2024-08-08 03:00:44.261759+00:00 | 2024-08-08 02:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T02:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 03:05:23.027339+00:00 | 2024-08-08 02:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-Z9X3IAM4F4V5 | 2024-08-08 03:05:44.436234+00:00 | 2024-08-08 02:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-dsp-reporting", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "DSPExchangeTradeJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T04:00:00+00:00", "--class", "com.viooh.smex.dsp.DSPExchangeTrade", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dsp-exchange-trade/2.11.0/dsp-exchange-trade-2.11.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopyDSPReports", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-dsp-reporting/1.0.0/job.sh", "--environment", "uat", "--reportdate", "2024-08-07", "--region", "cn", "--skipFailure"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-dsp-reporting"}], "Configurations": []} | 2024-08-08 04:00:07.851045+00:00 | 2024-08-07 04:00:00+00:00 | create_emr_steps | exchange-dsp-reporting_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T03:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T03:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T03:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T03:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T03:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T03:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "4", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "3", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "4", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "3", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "4", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "3", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T03:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T03:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 04:00:19.550102+00:00 | 2024-08-08 03:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-2ATQFSMK8DP2P | 2024-08-08 04:00:26.648300+00:00 | 2024-08-07 04:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-dsp-reporting_v002 | ||
| return_value | j-16E34LGJRQ0Y5 | 2024-08-08 04:00:38.156653+00:00 | 2024-08-08 03:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T03:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 04:05:09.274344+00:00 | 2024-08-08 03:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-12SV3HNNWVJUI | 2024-08-08 04:05:27.312076+00:00 | 2024-08-08 03:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T04:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T04:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T04:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T04:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T04:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T04:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "5", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "4", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "5", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "4", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "5", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "4", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T04:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T04:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 05:00:29.226102+00:00 | 2024-08-08 04:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-10TLJFKSRE2J4 | 2024-08-08 05:00:48.136331+00:00 | 2024-08-08 04:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T04:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 05:05:12.655227+00:00 | 2024-08-08 04:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3H8KTSIQV3CQ2 | 2024-08-08 05:05:35.140394+00:00 | 2024-08-08 04:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2NVU3WV43Y2YQ | 2024-08-08 05:23:22.027370+00:00 | 2024-07-22 19:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-07-23T08:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.6.0/campaign-extractor-1.6.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-07-23T08:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.11.0/campaign-processing-job-1.11.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-07-23T08:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.22.0/pandora-campaign-delta-1.22.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2024-08-08 05:24:48.860788+00:00 | 2024-07-23 08:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T04:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.6.0/campaign-extractor-1.6.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T04:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.11.0/campaign-processing-job-1.11.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T04:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.22.0/pandora-campaign-delta-1.22.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2024-08-08 05:25:12.838701+00:00 | 2024-08-08 04:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-3RRJ413JQ2RSW | 2024-08-08 05:25:24.711415+00:00 | 2024-08-08 04:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T05:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T05:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T05:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T05:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T05:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T05:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "6", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "5", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "6", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "5", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "6", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "5", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T05:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T05:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 06:00:13.655251+00:00 | 2024-08-08 05:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-359M5XT10KCAU | 2024-08-08 06:00:34.242648+00:00 | 2024-08-08 05:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T05:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 06:05:07.835049+00:00 | 2024-08-08 05:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-11NJZAX4G3YZL | 2024-08-08 06:05:33.093635+00:00 | 2024-08-08 05:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T06:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T06:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T06:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T06:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T06:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T06:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "7", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "6", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "7", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "6", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "7", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "6", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T06:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T06:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 07:00:15.982075+00:00 | 2024-08-08 06:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-S8MV0P2TCKZD | 2024-08-08 07:00:36.439681+00:00 | 2024-08-08 06:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T06:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 07:05:20.916719+00:00 | 2024-08-08 06:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-N9CP808AJS84 | 2024-08-08 07:05:42.073609+00:00 | 2024-08-08 06:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T07:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T07:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T07:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T07:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T07:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T07:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "8", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "7", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "8", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "7", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "8", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "7", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T07:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T07:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 08:00:10.222141+00:00 | 2024-08-08 07:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1IMU4C01KHEX3 | 2024-08-08 08:00:57.980054+00:00 | 2024-08-08 07:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T07:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 08:05:25.565684+00:00 | 2024-08-08 07:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-21QTF8VC8Y6JK | 2024-08-08 08:05:51.996250+00:00 | 2024-08-08 07:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T08:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T08:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T08:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T08:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T08:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T08:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "9", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "8", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "9", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "8", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "9", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "8", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T08:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T08:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 09:00:15.608605+00:00 | 2024-08-08 08:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-daily-revenue-report", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 2, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "DailyRevenueReportJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T09:00:00+00:00", "--class", "com.viooh.smex.dailyrevenue.DailyRevenueReport", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/daily-revenue-report/1.3.0/daily-revenue-report-1.3.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-daily-revenue-report"}], "Configurations": []} | 2024-08-08 09:00:16.457592+00:00 | 2024-08-07 09:00:00+00:00 | create_emr_steps | exchange-daily-revenue-report_v002 | ||
| return_value | j-BZUKGCUW2C4D | 2024-08-08 09:00:34.287669+00:00 | 2024-08-08 08:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-19BA5U9T247IA | 2024-08-08 09:00:36.276974+00:00 | 2024-08-07 09:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-daily-revenue-report_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T08:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 09:05:08.059255+00:00 | 2024-08-08 08:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1A3BVT0OF1Y7C | 2024-08-08 09:05:28.354743+00:00 | 2024-08-08 08:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T09:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T09:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T09:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T09:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T09:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T09:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "10", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "9", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "10", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "9", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "10", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "9", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T09:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T09:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 10:00:22.955756+00:00 | 2024-08-08 09:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1JGDNCCASCG6 | 2024-08-08 10:00:42.354052+00:00 | 2024-08-08 09:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T09:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 10:05:10.140657+00:00 | 2024-08-08 09:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1FENYPTN28Q34 | 2024-08-08 10:05:31.848552+00:00 | 2024-08-08 09:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T10:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T10:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T10:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T10:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T10:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T10:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "11", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "10", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "11", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "10", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "11", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "10", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T10:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T10:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 11:00:22.167020+00:00 | 2024-08-08 10:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-3U6OS3NOWJ2T7 | 2024-08-08 11:00:40.624477+00:00 | 2024-08-08 10:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T10:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 11:05:24.947125+00:00 | 2024-08-08 10:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-371METALDIY4C | 2024-08-08 11:05:44.220804+00:00 | 2024-08-08 10:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-processing-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignExtractor", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T10:00:00+00:00", "--class", "com.viooh.campaignextractor.CampaignExtractorMain", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaignextractor/campaign-extractor/1.6.0/campaign-extractor-1.6.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignProcessingJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T10:00:00+00:00", "--class", "com.viooh.CampaignProcessingMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/campaign-processing-job/1.11.0/campaign-processing-job-1.11.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CampaignDeltaJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T10:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.22.0/pandora-campaign-delta-1.22.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-processing-job"}], "Configurations": []} | 2024-08-08 11:34:06.836920+00:00 | 2024-08-08 10:00:00+00:00 | create_emr_steps | booking-processing-job_v002 | ||
| return_value | j-3D99WDKZ3GASE | 2024-08-08 11:34:28.103064+00:00 | 2024-08-08 10:00:00+00:00 | create_cluster_and_add_emr_steps | booking-processing-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T11:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T11:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T11:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T11:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T11:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T11:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "12", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "11", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "12", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "11", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "12", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "11", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T11:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T11:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 12:00:07.896420+00:00 | 2024-08-08 11:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-37DJZR1P3CAF6 | 2024-08-08 12:00:30.055722+00:00 | 2024-08-08 11:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T11:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 12:05:10.698535+00:00 | 2024-08-08 11:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-145LDYYP1SYAE | 2024-08-08 12:05:32.578720+00:00 | 2024-08-08 11:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T12:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T12:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T12:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T12:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T12:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T12:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "13", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "12", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "13", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "12", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "13", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "12", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T12:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T12:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 13:00:08.175153+00:00 | 2024-08-08 12:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-ZJPXBP13AXPJ | 2024-08-08 13:00:30.459333+00:00 | 2024-08-08 12:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T12:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 13:05:06.858398+00:00 | 2024-08-08 12:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1Y1XI43IOZP0F | 2024-08-08 13:05:26.180681+00:00 | 2024-08-08 12:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T13:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T13:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T13:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T13:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T13:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T13:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "14", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "13", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "14", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "13", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "14", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "13", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T13:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T13:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 14:00:11.234367+00:00 | 2024-08-08 13:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1YEAXKIV8C46Z | 2024-08-08 14:00:30.059855+00:00 | 2024-08-08 13:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T13:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 14:05:14.631983+00:00 | 2024-08-08 13:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2BAQQ0U9UF63V | 2024-08-08 14:05:34.378168+00:00 | 2024-08-08 13:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T14:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T14:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T14:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T14:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T14:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T14:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "15", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "14", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "15", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "14", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "15", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "14", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T14:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T14:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 15:00:15.444114+00:00 | 2024-08-08 14:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-14AZJ5FZ5ZA0C | 2024-08-08 15:00:34.300184+00:00 | 2024-08-08 14:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T14:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 15:05:18.971815+00:00 | 2024-08-08 14:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-J4WDPTY7VRDW | 2024-08-08 15:05:41.529404+00:00 | 2024-08-08 14:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T15:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T15:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T15:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T15:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T15:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T15:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "16", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "15", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "16", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "15", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "16", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "15", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T15:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T15:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 16:00:18.884253+00:00 | 2024-08-08 15:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1P98OM6VADARU | 2024-08-08 16:00:37.537441+00:00 | 2024-08-08 15:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T15:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 16:05:22.324775+00:00 | 2024-08-08 15:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-25LERZNZE7ASU | 2024-08-08 16:05:41.717907+00:00 | 2024-08-08 15:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T16:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T16:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T16:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T16:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T16:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T16:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "17", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "16", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "17", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "16", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "17", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "16", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T16:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T16:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 17:00:07.892262+00:00 | 2024-08-08 16:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-88TZX8CVERMY | 2024-08-08 17:00:28.376276+00:00 | 2024-08-08 16:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T16:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 17:05:08.763466+00:00 | 2024-08-08 16:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-26XTZOPIMXVMD | 2024-08-08 17:05:30.318686+00:00 | 2024-08-08 16:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T17:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T17:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T17:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T17:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T17:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T17:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "18", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "17", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "18", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "17", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "18", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "17", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T17:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T17:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 18:00:13.294481+00:00 | 2024-08-08 17:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-3R8K5P7QIDQBY | 2024-08-08 18:00:34.446885+00:00 | 2024-08-08 17:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 |