| Key | Value | Timestamp | Execution Date | Task Id | Dag Id | ||
|---|---|---|---|---|---|---|---|
| return_value | j-2H711BZYZBS0F | 2024-08-21 04:00:44.850660+00:00 | 2024-08-21 03:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-21RP2SDWTCNJU | 2024-08-21 04:00:46.229792+00:00 | 2024-08-20 04:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-dsp-reporting_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T03:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-21 04:05:09.738962+00:00 | 2024-08-21 03:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-K75T95WU5BXI | 2024-08-21 04:05:28.457904+00:00 | 2024-08-21 03:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T04:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T04:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T04:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T04:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T04:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T04:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "5", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "4", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "5", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "4", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "5", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "4", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-21T04:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-21T04:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-21 05:00:24.213954+00:00 | 2024-08-21 04:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-3RBTKNU83ZKD8 | 2024-08-21 05:00:43.923303+00:00 | 2024-08-21 04:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T04:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-21 05:05:20.233535+00:00 | 2024-08-21 04:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-10KKBNW7BAL79 | 2024-08-21 05:05:42.028831+00:00 | 2024-08-21 04:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T05:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T05:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T05:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T05:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T05:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T05:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "6", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "5", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "6", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "5", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "6", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "5", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-21T05:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-21T05:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-21 06:00:07.892317+00:00 | 2024-08-21 05:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-3LCVO07CJT5EM | 2024-08-21 06:00:28.182922+00:00 | 2024-08-21 05:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T05:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-21 06:05:16.437732+00:00 | 2024-08-21 05:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3UX26H3UJILHS | 2024-08-21 06:05:38.507023+00:00 | 2024-08-21 05:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T06:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T06:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T06:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T06:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T06:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T06:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "7", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "6", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "7", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "6", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "7", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "6", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-21T06:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-21T06:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-21 07:00:20.529725+00:00 | 2024-08-21 06:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-2RQ9JT2AYRRE6 | 2024-08-21 07:00:41.103423+00:00 | 2024-08-21 06:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T06:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-21 07:05:21.379295+00:00 | 2024-08-21 06:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3L2D3J9ACL4D1 | 2024-08-21 07:11:18.051130+00:00 | 2024-08-21 06:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T07:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T07:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T07:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T07:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T07:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T07:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "8", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "7", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "8", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "7", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "8", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "7", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-21T07:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-21T07:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-21 08:00:25.950544+00:00 | 2024-08-21 07:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-3OV2O8ER3ZGQT | 2024-08-21 08:00:47.158400+00:00 | 2024-08-21 07:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T07:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-21 08:05:19.331790+00:00 | 2024-08-21 07:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1RW3I9WPHSRIS | 2024-08-21 08:05:36.059620+00:00 | 2024-08-21 07:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-daily-revenue-report", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 2, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "DailyRevenueReportJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-20T09:00:00+00:00", "--class", "com.viooh.smex.dailyrevenue.DailyRevenueReport", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/daily-revenue-report/1.3.0/daily-revenue-report-1.3.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-daily-revenue-report"}], "Configurations": []} | 2024-08-21 09:00:21.139908+00:00 | 2024-08-20 09:00:00+00:00 | create_emr_steps | exchange-daily-revenue-report_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T08:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T08:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T08:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T08:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T08:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T08:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "9", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "8", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "9", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "8", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "9", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "8", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-21T08:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-21T08:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-21 09:00:27.479015+00:00 | 2024-08-21 08:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-2Z9YAVDJODFI6 | 2024-08-21 09:00:42.867517+00:00 | 2024-08-20 09:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-daily-revenue-report_v002 | ||
| return_value | j-2N84X9D1VK0O9 | 2024-08-21 09:00:50.515910+00:00 | 2024-08-21 08:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T08:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-21 09:05:13.896531+00:00 | 2024-08-21 08:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3OA6HRIHEBZJ7 | 2024-08-21 09:05:32.820940+00:00 | 2024-08-21 08:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T09:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T09:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T09:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T09:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T09:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T09:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "10", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "9", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "10", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "9", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "10", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "9", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-21T09:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-21T09:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-21 10:00:12.522211+00:00 | 2024-08-21 09:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1MY6VW4YU7E3L | 2024-08-21 10:00:38.752858+00:00 | 2024-08-21 09:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T09:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-21 10:05:13.174524+00:00 | 2024-08-21 09:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1IAB8NYJE9JFD | 2024-08-21 10:05:35.656732+00:00 | 2024-08-21 09:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T10:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-21 11:05:13.248978+00:00 | 2024-08-21 10:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-39Z3UB22Y8PBL | 2024-08-21 11:05:34.583574+00:00 | 2024-08-21 10:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T10:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T10:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T10:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T10:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T10:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T10:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.11.0/exchange-trade-summary-4.11.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "11", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "10", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "11", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "10", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "11", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "10", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-21T10:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-21T10:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-08-21 11:38:01.760805+00:00 | 2024-08-21 10:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-3NCQ3HB177Q6 | 2024-08-21 11:38:22.338378+00:00 | 2024-08-21 10:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T11:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-21 12:05:10.453800+00:00 | 2024-08-21 11:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1P4SSYCUSB12H | 2024-08-21 12:05:32.784398+00:00 | 2024-08-21 11:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T12:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-21 13:05:20.913326+00:00 | 2024-08-21 12:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2DGK399LESBJW | 2024-08-21 13:05:40.757728+00:00 | 2024-08-21 12:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T13:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-21 14:05:18.505741+00:00 | 2024-08-21 13:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1U8S94RIS9SIR | 2024-08-21 14:05:38.311804+00:00 | 2024-08-21 13:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T11:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T11:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T11:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T11:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T11:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T11:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "12", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "11", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "12", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "11", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "12", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "11", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-21T11:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-21T11:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-08-21 14:31:12.983014+00:00 | 2024-08-21 11:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-STFV6Y2TYGHB | 2024-08-21 14:31:30.797082+00:00 | 2024-08-21 11:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T12:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T12:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T12:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T12:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T12:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T12:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "13", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "12", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "13", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "12", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "13", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "12", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-21T12:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-21T12:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-08-21 14:32:02.221764+00:00 | 2024-08-21 12:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-2RFQ4FGCRBP0 | 2024-08-21 14:50:14.755609+00:00 | 2024-08-21 12:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T13:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T13:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T13:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T13:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T13:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T13:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "14", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "13", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "14", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "13", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "14", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "13", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-21T13:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-21T13:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-08-21 14:50:35.586444+00:00 | 2024-08-21 13:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T14:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-21 15:05:19.326786+00:00 | 2024-08-21 14:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1TDC20BQ57WD4 | 2024-08-21 15:05:40.020793+00:00 | 2024-08-21 14:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | j-WWJ3DNE8M97O | 2024-08-21 15:08:58.658118+00:00 | 2024-08-21 13:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T14:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T14:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T14:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T14:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T14:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T14:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "15", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "14", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "15", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "14", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "15", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "14", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-21T14:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-21T14:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-08-21 15:09:33.469667+00:00 | 2024-08-21 14:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-TKZ1CBIIPPT | 2024-08-21 15:26:47.345618+00:00 | 2024-08-21 14:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T15:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T15:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T15:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T15:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T15:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T15:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "16", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "15", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "16", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "15", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "16", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "15", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-21T15:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-21T15:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-08-21 16:00:22.887906+00:00 | 2024-08-21 15:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-3EU3MN7E4DBS4 | 2024-08-21 16:00:42.808192+00:00 | 2024-08-21 15:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T15:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-21 16:05:28.211092+00:00 | 2024-08-21 15:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-364IF1PNPLW5E | 2024-08-21 16:05:48.470721+00:00 | 2024-08-21 15:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T16:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T16:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T16:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T16:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T16:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T16:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "17", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "16", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "17", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "16", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "17", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "16", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-21T16:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-21T16:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-08-21 17:00:23.919416+00:00 | 2024-08-21 16:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-3AO7QRN9J91PK | 2024-08-21 17:00:44.366091+00:00 | 2024-08-21 16:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T16:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-21 17:06:09.805069+00:00 | 2024-08-21 16:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1DZBA22LK73KT | 2024-08-21 17:06:30.178834+00:00 | 2024-08-21 16:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T17:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T17:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T17:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T17:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T17:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T17:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "18", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "17", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "18", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "17", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "18", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "17", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-21T17:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-21T17:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-08-21 18:00:28.307111+00:00 | 2024-08-21 17:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-11OPHH385RXLT | 2024-08-21 18:00:48.070696+00:00 | 2024-08-21 17:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T17:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-21 18:05:23.094521+00:00 | 2024-08-21 17:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-39K20OP3H4S9F | 2024-08-21 18:05:48.539906+00:00 | 2024-08-21 17:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T18:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T18:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T18:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T18:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T18:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T18:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "19", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "18", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "19", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "18", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "19", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "18", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-21T18:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-21T18:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-08-21 19:00:15.857878+00:00 | 2024-08-21 18:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-P1FHS0P6NSL0 | 2024-08-21 19:00:36.596195+00:00 | 2024-08-21 18:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T18:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-21 19:05:20.696722+00:00 | 2024-08-21 18:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2MA3INUVO9XZ4 | 2024-08-21 19:05:41.728941+00:00 | 2024-08-21 18:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T19:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T19:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T19:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T19:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T19:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T19:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "20", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "19", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "20", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "19", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "20", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "19", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-21T19:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-21T19:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-08-21 20:00:21.586856+00:00 | 2024-08-21 19:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-3E0EIE0S1R81H | 2024-08-21 20:00:42.573802+00:00 | 2024-08-21 19:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T19:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-21 20:05:14.805081+00:00 | 2024-08-21 19:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1SBXUAFPW6JEE | 2024-08-21 20:05:32.736341+00:00 | 2024-08-21 19:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T20:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T20:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T20:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T20:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T20:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T20:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "21", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "20", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "21", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "20", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "21", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "20", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-21T20:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-21T20:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-08-21 21:00:16.048258+00:00 | 2024-08-21 20:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-2E8UGI8SEBQP4 | 2024-08-21 21:00:37.689757+00:00 | 2024-08-21 20:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T20:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-21 21:05:14.945395+00:00 | 2024-08-21 20:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2BO99O2TFOJXL | 2024-08-21 21:05:33.782440+00:00 | 2024-08-21 20:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T21:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T21:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T21:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T21:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T21:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T21:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "22", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "21", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "22", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "21", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "22", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "21", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-21T21:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-21T21:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-08-21 22:00:19.634382+00:00 | 2024-08-21 21:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-3VL4TS4WLT2TN | 2024-08-21 22:00:44.269185+00:00 | 2024-08-21 21:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T21:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-21 22:05:28.314540+00:00 | 2024-08-21 21:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2JCEILPEGGJ1X | 2024-08-21 22:05:51.470232+00:00 | 2024-08-21 21:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-delta-past-delete-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignDeltaPastDeleteJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-20T23:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaPastDeleteMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.22.0/pandora-campaign-delta-1.22.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-delta-past-delete-job"}], "Configurations": []} | 2024-08-21 23:00:11.396852+00:00 | 2024-08-20 23:00:00+00:00 | create_emr_steps | campaign-delta-past-delete-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T22:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T22:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T22:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T22:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T22:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T22:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "23", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "22", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "23", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "22", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "23", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "22", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-21T22:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-21T22:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-08-21 23:00:14.555171+00:00 | 2024-08-21 22:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-F2HGF428X119 | 2024-08-21 23:00:34.494298+00:00 | 2024-08-20 23:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-delta-past-delete-job_v002 | ||
| return_value | j-2M4GP5CRNDR43 | 2024-08-21 23:00:38.078882+00:00 | 2024-08-21 22:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T22:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-21 23:05:10.488105+00:00 | 2024-08-21 22:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-191RJAS1192TN | 2024-08-21 23:05:36.133721+00:00 | 2024-08-21 22:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T23:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T23:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T23:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T23:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T23:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T23:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-22", "0", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "23", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-22", "0", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "23", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-22", "0", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-21", "23", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-21T23:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-21T23:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-08-22 00:00:17.562119+00:00 | 2024-08-21 23:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1HPH4VENK2AKY | 2024-08-22 00:00:38.288320+00:00 | 2024-08-21 23:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T23:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-22 00:05:15.633199+00:00 | 2024-08-21 23:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-109SL17OFGX11 | 2024-08-22 00:05:38.052998+00:00 | 2024-08-21 23:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-22T00:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-22T00:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-22T00:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-22T00:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-22T00:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-22T00:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-22", "1", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-22", "0", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-22", "1", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-22", "0", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-22", "1", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-22", "0", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-22T00:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-22T00:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-08-22 01:00:14.157917+00:00 | 2024-08-22 00:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-15UDTINL5HMRX | 2024-08-22 01:00:45.846805+00:00 | 2024-08-22 00:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-22T00:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-22 01:05:28.830130+00:00 | 2024-08-22 00:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2HR2YJ3NI668E | 2024-08-22 01:05:58.987733+00:00 | 2024-08-22 00:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-by-hour-model-transform", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "BookingByHour-ModelTransform", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=20g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-21T01:30:00+00:00", "--class", "com.viooh.transform.BookingModelTransform", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/uk/co/viooh/pandora-model-transform/1.7.0/pandora-model-transform-1.7.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-by-hour-model-transform"}], "Configurations": []} | 2024-08-22 01:30:37.891605+00:00 | 2024-08-21 01:30:00+00:00 | create_emr_steps | booking-by-hour-model-transform_v002 | ||
| return_value | j-38FT7R5QW2WU4 | 2024-08-22 01:31:05.090591+00:00 | 2024-08-21 01:30:00+00:00 | create_cluster_and_add_emr_steps | booking-by-hour-model-transform_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-22T01:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-22T01:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-22T01:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-22T01:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-22T01:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-22T01:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-22", "2", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-22", "1", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-22", "2", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-22", "1", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-22", "2", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-22", "1", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-22T01:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-22T01:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-08-22 02:00:10.680235+00:00 | 2024-08-22 01:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-205Y5VW8K1YIL | 2024-08-22 02:00:37.133988+00:00 | 2024-08-22 01:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-22T01:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-22 02:05:18.050790+00:00 | 2024-08-22 01:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-252JJQN0FGMJF | 2024-08-22 02:05:41.297069+00:00 | 2024-08-22 01:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-22T02:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-22T02:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-22T02:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-22T02:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-22T02:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-22T02:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-22", "3", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-22", "2", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-22", "3", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-22", "2", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-22", "3", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-22", "2", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-22T02:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-22T02:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-08-22 03:00:31.705303+00:00 | 2024-08-22 02:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-8G45SEYUSDDM | 2024-08-22 03:00:50.502365+00:00 | 2024-08-22 02:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 |