| Key | Value | Timestamp | Execution Date | Task Id | Dag Id | ||
|---|---|---|---|---|---|---|---|
| return_value | j-2DZWLIQ10JV0O | 2024-09-25 06:00:31.422351+00:00 | 2024-09-25 05:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T05:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-25 06:05:07.780389+00:00 | 2024-09-25 05:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3F4HW9DA38G9Z | 2024-09-25 06:05:29.696438+00:00 | 2024-09-25 05:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T06:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T06:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T06:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T06:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T06:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T06:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "7", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "6", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "7", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "6", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "7", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "6", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-25T06:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-25T06:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-25 07:00:21.404128+00:00 | 2024-09-25 06:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-NAHZO874OU6D | 2024-09-25 07:00:36.245658+00:00 | 2024-09-25 06:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T06:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-25 07:05:16.604630+00:00 | 2024-09-25 06:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-TKR1TWTTRPW | 2024-09-25 07:05:38.395550+00:00 | 2024-09-25 06:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T07:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T07:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T07:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T07:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T07:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T07:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "8", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "7", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "8", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "7", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "8", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "7", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-25T07:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-25T07:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-25 08:00:19.120636+00:00 | 2024-09-25 07:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-20VCOC877KVMX | 2024-09-25 08:00:40.011238+00:00 | 2024-09-25 07:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T07:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-25 08:05:24.384509+00:00 | 2024-09-25 07:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-Q9JZMKAXS6IA | 2024-09-25 08:05:44.160652+00:00 | 2024-09-25 07:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T08:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T08:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T08:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T08:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T08:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T08:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "9", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "8", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "9", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "8", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "9", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "8", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-25T08:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-25T08:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-25 09:00:13.304314+00:00 | 2024-09-25 08:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-daily-revenue-report", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 2, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "DailyRevenueReportJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-24T09:00:00+00:00", "--class", "com.viooh.smex.dailyrevenue.DailyRevenueReport", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/daily-revenue-report/1.3.0/daily-revenue-report-1.3.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-daily-revenue-report"}], "Configurations": []} | 2024-09-25 09:00:26.461120+00:00 | 2024-09-24 09:00:00+00:00 | create_emr_steps | exchange-daily-revenue-report_v002 | ||
| return_value | j-2YOE7HFT23MGY | 2024-09-25 09:00:31.992896+00:00 | 2024-09-25 08:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-KPW9CYGKFBBK | 2024-09-25 09:00:53.307736+00:00 | 2024-09-24 09:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-daily-revenue-report_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T08:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-25 09:05:22.480106+00:00 | 2024-09-25 08:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-32NUIP4NZE66V | 2024-09-25 09:05:45.917347+00:00 | 2024-09-25 08:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T09:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T09:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T09:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T09:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T09:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T09:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "10", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "9", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "10", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "9", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "10", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "9", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-25T09:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-25T09:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-25 10:00:17.563270+00:00 | 2024-09-25 09:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-2WIVZ6WVD11QC | 2024-09-25 10:00:38.339199+00:00 | 2024-09-25 09:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T09:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-25 10:05:20.573986+00:00 | 2024-09-25 09:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3UFKNX0JPBGLC | 2024-09-25 10:05:42.420360+00:00 | 2024-09-25 09:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T10:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T10:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T10:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T10:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T10:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T10:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "11", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "10", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "11", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "10", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "11", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "10", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-25T10:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-25T10:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-25 11:00:30.616459+00:00 | 2024-09-25 10:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1JSWZX2WHFKFJ | 2024-09-25 11:00:56.144755+00:00 | 2024-09-25 10:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T10:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-25 11:05:25.247956+00:00 | 2024-09-25 10:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-32J7CNP61YOVB | 2024-09-25 11:05:46.864820+00:00 | 2024-09-25 10:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T11:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T11:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T11:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T11:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T11:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T11:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "12", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "11", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "12", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "11", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "12", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "11", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-25T11:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-25T11:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-25 12:00:21.417534+00:00 | 2024-09-25 11:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1KNCHZKMTLKAC | 2024-09-25 12:00:42.533097+00:00 | 2024-09-25 11:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T11:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-25 12:05:18.782344+00:00 | 2024-09-25 11:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1UNGCY9XTKO8Q | 2024-09-25 12:05:42.016731+00:00 | 2024-09-25 11:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T12:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T12:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T12:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T12:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T12:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T12:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "13", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "12", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "13", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "12", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "13", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "12", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-25T12:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-25T12:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-25 13:00:25.745369+00:00 | 2024-09-25 12:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1Y0B8PL108SXA | 2024-09-25 13:00:45.701861+00:00 | 2024-09-25 12:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T12:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-25 13:05:14.818766+00:00 | 2024-09-25 12:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-MQONNPZ6F5D5 | 2024-09-25 13:05:39.534817+00:00 | 2024-09-25 12:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T13:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T13:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T13:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T13:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T13:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T13:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "14", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "13", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "14", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "13", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "14", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "13", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-25T13:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-25T13:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-25 14:00:25.730984+00:00 | 2024-09-25 13:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1GC40D9W7QWE4 | 2024-09-25 14:00:45.686510+00:00 | 2024-09-25 13:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T13:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-25 14:05:22.384010+00:00 | 2024-09-25 13:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1MNORUVUQ2KKW | 2024-09-25 14:05:42.577544+00:00 | 2024-09-25 13:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T14:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T14:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T14:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T14:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T14:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T14:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "15", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "14", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "15", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "14", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "15", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "14", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-25T14:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-25T14:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-25 15:00:18.111564+00:00 | 2024-09-25 14:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1N46DCF3HP1W3 | 2024-09-25 15:00:35.708523+00:00 | 2024-09-25 14:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T14:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-25 15:05:20.222108+00:00 | 2024-09-25 14:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-83YCIW0XKFPE | 2024-09-25 15:05:41.548604+00:00 | 2024-09-25 14:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T15:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T15:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T15:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T15:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T15:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T15:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "16", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "15", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "16", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "15", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "16", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "15", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-25T15:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-25T15:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-25 16:00:26.982316+00:00 | 2024-09-25 15:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-EDXDTA3WSAA0 | 2024-09-25 16:00:57.110587+00:00 | 2024-09-25 15:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T15:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-25 16:05:28.223466+00:00 | 2024-09-25 15:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-306VKJ3OQADZU | 2024-09-25 16:05:55.974076+00:00 | 2024-09-25 15:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T16:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T16:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T16:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T16:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T16:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T16:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "17", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "16", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "17", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "16", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "17", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "16", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-25T16:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-25T16:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-25 17:00:21.814481+00:00 | 2024-09-25 16:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-9O683LPWTPA4 | 2024-09-25 17:00:38.453308+00:00 | 2024-09-25 16:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T16:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-25 17:05:24.342822+00:00 | 2024-09-25 16:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3CIS7Z5SH05PK | 2024-09-25 17:05:45.802839+00:00 | 2024-09-25 16:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T17:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T17:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T17:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T17:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T17:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T17:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "18", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "17", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "18", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "17", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "18", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "17", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-25T17:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-25T17:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-25 18:00:22.337258+00:00 | 2024-09-25 17:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-2P13LFVDRIFO1 | 2024-09-25 18:00:41.764660+00:00 | 2024-09-25 17:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T17:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-25 18:05:23.315843+00:00 | 2024-09-25 17:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1IN4QMHX6MWM9 | 2024-09-25 18:05:44.624186+00:00 | 2024-09-25 17:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T18:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T18:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T18:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T18:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T18:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T18:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "19", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "18", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "19", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "18", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "19", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "18", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-25T18:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-25T18:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-25 19:00:19.837097+00:00 | 2024-09-25 18:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-2OY7276DLZH05 | 2024-09-25 19:00:39.736384+00:00 | 2024-09-25 18:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T18:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-25 19:05:20.861548+00:00 | 2024-09-25 18:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-23YWUJHTXMHLO | 2024-09-25 19:05:41.972618+00:00 | 2024-09-25 18:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T19:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T19:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T19:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T19:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T19:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T19:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "20", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "19", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "20", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "19", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "20", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "19", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-25T19:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-25T19:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-25 20:00:16.660594+00:00 | 2024-09-25 19:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-J13EFNCIGE9D | 2024-09-25 20:00:36.634232+00:00 | 2024-09-25 19:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T19:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-25 20:05:18.786905+00:00 | 2024-09-25 19:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1PR7CL1J1F45P | 2024-09-25 20:05:40.680074+00:00 | 2024-09-25 19:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T20:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T20:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T20:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T20:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T20:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T20:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "21", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "20", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "21", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "20", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "21", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "20", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-25T20:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-25T20:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-25 21:00:17.991690+00:00 | 2024-09-25 20:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1RXVVGV8E2MOD | 2024-09-25 21:00:37.951119+00:00 | 2024-09-25 20:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T20:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-25 21:05:08.343835+00:00 | 2024-09-25 20:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-AT6BFMCDSO2W | 2024-09-25 21:05:31.559497+00:00 | 2024-09-25 20:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T21:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T21:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T21:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T21:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T21:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T21:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "22", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "21", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "22", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "21", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "22", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "21", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-25T21:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-25T21:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-25 22:00:27.353950+00:00 | 2024-09-25 21:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-2BF878EE6RMCH | 2024-09-25 22:00:57.299556+00:00 | 2024-09-25 21:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T21:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-25 22:05:06.642237+00:00 | 2024-09-25 21:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3L8HJL9ZOYL6Z | 2024-09-25 22:05:27.488099+00:00 | 2024-09-25 21:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T22:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T22:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T22:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T22:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T22:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T22:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "23", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "22", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "23", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "22", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "23", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "22", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-25T22:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-25T22:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-25 23:00:09.227365+00:00 | 2024-09-25 22:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-delta-past-delete-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignDeltaPastDeleteJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-24T23:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaPastDeleteMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.22.0/pandora-campaign-delta-1.22.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-delta-past-delete-job"}], "Configurations": []} | 2024-09-25 23:00:21.353870+00:00 | 2024-09-24 23:00:00+00:00 | create_emr_steps | campaign-delta-past-delete-job_v002 | ||
| return_value | j-1JAU2K9DIW2T5 | 2024-09-25 23:00:31.427353+00:00 | 2024-09-25 22:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1FKEMKWICHVXF | 2024-09-25 23:00:43.269122+00:00 | 2024-09-24 23:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-delta-past-delete-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T22:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-25 23:05:25.871080+00:00 | 2024-09-25 22:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1IGNKJ6IF9JYS | 2024-09-25 23:05:52.246716+00:00 | 2024-09-25 22:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T23:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T23:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T23:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T23:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T23:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T23:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "0", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "23", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "0", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "23", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "0", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-25", "23", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-25T23:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-25T23:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-26 00:00:24.735205+00:00 | 2024-09-25 23:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-38JTLACZFS93O | 2024-09-26 00:00:42.547180+00:00 | 2024-09-25 23:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T23:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-26 00:05:16.907646+00:00 | 2024-09-25 23:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-I3LYCRKIV53W | 2024-09-26 00:05:39.016104+00:00 | 2024-09-25 23:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T00:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T00:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T00:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T00:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T00:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T00:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "1", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "0", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "1", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "0", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "1", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "0", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-26T00:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-26T00:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-26 01:00:26.169675+00:00 | 2024-09-26 00:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1RA99PKRVYCKY | 2024-09-26 01:00:52.838289+00:00 | 2024-09-26 00:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T00:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-26 01:05:27.341903+00:00 | 2024-09-26 00:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-306FG4UFDR6D4 | 2024-09-26 01:05:54.842007+00:00 | 2024-09-26 00:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-by-hour-model-transform", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "BookingByHour-ModelTransform", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=20g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T01:30:00+00:00", "--class", "com.viooh.transform.BookingModelTransform", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/uk/co/viooh/pandora-model-transform/1.7.0/pandora-model-transform-1.7.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-by-hour-model-transform"}], "Configurations": []} | 2024-09-26 01:30:25.649971+00:00 | 2024-09-25 01:30:00+00:00 | create_emr_steps | booking-by-hour-model-transform_v002 | ||
| return_value | j-SAE45LHLNQWA | 2024-09-26 01:30:47.294629+00:00 | 2024-09-25 01:30:00+00:00 | create_cluster_and_add_emr_steps | booking-by-hour-model-transform_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T01:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T01:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T01:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T01:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T01:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T01:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "2", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "1", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "2", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "1", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "2", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "1", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-26T01:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-26T01:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-26 02:00:19.567863+00:00 | 2024-09-26 01:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-3EVRHNVWHU28Y | 2024-09-26 02:00:39.044048+00:00 | 2024-09-26 01:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T01:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-26 02:05:28.449102+00:00 | 2024-09-26 01:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-22AKWJK4ABUBE | 2024-09-26 02:05:50.822736+00:00 | 2024-09-26 01:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T02:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T02:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T02:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T02:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T02:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T02:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "3", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "2", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "3", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "2", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "3", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "2", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-26T02:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-26T02:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-26 03:00:12.120847+00:00 | 2024-09-26 02:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-30V3DLREBJSO4 | 2024-09-26 03:00:31.816021+00:00 | 2024-09-26 02:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T02:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-26 03:05:26.705933+00:00 | 2024-09-26 02:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-31NGB6N6MHOY4 | 2024-09-26 03:05:49.530015+00:00 | 2024-09-26 02:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T03:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T03:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T03:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T03:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T03:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T03:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "4", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "3", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "4", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "3", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "4", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "3", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-26T03:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-26T03:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-26 04:00:19.528323+00:00 | 2024-09-26 03:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-dsp-reporting", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "DSPExchangeTradeJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-25T04:00:00+00:00", "--class", "com.viooh.smex.dsp.DSPExchangeTrade", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dsp-exchange-trade/3.8.0/dsp-exchange-trade-3.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopyDSPReports", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-dsp-reporting/1.0.0/job.sh", "--environment", "uat", "--reportdate", "2024-09-25", "--region", "cn", "--skipFailure"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-dsp-reporting"}], "Configurations": [{"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-26 04:00:26.467339+00:00 | 2024-09-25 04:00:00+00:00 | create_emr_steps | exchange-dsp-reporting_v002 | ||
| return_value | j-15G6LSHHEKGGT | 2024-09-26 04:00:33.817729+00:00 | 2024-09-26 03:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-3M2ZJTGT56GZX | 2024-09-26 04:00:43.263923+00:00 | 2024-09-25 04:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-dsp-reporting_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T03:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.14.0/daily-import-1.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-09-26 04:05:11.267012+00:00 | 2024-09-26 03:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1FLKQ32ECHNYA | 2024-09-26 04:05:30.883716+00:00 | 2024-09-26 03:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T04:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/3.9.0/supply-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T04:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/3.9.0/demand-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T04:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/3.9.0/dealsync-denormalization-3.9.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T04:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/3.10.0/moderation-denormalization-3.10.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T04:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/connector/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/2.8.0/deal-denormalization-2.8.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-09-26T04:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/4.12.0/exchange-trade-summary-4.12.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "5", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "4", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "5", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "4", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "5", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-09-26", "4", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-09-26T04:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-09-26T04:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}, {"Classification": "hadoop-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}, {"Classification": "spark-env", "Configurations": [{"Classification": "export", "Configurations": [], "Properties": {"JAVA_HOME": "/usr/lib/jvm/java-11-amazon-corretto.x86_64"}}], "Properties": {}}]} | 2024-09-26 05:00:30.733472+00:00 | 2024-09-26 04:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 |